From 72672d22f3852349e166ab930efa3617945d82f0 Mon Sep 17 00:00:00 2001 From: Linfang Wang Date: Thu, 23 Jul 2026 13:47:55 +0800 Subject: [PATCH 01/19] benchmark: add Codex CLI model adapter --- benchmarks/discover-call/README.md | 29 ++- .../discover-call/adapters/codex-cli.mjs | 193 ++++++++++++++++++ .../discover-call/test/adapter.test.mjs | 88 ++++++++ 3 files changed, 309 insertions(+), 1 deletion(-) create mode 100644 benchmarks/discover-call/adapters/codex-cli.mjs diff --git a/benchmarks/discover-call/README.md b/benchmarks/discover-call/README.md index 8033a67..585a263 100644 --- a/benchmarks/discover-call/README.md +++ b/benchmarks/discover-call/README.md @@ -127,9 +127,36 @@ print prompts, keys, tokens, or provider error bodies to stdout. A first-result heuristic adapter is included only as a transport example; it does not construct parameters and is not an official model result. +### Codex CLI adapter + +`adapters/codex-cli.mjs` supports authenticated Codex CLI installations. It +maps the canonical system message to `developer_instructions`, sends the +canonical user message unchanged on stdin, and writes the unchanged response +schema to a temporary file for `--output-schema`. The adapter uses an ephemeral +session in an isolated temporary directory, ignores user configuration and +rules, uses a read-only sandbox, and rejects runs that emit tool-use events. +Reasoning effort is fixed at `medium`; record the exact Codex CLI version +alongside the adapter commit in `--adapter-revision`: + +```bash +node src/run.mjs \ + --model gpt-5.6-sol \ + --adapter node \ + --adapter-arg "$PWD/adapters/codex-cli.mjs" \ + --adapter-revision adapter-git-sha/codex-cli-0.144.1/medium \ + --tasks tasks/v2.jsonl \ + --trials 3 \ + --execute \ + --output runs/gpt-5.6-sol.jsonl +``` + +The adapter reuses the CLI's existing authentication and never receives +`QVERIS_API_KEY` (both the harness and adapter remove it from the model +subprocess environment). + ### Claude CLI adapter -`adapters/claude-cli.mjs` is the reference adapter for authenticated Claude CLI +`adapters/claude-cli.mjs` remains available for authenticated Claude CLI installations. It passes the canonical system message as `--system-prompt`, the canonical user message on stdin, and the unchanged response schema through `--json-schema`. Safe mode disables project customizations, tools are disabled, diff --git a/benchmarks/discover-call/adapters/codex-cli.mjs b/benchmarks/discover-call/adapters/codex-cli.mjs new file mode 100644 index 0000000..4ba0a58 --- /dev/null +++ b/benchmarks/discover-call/adapters/codex-cli.mjs @@ -0,0 +1,193 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +export const CODEX_REASONING_EFFORT = 'medium'; + +const ALLOWED_ITEM_TYPES = new Set(['agent_message', 'reasoning']); + +export function buildCodexInvocation(payload, { schemaPath, workingDirectory }) { + if (!payload || typeof payload !== 'object') throw new Error('Invalid adapter payload'); + if (typeof payload.model !== 'string' || !payload.model.trim()) throw new Error('Missing model'); + if (!Array.isArray(payload.messages) || payload.messages.length !== 2) { + throw new Error('Expected exactly two canonical messages'); + } + const [systemMessage, userMessage] = payload.messages; + if (systemMessage?.role !== 'system' || typeof systemMessage.content !== 'string') { + throw new Error('Missing canonical system message'); + } + if (userMessage?.role !== 'user' || typeof userMessage.content !== 'string') { + throw new Error('Missing canonical user message'); + } + if (!isObject(payload.response_schema)) throw new Error('Missing response schema'); + if (typeof schemaPath !== 'string' || !schemaPath) throw new Error('Missing schema path'); + if (typeof workingDirectory !== 'string' || !workingDirectory) { + throw new Error('Missing working directory'); + } + + return { + command: process.env.CODEX_BIN || 'codex', + args: [ + 'exec', + '--model', + payload.model, + '--config', + `developer_instructions=${JSON.stringify(systemMessage.content)}`, + '--config', + `model_reasoning_effort=${JSON.stringify(CODEX_REASONING_EFFORT)}`, + '--ephemeral', + '--ignore-user-config', + '--ignore-rules', + '--strict-config', + '--sandbox', + 'read-only', + '--skip-git-repo-check', + '--cd', + workingDirectory, + '--output-schema', + schemaPath, + '--color', + 'never', + '--json', + '-', + ], + schema: payload.response_schema, + stdin: userMessage.content, + }; +} + +export function parseCodexEvents(stdout) { + const lines = stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + if (lines.length === 0) throw new Error('Codex CLI returned no events'); + + let finalMessage; + for (const line of lines) { + let event; + try { + event = JSON.parse(line); + } catch { + throw new Error('Codex CLI returned invalid JSONL'); + } + if (!isObject(event)) throw new Error('Codex CLI returned an invalid event'); + if (event.type === 'error' || event.type === 'turn.failed') { + throw new Error('Codex CLI reported an unsuccessful result'); + } + + if ((event.type === 'item.started' || event.type === 'item.completed') && isObject(event.item)) { + if (!ALLOWED_ITEM_TYPES.has(event.item.type)) { + throw new Error('Codex CLI attempted to use a tool'); + } + if (event.type === 'item.completed' && event.item.type === 'agent_message') { + if (typeof event.item.text !== 'string') { + throw new Error('Codex CLI returned an invalid agent message'); + } + finalMessage = event.item.text; + } + } + } + + if (typeof finalMessage !== 'string') throw new Error('Codex CLI returned no structured output'); + try { + const result = JSON.parse(finalMessage); + if (isObject(result)) return result; + } catch { + // Fall through to the generic, non-sensitive error below. + } + throw new Error('Codex CLI returned no structured output'); +} + +export async function invokeCodex(payload) { + const workingDirectory = await mkdtemp(join(tmpdir(), 'qveris-codex-adapter-')); + const schemaPath = join(workingDirectory, 'response-schema.json'); + try { + const invocation = buildCodexInvocation(payload, { schemaPath, workingDirectory }); + await writeFile(schemaPath, JSON.stringify(invocation.schema), { encoding: 'utf8', mode: 0o600 }); + const stdout = await runCodex(invocation); + return parseCodexEvents(stdout); + } finally { + await rm(workingDirectory, { recursive: true, force: true }); + } +} + +function runCodex(invocation) { + return new Promise((resolve, reject) => { + const env = { ...process.env }; + delete env.QVERIS_API_KEY; + const child = spawn(invocation.command, invocation.args, { + shell: false, + stdio: ['pipe', 'pipe', 'pipe'], + env, + }); + let output = ''; + let outputExceeded = false; + let forwardedSignal; + let forceKillTimer; + + const cleanup = () => { + process.off('SIGINT', forwardSigint); + process.off('SIGTERM', forwardSigterm); + if (forceKillTimer) clearTimeout(forceKillTimer); + }; + const forwardSignal = (signal) => { + if (forwardedSignal) return; + forwardedSignal = signal; + child.kill(signal); + forceKillTimer = setTimeout(() => child.kill('SIGKILL'), 1_000); + forceKillTimer.unref(); + }; + const forwardSigint = () => forwardSignal('SIGINT'); + const forwardSigterm = () => forwardSignal('SIGTERM'); + process.once('SIGINT', forwardSigint); + process.once('SIGTERM', forwardSigterm); + + child.stdin.on('error', () => {}); + child.stdout.on('data', (chunk) => { + output += chunk; + if (output.length > 1_000_000 && !outputExceeded) { + outputExceeded = true; + child.kill('SIGTERM'); + } + }); + child.stderr.resume(); + child.on('error', () => { + cleanup(); + reject(new Error('Codex CLI could not be started')); + }); + child.on('close', (code) => { + cleanup(); + if (forwardedSignal) { + process.kill(process.pid, forwardedSignal); + return; + } + if (outputExceeded) return reject(new Error('Codex CLI output exceeded the adapter limit')); + if (code === 0) resolve(output); + else reject(new Error('Codex CLI invocation failed')); + }); + child.stdin.end(invocation.stdin); + }); +} + +function isObject(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +async function main() { + let input = ''; + for await (const chunk of process.stdin) input += chunk; + const result = await invokeCodex(JSON.parse(input)); + process.stdout.write(JSON.stringify(result)); +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + main().catch((error) => { + process.stderr.write(`${error.message}\n`); + process.exitCode = 1; + }); +} diff --git a/benchmarks/discover-call/test/adapter.test.mjs b/benchmarks/discover-call/test/adapter.test.mjs index 0a13cdd..574d13d 100644 --- a/benchmarks/discover-call/test/adapter.test.mjs +++ b/benchmarks/discover-call/test/adapter.test.mjs @@ -5,6 +5,11 @@ import { fileURLToPath } from 'node:url'; import { createProcessAdapter, main } from '../src/run.mjs'; import { buildClaudeInvocation, parseClaudeEnvelope } from '../adapters/claude-cli.mjs'; +import { + buildCodexInvocation, + CODEX_REASONING_EFFORT, + parseCodexEvents, +} from '../adapters/codex-cli.mjs'; const adapterPath = resolve(fileURLToPath(new URL('../adapters/first-result.mjs', import.meta.url))); @@ -96,3 +101,86 @@ test('Claude adapter extracts only structured model output', () => { ); assert.throws(() => parseClaudeEnvelope(JSON.stringify({ result: 'not-json' })), /no structured output/); }); + +test('Codex adapter preserves canonical messages and response schema', () => { + const payload = { + model: 'gpt-5.6-sol', + messages: [ + { role: 'system', content: 'Return one grounded tool.' }, + { role: 'user', content: '{"input":"unchanged"}' }, + ], + response_schema: { + type: 'object', + additionalProperties: false, + required: ['tool_id'], + properties: { tool_id: { type: 'string' } }, + }, + }; + + const invocation = buildCodexInvocation(payload, { + schemaPath: '/tmp/response-schema.json', + workingDirectory: '/tmp/codex-adapter', + }); + const configs = invocation.args + .map((value, index) => (value === '--config' ? invocation.args[index + 1] : undefined)) + .filter(Boolean); + + assert.equal(invocation.stdin, payload.messages[1].content); + assert.deepEqual(invocation.schema, payload.response_schema); + assert.equal(invocation.args[invocation.args.indexOf('--model') + 1], payload.model); + assert.equal(invocation.args[invocation.args.indexOf('--output-schema') + 1], '/tmp/response-schema.json'); + assert.equal( + configs.includes(`developer_instructions=${JSON.stringify(payload.messages[0].content)}`), + true, + ); + assert.equal( + configs.includes(`model_reasoning_effort=${JSON.stringify(CODEX_REASONING_EFFORT)}`), + true, + ); + assert.equal(invocation.args.includes('--ephemeral'), true); + assert.equal(invocation.args.includes('--ignore-user-config'), true); + assert.equal(invocation.args.includes('--ignore-rules'), true); + assert.equal(invocation.args[invocation.args.indexOf('--sandbox') + 1], 'read-only'); +}); + +test('Codex adapter extracts the final structured message and rejects tool use', () => { + assert.deepEqual( + parseCodexEvents( + [ + JSON.stringify({ type: 'thread.started', thread_id: 'test' }), + JSON.stringify({ type: 'turn.started' }), + JSON.stringify({ type: 'item.completed', item: { type: 'reasoning', text: 'internal' } }), + JSON.stringify({ + type: 'item.completed', + item: { type: 'agent_message', text: '{"tool_id":"weather.forecast"}' }, + }), + JSON.stringify({ type: 'turn.completed', usage: { input_tokens: 1, output_tokens: 1 } }), + ].join('\n'), + ), + { tool_id: 'weather.forecast' }, + ); + assert.throws( + () => + parseCodexEvents( + JSON.stringify({ + type: 'item.started', + item: { type: 'command_execution', command: 'pwd' }, + }), + ), + /attempted to use a tool/, + ); + assert.throws( + () => parseCodexEvents(JSON.stringify({ type: 'turn.failed', error: { message: 'private' } })), + /unsuccessful result/, + ); + assert.throws( + () => + parseCodexEvents( + JSON.stringify({ + type: 'item.completed', + item: { type: 'agent_message', text: 'not-json' }, + }), + ), + /no structured output/, + ); +}); From 4508c3b782e1e7d352d01e3cf8de7d718f52f8f6 Mon Sep 17 00:00:00 2001 From: Linfang Wang Date: Thu, 23 Jul 2026 13:53:18 +0800 Subject: [PATCH 02/19] benchmark: generate strict parameter schemas --- benchmarks/discover-call/README.md | 7 ++ benchmarks/discover-call/src/harness.mjs | 89 +++++++++++++++++-- .../discover-call/test/harness.test.mjs | 65 +++++++++++++- 3 files changed, 150 insertions(+), 11 deletions(-) diff --git a/benchmarks/discover-call/README.md b/benchmarks/discover-call/README.md index 585a263..77141b5 100644 --- a/benchmarks/discover-call/README.md +++ b/benchmarks/discover-call/README.md @@ -122,6 +122,13 @@ user prompt and the inspected `selected_tool`. Return: {"parameters":{"city":"London"}} ``` +For parameterization, the harness derives a strict response schema from the +inspected tool's parameter metadata. Every known parameter is represented; +optional parameters are nullable because strict structured-output providers +require every declared property to appear. The harness removes top-level null +values before scoring and execution, so omitted optional parameters retain +their normal API semantics. + Adapters may call any model provider, but must not alter canonical messages or print prompts, keys, tokens, or provider error bodies to stdout. A first-result heuristic adapter is included only as a transport example; it does not construct diff --git a/benchmarks/discover-call/src/harness.mjs b/benchmarks/discover-call/src/harness.mjs index b076a98..dcd3514 100644 --- a/benchmarks/discover-call/src/harness.mjs +++ b/benchmarks/discover-call/src/harness.mjs @@ -116,14 +116,14 @@ async function runTrial({ task, model, trial, execute, limit, api, invokeAdapter if (!isPlainObject(parameterized?.parameters)) { throw stageError('parameterize', 'Adapter did not return a parameters object'); } - record.parameters = parameterized.parameters; + record.parameters = omitNullParameters(parameterized.parameters); if (execute) { record.call.attempted = true; const response = await api.call({ toolId: selectedToolId, discoveryId: discovered?.search_id, - parameters: parameterized.parameters, + parameters: record.parameters, }); record.call.success = response?.success === true; record.call.execution_id = response?.execution_id ?? null; @@ -146,6 +146,29 @@ function requiredParameterNames(tool) { .map((parameter) => parameter.name); } +export function parameterResponseSchema(tool) { + const properties = {}; + for (const parameter of array(tool?.params)) { + if (typeof parameter?.name !== 'string' || !parameter.name || Object.hasOwn(properties, parameter.name)) { + continue; + } + properties[parameter.name] = strictParameterSchema(parameter); + } + return { + type: 'object', + additionalProperties: false, + required: ['parameters'], + properties: { + parameters: { + type: 'object', + additionalProperties: false, + required: Object.keys(properties), + properties, + }, + }, + }; +} + function adapterPayload({ stage, model, task, discovery, selectedTool, discoveryId, trial }) { const input = stage === 'select' @@ -159,12 +182,7 @@ function adapterPayload({ stage, model, task, discovery, selectedTool, discovery required: ['tool_id'], properties: { tool_id: { type: 'string' } }, } - : { - type: 'object', - additionalProperties: false, - required: ['parameters'], - properties: { parameters: { type: 'object' } }, - }; + : parameterResponseSchema(selectedTool); const instruction = stage === 'select' ? 'Select one tool from the discovery results that best fulfills the user request. Return JSON only.' @@ -183,6 +201,61 @@ function adapterPayload({ stage, model, task, discovery, selectedTool, discovery }; } +function strictParameterSchema(parameter, forceRequired = parameter?.required === true) { + const type = normalizeParameterType(parameter?.type); + const schema = {}; + if (typeof parameter?.description === 'string' && parameter.description) { + schema.description = forceRequired + ? parameter.description + : `${parameter.description} Use null when this optional parameter is not needed.`; + } + + if (Array.isArray(parameter?.enum) && parameter.enum.length > 0) { + schema.type = nullableType(type, forceRequired); + schema.enum = forceRequired ? [...parameter.enum] : [...parameter.enum, null]; + return schema; + } + + schema.type = nullableType(type, forceRequired); + if (type === 'array') { + schema.items = strictParameterSchema( + isPlainObject(parameter?.items) ? parameter.items : { type: 'string', required: true }, + true, + ); + } else if (type === 'object') { + const nestedProperties = {}; + for (const [name, definition] of Object.entries( + isPlainObject(parameter?.properties) ? parameter.properties : {}, + )) { + nestedProperties[name] = strictParameterSchema( + isPlainObject(definition) ? definition : { type: 'string' }, + ); + } + schema.additionalProperties = false; + schema.required = Object.keys(nestedProperties); + schema.properties = nestedProperties; + } + return schema; +} + +function normalizeParameterType(value) { + const type = typeof value === 'string' ? value.toLowerCase() : 'string'; + if (['integer', 'int'].includes(type)) return 'integer'; + if (['number', 'float', 'double'].includes(type)) return 'number'; + if (['boolean', 'bool'].includes(type)) return 'boolean'; + if (['array', 'list'].includes(type)) return 'array'; + if (['object', 'map'].includes(type)) return 'object'; + return 'string'; +} + +function nullableType(type, required) { + return required ? type : [type, 'null']; +} + +function omitNullParameters(parameters) { + return Object.fromEntries(Object.entries(parameters).filter(([, value]) => value !== null)); +} + function toolId(value) { const id = value?.tool_id; return typeof id === 'string' && id.trim() ? id.trim() : null; diff --git a/benchmarks/discover-call/test/harness.test.mjs b/benchmarks/discover-call/test/harness.test.mjs index e56fd28..0e881af 100644 --- a/benchmarks/discover-call/test/harness.test.mjs +++ b/benchmarks/discover-call/test/harness.test.mjs @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { runBenchmark, validateTask } from '../src/harness.mjs'; +import { parameterResponseSchema, runBenchmark, validateTask } from '../src/harness.mjs'; const task = { id: 'weather-london', @@ -24,7 +24,17 @@ test('orchestrates discover, select, inspect, parameterize, and call', async () }, async inspect(input) { events.push(['inspect', input]); - return { results: [{ tool_id: 'weather.forecast', params: [{ name: 'city', required: true }] }] }; + return { + results: [ + { + tool_id: 'weather.forecast', + params: [ + { name: 'city', type: 'string', required: true }, + { name: 'units', type: 'string', required: false, enum: ['metric', 'imperial'] }, + ], + }, + ], + }; }, async call(input) { events.push(['call', input]); @@ -36,7 +46,10 @@ test('orchestrates discover, select, inspect, parameterize, and call', async () assert.equal(payload.messages.length, 2); assert.equal(payload.messages[1].content, JSON.stringify(payload.input)); events.push([payload.stage, payload.input.task_id]); - return payload.stage === 'select' ? { tool_id: 'weather.forecast' } : { parameters: { city: 'London' } }; + if (payload.stage === 'select') return { tool_id: 'weather.forecast' }; + assert.deepEqual(payload.response_schema.properties.parameters.required, ['city', 'units']); + assert.deepEqual(payload.response_schema.properties.parameters.properties.units.type, ['string', 'null']); + return { parameters: { city: 'London', units: null } }; }, metadata: { adapter_revision: 'adapter-sha' }, now: () => '2026-07-15T00:00:00.000Z', @@ -49,11 +62,57 @@ test('orchestrates discover, select, inspect, parameterize, and call', async () ); assert.equal(records[0].status, 'completed'); assert.deepEqual(records[0].inspection.required_parameters, ['city']); + assert.deepEqual(records[0].parameters, { city: 'London' }); assert.equal(records[0].call.success, true); assert.equal(records[0].call.execution_id, 'exec-1'); assert.equal(records[0].metadata.adapter_revision, 'adapter-sha'); }); +test('builds a strict provider-neutral schema from inspected parameters', () => { + assert.deepEqual( + parameterResponseSchema({ + params: [ + { name: 'limit', type: 'integer', required: true }, + { name: 'tags', type: 'array', required: false, items: { type: 'string' } }, + { + name: 'options', + type: 'object', + required: false, + properties: { + enabled: { type: 'boolean', required: true }, + label: { type: 'string', required: false }, + }, + }, + ], + }), + { + type: 'object', + additionalProperties: false, + required: ['parameters'], + properties: { + parameters: { + type: 'object', + additionalProperties: false, + required: ['limit', 'tags', 'options'], + properties: { + limit: { type: 'integer' }, + tags: { type: ['array', 'null'], items: { type: 'string' } }, + options: { + type: ['object', 'null'], + additionalProperties: false, + required: ['enabled', 'label'], + properties: { + enabled: { type: 'boolean' }, + label: { type: ['string', 'null'] }, + }, + }, + }, + }, + }, + }, + ); +}); + test('records adapter failures without copying their error text', async () => { const records = await runBenchmark({ tasks: [task], From fd08165e979a8ffaa030c56a4bd4853d805bf095 Mon Sep 17 00:00:00 2001 From: Linfang Wang Date: Thu, 23 Jul 2026 14:26:24 +0800 Subject: [PATCH 03/19] benchmark: infer enum value types --- benchmarks/discover-call/src/harness.mjs | 17 ++++++++++++++++- benchmarks/discover-call/test/harness.test.mjs | 6 +++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/benchmarks/discover-call/src/harness.mjs b/benchmarks/discover-call/src/harness.mjs index dcd3514..12c7741 100644 --- a/benchmarks/discover-call/src/harness.mjs +++ b/benchmarks/discover-call/src/harness.mjs @@ -211,7 +211,7 @@ function strictParameterSchema(parameter, forceRequired = parameter?.required == } if (Array.isArray(parameter?.enum) && parameter.enum.length > 0) { - schema.type = nullableType(type, forceRequired); + schema.type = enumType(parameter.enum, forceRequired); schema.enum = forceRequired ? [...parameter.enum] : [...parameter.enum, null]; return schema; } @@ -252,6 +252,21 @@ function nullableType(type, required) { return required ? type : [type, 'null']; } +function enumType(values, required) { + const types = [...new Set(values.map(jsonValueType).filter(Boolean))]; + const normalized = types.includes('number') ? types.filter((type) => type !== 'integer') : types; + if (!required) normalized.push('null'); + return normalized.length === 1 ? normalized[0] : normalized; +} + +function jsonValueType(value) { + if (typeof value === 'string') return 'string'; + if (typeof value === 'number') return Number.isInteger(value) ? 'integer' : 'number'; + if (typeof value === 'boolean') return 'boolean'; + if (value === null) return 'null'; + return null; +} + function omitNullParameters(parameters) { return Object.fromEntries(Object.entries(parameters).filter(([, value]) => value !== null)); } diff --git a/benchmarks/discover-call/test/harness.test.mjs b/benchmarks/discover-call/test/harness.test.mjs index 0e881af..19711df 100644 --- a/benchmarks/discover-call/test/harness.test.mjs +++ b/benchmarks/discover-call/test/harness.test.mjs @@ -73,6 +73,8 @@ test('builds a strict provider-neutral schema from inspected parameters', () => parameterResponseSchema({ params: [ { name: 'limit', type: 'integer', required: true }, + { name: 'count', type: 'integer', required: false, enum: ['10', '20'] }, + { name: 'spellcheck', type: 'boolean', required: false, enum: ['1'] }, { name: 'tags', type: 'array', required: false, items: { type: 'string' } }, { name: 'options', @@ -93,9 +95,11 @@ test('builds a strict provider-neutral schema from inspected parameters', () => parameters: { type: 'object', additionalProperties: false, - required: ['limit', 'tags', 'options'], + required: ['limit', 'count', 'spellcheck', 'tags', 'options'], properties: { limit: { type: 'integer' }, + count: { type: ['string', 'null'], enum: ['10', '20', null] }, + spellcheck: { type: ['string', 'null'], enum: ['1', null] }, tags: { type: ['array', 'null'], items: { type: 'string' } }, options: { type: ['object', 'null'], From 4ced1f16bb6d185db53f55570e827f6645b80d50 Mon Sep 17 00:00:00 2001 From: Linfang Wang Date: Thu, 23 Jul 2026 15:01:11 +0800 Subject: [PATCH 04/19] benchmark: publish first controlled model result --- .../2026-07-23-gpt-5.6-sol-v2.runs.jsonl | 54 + .../2026-07-23-gpt-5.6-sol-v2.summary.json | 1272 +++++++++++++++++ benchmarks/discover-call/results/README.md | 57 + docs/en-US/discover-call-benchmark.md | 30 +- docs/zh-CN/discover-call-benchmark.md | 24 +- 5 files changed, 1431 insertions(+), 6 deletions(-) create mode 100644 benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.runs.jsonl create mode 100644 benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.summary.json create mode 100644 benchmarks/discover-call/results/README.md diff --git a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.runs.jsonl b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.runs.jsonl new file mode 100644 index 0000000..e36983d --- /dev/null +++ b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.runs.jsonl @@ -0,0 +1,54 @@ +{"schema_version":1,"benchmark_version":"v1","run_id":"ac88304b-0662-4d2b-bb4b-778a65d4d84d","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"weather-london","trial":1,"started_at":"2026-07-23T06:26:39.032Z","status":"completed","discovery":{"result_tool_ids":["weather_gov.gridpoints.forecast.retrieve.v3.29bbec80","weather_gov.gridpoints.forecast.retrieve.v1.40a9a3ce","weather_gov.gridpoints.forecast.hourly.retrieve.v3.808ea240","amap_webservice.weather.weatherinfo.retrieve.v3","weather_gov.icons.retrieve.v3.ad0b4d80","weather_gov.gridpoints.retrieve.v3.a0c89365","weather_gov.products.retrieve.v1.18657145","openweathermap.forecast.retrieve.v2.5","weather_gov.gridpoints.retrieve.v3.2cf46664","visualcrossing.timeline.retrieve.v1"]},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"returned_tool_ids":["visualcrossing.timeline.retrieve.v1"],"required_parameters":["location"]},"parameters":{"location":"London"},"call":{"attempted":true,"success":true,"execution_id":"2ecc61c2-a8bd-4c5f-9b76-efdb23b44641"},"finished_at":"2026-07-23T06:27:00.228Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"5b911181-4d6c-472a-a2a0-4cb12e433f06","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"weather-london","trial":2,"started_at":"2026-07-23T06:27:00.228Z","status":"completed","discovery":{"result_tool_ids":["weather_gov.gridpoints.forecast.retrieve.v3.29bbec80","weather_gov.gridpoints.forecast.retrieve.v1.40a9a3ce","weather_gov.gridpoints.forecast.hourly.retrieve.v3.808ea240","amap_webservice.weather.weatherinfo.retrieve.v3","weather_gov.icons.retrieve.v3.ad0b4d80","weather_gov.gridpoints.retrieve.v3.a0c89365","weather_gov.products.retrieve.v1.18657145","openweathermap.forecast.retrieve.v2.5","weather_gov.gridpoints.retrieve.v3.2cf46664","visualcrossing.timeline.retrieve.v1"]},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"returned_tool_ids":["visualcrossing.timeline.retrieve.v1"],"required_parameters":["location"]},"parameters":{"location":"London"},"call":{"attempted":true,"success":true,"execution_id":"435720e5-bf11-47df-881b-c5056e26b6ad"},"finished_at":"2026-07-23T06:27:23.671Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"a8268395-01a9-413b-b2af-ba30d2e71f9a","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"weather-london","trial":3,"started_at":"2026-07-23T06:27:23.671Z","status":"completed","discovery":{"result_tool_ids":["weather_gov.gridpoints.forecast.retrieve.v3.29bbec80","weather_gov.gridpoints.forecast.retrieve.v1.40a9a3ce","weather_gov.gridpoints.forecast.hourly.retrieve.v3.808ea240","amap_webservice.weather.weatherinfo.retrieve.v3","weather_gov.icons.retrieve.v3.ad0b4d80","weather_gov.gridpoints.retrieve.v3.a0c89365","weather_gov.products.retrieve.v1.18657145","openweathermap.forecast.retrieve.v2.5","weather_gov.gridpoints.retrieve.v3.2cf46664","visualcrossing.timeline.retrieve.v1"]},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"returned_tool_ids":["visualcrossing.timeline.retrieve.v1"],"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"execution_id":"01fdd003-d088-4dfc-8232-444e096d9008"},"finished_at":"2026-07-23T06:27:50.399Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"4e925856-364c-4ca7-b769-c8d4d43e85c2","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-23T06:27:50.399Z","status":"completed","discovery":{"result_tool_ids":["eodhd.live_v2.us_quote_delayed.retrieve.v1.f0e13d45","twelvedata.quote.retrieve.v1.affbefe3","massive_stocks.options.reference.conditions.list.v3.41a82e07","alphavantage.realtime_bulk_quotes.retrieve.v1","fiu_mcp_server.postv1stockquoteextend.create.v2.a30e86e7","hangseng_polysource.a_shares_live_quote.query.v2.10fe0581","mcp_gildata.usstockdailyquotes.v1","massive_stocks.stocks.marketstatus.retrieve.v1.3045ae6e","fiu_mcp_server.postv3stockquoteextend.create.v2.b35a7f02","financialmodelingprep.cusip.search.v1.ad19189f"]},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"returned_tool_ids":["twelvedata.quote.retrieve.v1.affbefe3"],"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"0229f37b-abf6-411e-927c-91c372c00d76"},"finished_at":"2026-07-23T06:28:19.862Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"9c2b558d-b803-4299-9b8d-7778e56d0fcb","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-23T06:28:19.862Z","status":"completed","discovery":{"result_tool_ids":["eodhd.live_v2.us_quote_delayed.retrieve.v1.f0e13d45","twelvedata.quote.retrieve.v1.affbefe3","massive_stocks.options.reference.conditions.list.v3.41a82e07","alphavantage.realtime_bulk_quotes.retrieve.v1","fiu_mcp_server.postv1stockquoteextend.create.v2.a30e86e7","hangseng_polysource.a_shares_live_quote.query.v2.10fe0581","mcp_gildata.usstockdailyquotes.v1","massive_stocks.stocks.marketstatus.retrieve.v1.3045ae6e","fiu_mcp_server.postv3stockquoteextend.create.v2.b35a7f02","financialmodelingprep.cusip.search.v1.ad19189f"]},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"returned_tool_ids":["twelvedata.quote.retrieve.v1.affbefe3"],"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"0c743985-72c3-4029-8bfe-adc9d90d4cd1"},"finished_at":"2026-07-23T06:28:47.286Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"6762a6af-2002-4349-9239-89c6c62a597b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-23T06:28:47.286Z","status":"completed","discovery":{"result_tool_ids":["eodhd.live_v2.us_quote_delayed.retrieve.v1.f0e13d45","twelvedata.quote.retrieve.v1.affbefe3","massive_stocks.options.reference.conditions.list.v3.41a82e07","alphavantage.realtime_bulk_quotes.retrieve.v1","fiu_mcp_server.postv1stockquoteextend.create.v2.a30e86e7","hangseng_polysource.a_shares_live_quote.query.v2.10fe0581","mcp_gildata.usstockdailyquotes.v1","massive_stocks.stocks.marketstatus.retrieve.v1.3045ae6e","fiu_mcp_server.postv3stockquoteextend.create.v2.b35a7f02","financialmodelingprep.cusip.search.v1.ad19189f"]},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"returned_tool_ids":["twelvedata.quote.retrieve.v1.affbefe3"],"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"d04c5819-306b-4a68-8464-68807a24fa3f"},"finished_at":"2026-07-23T06:29:13.981Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"aa24674f-2a54-49ce-9652-95a7afc5a139","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-23T06:29:13.981Z","status":"completed","discovery":{"result_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4","coinmarketcap.tools.priceconversion.retrieve.v2.24e991d9","twelvedata.exchangerate.retrieve.v1.9eeb3b0d","bybitp2p.pay.fx.convert.v5.9221b0c4","coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","coinmarketcap.tools.priceconversion.retrieve.v2.52e4cb2d","coingecko_pro.exchangerates.retrieve.v3.0f6133ea","bybitp2p.payment.fx.convert.v5.7883b755","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d","okx_api_v5.market.exchange-rate.retrieve.v5.0d3fd1c2"]},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"returned_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4"],"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":100,"format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"0da4d9b1-d29f-47bd-b44f-d0c1b56c1588"},"finished_at":"2026-07-23T06:29:40.942Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"d038e88d-be8e-4d2e-810c-ec4f155b6871","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-23T06:29:40.943Z","status":"completed","discovery":{"result_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4","coinmarketcap.tools.priceconversion.retrieve.v2.24e991d9","twelvedata.exchangerate.retrieve.v1.9eeb3b0d","bybitp2p.pay.fx.convert.v5.9221b0c4","coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","coinmarketcap.tools.priceconversion.retrieve.v2.52e4cb2d","coingecko_pro.exchangerates.retrieve.v3.0f6133ea","bybitp2p.payment.fx.convert.v5.7883b755","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d","okx_api_v5.market.exchange-rate.retrieve.v5.0d3fd1c2"]},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"returned_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4"],"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":100,"format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"60b1956f-2bec-40f6-a5f8-5b8d4d263a9f"},"finished_at":"2026-07-23T06:30:10.765Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"c54fe3bb-0b0a-4758-beec-b7a5ce2bb540","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-23T06:30:10.765Z","status":"completed","discovery":{"result_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4","coinmarketcap.tools.priceconversion.retrieve.v2.24e991d9","twelvedata.exchangerate.retrieve.v1.9eeb3b0d","bybitp2p.pay.fx.convert.v5.9221b0c4","coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","coinmarketcap.tools.priceconversion.retrieve.v2.52e4cb2d","coingecko_pro.exchangerates.retrieve.v3.0f6133ea","bybitp2p.payment.fx.convert.v5.7883b755","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d","okx_api_v5.market.exchange-rate.retrieve.v5.0d3fd1c2"]},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"returned_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4"],"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"62073171-a903-4900-b9b5-084eca6eeb40"},"finished_at":"2026-07-23T06:30:38.654Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"6bede096-05fb-47c7-a2eb-5a2f12dad17d","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"news-ai","trial":1,"started_at":"2026-07-23T06:30:38.654Z","status":"completed","discovery":{"result_tool_ids":["x_developer.2.news.search.retrieve.v1.19b3e678","newsdata.news.search.v1.b65ccc56","newsapi.everything.retrieve.v2.e77ab1df","qveris_finance.finance_news_aggregation_v1","x_developer.2.tweets.search.recent.retrieve.v2.f424c8f9","newsapi.top_headlines.list.v2.38face94","linkup.search.v1","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic","jina_search.rerank.create.v2026.23dc486d","brave_search.news.search.list.v1.6ab52eaa"]},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"returned_tool_ids":["newsapi.everything.retrieve.v2.e77ab1df"],"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20%22generative%20AI%22","from":"2026-07-16","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"execution_id":"49aa189a-d829-4250-90af-8eb1d560f478"},"finished_at":"2026-07-23T06:31:10.595Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"44918a28-652b-430d-ac7e-53daadb0e69b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"news-ai","trial":2,"started_at":"2026-07-23T06:31:10.595Z","status":"completed","discovery":{"result_tool_ids":["x_developer.2.news.search.retrieve.v1.19b3e678","newsdata.news.search.v1.b65ccc56","newsapi.everything.retrieve.v2.e77ab1df","qveris_finance.finance_news_aggregation_v1","x_developer.2.tweets.search.recent.retrieve.v2.f424c8f9","newsapi.top_headlines.list.v2.38face94","linkup.search.v1","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic","jina_search.rerank.create.v2026.23dc486d","brave_search.news.search.list.v1.6ab52eaa"]},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"returned_tool_ids":["newsapi.everything.retrieve.v2.e77ab1df"],"required_parameters":["q"]},"parameters":{"q":"%28%22artificial%20intelligence%22%20OR%20%22generative%20AI%22%20OR%20OpenAI%20OR%20Anthropic%29","from":"2026-07-16T00:00:00","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"execution_id":"6acf3bf4-4c36-4b8b-82a9-3eddbb04ed86"},"finished_at":"2026-07-23T06:31:44.443Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"a306ce69-4682-4778-a8e5-47b0c598e895","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"news-ai","trial":3,"started_at":"2026-07-23T06:31:44.443Z","status":"completed","discovery":{"result_tool_ids":["x_developer.2.news.search.retrieve.v1.19b3e678","newsdata.news.search.v1.b65ccc56","x_developer.2.tweets.search.recent.retrieve.v2.f424c8f9","qveris_finance.finance_news_aggregation_v1","linkup.search.v1","newsapi.everything.retrieve.v2.e77ab1df","jina_search.rerank.create.v2026.23dc486d","newsapi.top_headlines.list.v2.38face94","brave_search.news.search.list.v1.6ab52eaa","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic"]},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"returned_tool_ids":["newsapi.everything.retrieve.v2.e77ab1df"],"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20%22generative%20AI%22","from":"2026-07-16","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"execution_id":"3e7f4c5d-b285-4304-be83-2973fb06448d"},"finished_at":"2026-07-23T06:32:18.995Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"9377cc54-be7e-42b4-bf14-f4d14d013c54","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T06:32:18.995Z","status":"completed","discovery":{"result_tool_ids":["semantic_scholar.paper.search.bulk.execute.v1.239dfd66","qveris_academic.paper_aggregation_v1","semantic_scholar.paper.search.bulk.retrieve.v1.096de627","semantic_scholar.paper.search.match.retrieve.v1.b9f166f3","semantic_scholar.paper.search.match.retrieve.v1.d3fd06f6","semantic_scholar.paper.autocomplete.retrieve.v1.e16c9798","semantic_scholar.paper.autocomplete.list.v1.99d870f6","semantic_scholar.author.search.query.v1.1c428051","xiaosu.scholar.search.v1.f79d2374","semantic_scholar.author.search.list.v1.ea3671fb"]},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"returned_tool_ids":["qveris_academic.paper_aggregation_v1"],"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"execution_id":"f4d795bf-c2bc-4801-b538-167c2e8cfe23"},"finished_at":"2026-07-23T06:32:52.395Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"c7815fa0-d6c8-4346-8463-1ae287470639","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T06:32:52.395Z","status":"completed","discovery":{"result_tool_ids":["semantic_scholar.paper.search.bulk.execute.v1.239dfd66","qveris_academic.paper_aggregation_v1","semantic_scholar.paper.search.bulk.retrieve.v1.096de627","semantic_scholar.paper.search.match.retrieve.v1.b9f166f3","semantic_scholar.paper.search.match.retrieve.v1.d3fd06f6","semantic_scholar.paper.autocomplete.retrieve.v1.e16c9798","semantic_scholar.paper.autocomplete.list.v1.99d870f6","semantic_scholar.author.search.query.v1.1c428051","xiaosu.scholar.search.v1.f79d2374","semantic_scholar.author.search.list.v1.ea3671fb"]},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"returned_tool_ids":["qveris_academic.paper_aggregation_v1"],"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"execution_id":"b390cf81-6110-49a5-8730-9a9e4ec1730f"},"finished_at":"2026-07-23T06:33:23.255Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"844e8bbf-ae65-4047-9349-cbd1c1bf7b50","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T06:33:23.255Z","status":"completed","discovery":{"result_tool_ids":["semantic_scholar.paper.search.bulk.execute.v1.239dfd66","qveris_academic.paper_aggregation_v1","semantic_scholar.paper.search.bulk.retrieve.v1.096de627","semantic_scholar.paper.search.match.retrieve.v1.b9f166f3","semantic_scholar.paper.search.match.retrieve.v1.d3fd06f6","semantic_scholar.paper.autocomplete.retrieve.v1.e16c9798","semantic_scholar.paper.autocomplete.list.v1.99d870f6","semantic_scholar.author.search.query.v1.1c428051","xiaosu.scholar.search.v1.f79d2374","semantic_scholar.author.search.list.v1.ea3671fb"]},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"returned_tool_ids":["qveris_academic.paper_aggregation_v1"],"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"execution_id":"eb3ac762-0475-4c55-827c-6a78a9b4551d"},"finished_at":"2026-07-23T06:33:51.402Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"cc32c600-4ef5-4fd8-b37c-6ae16c084f7e","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T06:33:51.402Z","status":"completed","discovery":{"result_tool_ids":["api_sports.timezone.retrieve.v1.e993615d","api_sports.timezone.retrieve.v1.6792e6aa","api_sports.timezone.list.v1.504ebeef","api_sports.timezone.retrieve.v1.9b882d11","api_sports.timezone.list.v1.5a9c7ba0","api_sports.timezone.retrieve.v1.ae6ae8d7","api_sports.timezone.list.v1.bd878c58","api_sports.timezone.list.v1.58366f46","api_sports.timezone.retrieve.v1.64cfc345","api_sports.timezone.retrieve.v1.895e2257"]},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"returned_tool_ids":["api_sports.timezone.retrieve.v1.e993615d"],"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"execution_id":"16a904e5-16af-473d-8cf5-69a3f3ad4924"},"finished_at":"2026-07-23T06:34:27.740Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"de6093e2-ce59-4692-ae3c-da0801ac1675","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T06:34:27.740Z","status":"completed","discovery":{"result_tool_ids":["api_sports.timezone.retrieve.v1.e993615d","api_sports.timezone.retrieve.v1.6792e6aa","api_sports.timezone.list.v1.504ebeef","api_sports.timezone.retrieve.v1.9b882d11","api_sports.timezone.list.v1.5a9c7ba0","api_sports.timezone.retrieve.v1.ae6ae8d7","api_sports.timezone.list.v1.bd878c58","api_sports.timezone.list.v1.58366f46","api_sports.timezone.retrieve.v1.64cfc345","api_sports.timezone.retrieve.v1.895e2257"]},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"returned_tool_ids":["api_sports.timezone.retrieve.v1.e993615d"],"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"execution_id":"a05ed585-79cd-4953-afc4-f9b28295ae27"},"finished_at":"2026-07-23T06:34:55.284Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"a66e8303-b561-4d85-a2fa-baf46e0a953b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T06:34:55.284Z","status":"completed","discovery":{"result_tool_ids":["api_sports.timezone.retrieve.v1.e993615d","api_sports.timezone.retrieve.v1.6792e6aa","api_sports.timezone.list.v1.504ebeef","api_sports.timezone.retrieve.v1.9b882d11","api_sports.timezone.list.v1.5a9c7ba0","api_sports.timezone.retrieve.v1.ae6ae8d7","api_sports.timezone.list.v1.bd878c58","api_sports.timezone.list.v1.58366f46","api_sports.timezone.retrieve.v1.64cfc345","api_sports.timezone.retrieve.v1.895e2257"]},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"returned_tool_ids":["api_sports.timezone.retrieve.v1.e993615d"],"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"execution_id":"3810f78b-e90d-4129-a174-eebc48bdb9dd"},"finished_at":"2026-07-23T06:35:32.765Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"28718552-be81-486c-b313-75a5550ff3cc","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T06:35:32.765Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.geocoder.reverse.v2","baidu_map.geocoder.query.v2","baidu_map.reverse_geocoding.query.v2","baidu_map.geocoding.reverse_geocode.v1","baidu_map.geocoder.wgeoreverse.create.v1","amap_webservice.geocode.geo.retrieve.v3","baidu_map.reversegeocoding.create.v1","baidu_map.geocoder.wgeobatch.create.v1","arcgis_ocean.geocoding.reverse_geocode.v1.4d40d93e","baidu_map.geoconv.convert.v1"]},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"returned_tool_ids":["amap_webservice.geocode.geo.retrieve.v3"],"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京","output":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"668c4105-b57e-46af-89be-b199a1a01735"},"finished_at":"2026-07-23T06:36:03.749Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"26e071b2-d139-4ee3-879c-5900578b1233","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T06:36:03.749Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.geocoder.reverse.v2","baidu_map.geocoder.query.v2","baidu_map.reverse_geocoding.query.v2","baidu_map.geocoding.reverse_geocode.v1","baidu_map.geocoder.wgeoreverse.create.v1","amap_webservice.geocode.geo.retrieve.v3","baidu_map.reversegeocoding.create.v1","baidu_map.geocoder.wgeobatch.create.v1","arcgis_ocean.geocoding.reverse_geocode.v1.4d40d93e","baidu_map.geoconv.convert.v1"]},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"returned_tool_ids":["amap_webservice.geocode.geo.retrieve.v3"],"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"84744b45-4016-4932-b441-c3f8a2b4b236"},"finished_at":"2026-07-23T06:36:38.806Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"e2f6f4c2-a8e7-4967-98b1-1a8c29aa3410","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T06:36:38.806Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.geocoder.reverse.v2","baidu_map.geocoder.query.v2","baidu_map.reverse_geocoding.query.v2","baidu_map.geocoding.reverse_geocode.v1","baidu_map.geocoder.wgeoreverse.create.v1","amap_webservice.geocode.geo.retrieve.v3","baidu_map.reversegeocoding.create.v1","baidu_map.geocoder.wgeobatch.create.v1","arcgis_ocean.geocoding.reverse_geocode.v1.4d40d93e","baidu_map.geoconv.convert.v1"]},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"returned_tool_ids":["amap_webservice.geocode.geo.retrieve.v3"],"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"b0668851-5bb2-4489-8abd-c23dd883fdf8"},"finished_at":"2026-07-23T06:37:14.052Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"fde5c3d4-8ff3-4813-a8c5-5e6a61ed3c2e","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-23T06:37:14.052Z","status":"completed","discovery":{"result_tool_ids":["decodo.isp_pay_per_ip.ip.retrieve.v1.bc2c6371","baidu_map.location.ip.retrieve.v1","baidu_map.location.ip.execute.v1","amap_webservice.ip.retrieve.v3","virustotal.ip.info.retrieve.v3.ab34b0d1","virustotal.ip.report.retrieve.v3.ab34b0d1","baidu_map.ip_location.retrieve.v1","weather_api.ip_lookup.retrieve.v1.e095d904","virustotal.ip_addresses.retrieve.v3.ab34b0d1","virustotal.ip-address.report.retrieve.v2.79606e33"]},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"returned_tool_ids":["weather_api.ip_lookup.retrieve.v1.e095d904"],"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"execution_id":"3da6c262-0b07-4ca0-90db-b627a142ad22"},"finished_at":"2026-07-23T06:37:44.040Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"cb085acd-e437-40df-9e07-e91343e2af9d","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-23T06:37:44.040Z","status":"completed","discovery":{"result_tool_ids":["decodo.isp_pay_per_ip.ip.retrieve.v1.bc2c6371","baidu_map.location.ip.retrieve.v1","baidu_map.location.ip.execute.v1","amap_webservice.ip.retrieve.v3","virustotal.ip.info.retrieve.v3.ab34b0d1","virustotal.ip.report.retrieve.v3.ab34b0d1","baidu_map.ip_location.retrieve.v1","weather_api.ip_lookup.retrieve.v1.e095d904","virustotal.ip_addresses.retrieve.v3.ab34b0d1","virustotal.ip-address.report.retrieve.v2.79606e33"]},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"returned_tool_ids":["weather_api.ip_lookup.retrieve.v1.e095d904"],"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"execution_id":"31469806-e596-4d4e-a570-362026d6c4f1"},"finished_at":"2026-07-23T06:38:14.212Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"143359c4-589b-437f-ae30-9a99b9f1636b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-23T06:38:14.212Z","status":"completed","discovery":{"result_tool_ids":["decodo.isp_pay_per_ip.ip.retrieve.v1.bc2c6371","baidu_map.location.ip.retrieve.v1","baidu_map.location.ip.execute.v1","amap_webservice.ip.retrieve.v3","virustotal.ip.info.retrieve.v3.ab34b0d1","virustotal.ip.report.retrieve.v3.ab34b0d1","baidu_map.ip_location.retrieve.v1","weather_api.ip_lookup.retrieve.v1.e095d904","virustotal.ip_addresses.retrieve.v3.ab34b0d1","virustotal.ip-address.report.retrieve.v2.79606e33"]},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"returned_tool_ids":["weather_api.ip_lookup.retrieve.v1.e095d904"],"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"execution_id":"9fd3e989-f14d-4d66-96db-e84835184d9e"},"finished_at":"2026-07-23T06:38:55.364Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"58e4b1fd-8de8-4f85-8db1-478b57309b00","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"crypto-price","trial":1,"started_at":"2026-07-23T06:38:55.364Z","status":"completed","discovery":{"result_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","binance.avg_price.retrieve.v1.aef0de4f","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.251242d7","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.9dc24be4","binance.ticker.price.list.v3.8675eca0","binance.avgPrice.retrieve.v3.7b0852b4","deribit.public.get_index_price.retrieve.v2.d47dcdcb","coinmarketcap.cryptocurrency.listings.latest.retrieve.v1.8dddc12f","etherscan.api.stats.ethprice.retrieve.v2.8eeca59e","binance.ticker.query.v1.0437113d"]},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"returned_tool_ids":["binance.ticker.price.list.v3.8675eca0"],"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"execution_id":"999554c2-cfb9-4494-802a-d945cf5d93d3"},"finished_at":"2026-07-23T06:39:36.286Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"f498a668-e254-4a1f-a109-0b11043a9a77","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"crypto-price","trial":2,"started_at":"2026-07-23T06:39:36.286Z","status":"completed","discovery":{"result_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","binance.avg_price.retrieve.v1.aef0de4f","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.251242d7","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.9dc24be4","binance.ticker.price.list.v3.8675eca0","binance.avgPrice.retrieve.v3.7b0852b4","deribit.public.get_index_price.retrieve.v2.d47dcdcb","coinmarketcap.cryptocurrency.listings.latest.retrieve.v1.8dddc12f","etherscan.api.stats.ethprice.retrieve.v2.8eeca59e","binance.ticker.query.v1.0437113d"]},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"returned_tool_ids":["binance.ticker.price.list.v3.8675eca0"],"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"execution_id":"e0046687-c092-40e5-9cbd-7e79da6e216a"},"finished_at":"2026-07-23T06:40:06.523Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"3b48193b-6026-491e-8472-51034ae93749","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"crypto-price","trial":3,"started_at":"2026-07-23T06:40:06.523Z","status":"completed","discovery":{"result_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","binance.avg_price.retrieve.v1.aef0de4f","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.251242d7","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.9dc24be4","binance.ticker.price.list.v3.8675eca0","binance.avgPrice.retrieve.v3.7b0852b4","deribit.public.get_index_price.retrieve.v2.d47dcdcb","coinmarketcap.cryptocurrency.listings.latest.retrieve.v1.8dddc12f","etherscan.api.stats.ethprice.retrieve.v2.8eeca59e","binance.ticker.query.v1.0437113d"]},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"returned_tool_ids":["binance.ticker.price.list.v3.8675eca0"],"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"execution_id":"883d9f06-9b8c-4d64-8a25-4b1c8020c4b3"},"finished_at":"2026-07-23T06:40:52.281Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"3d70bc1e-3063-41e6-8b57-40c6e6b963e2","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"web-search","trial":1,"started_at":"2026-07-23T06:40:52.281Z","status":"completed","discovery":{"result_tool_ids":["linkup.search.v1","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic","cloudsway.smart_search_overseas","firecrawl.search.create.v2.a99b6f74","brave_search.images.search.list.v1","x_developer.2.news.search.retrieve.v1.19b3e678","serpapi.locations.locations.list.v1","x_developer.2.spaces.search.retrieve.v1.43a1f31c","tavily.map.execute.v1","brave_search.web.search.list.v1.f02ed09d"]},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"returned_tool_ids":["cloudsway.smart_search_overseas"],"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"execution_id":"37c52450-eeda-417c-9849-d67132e689b0"},"finished_at":"2026-07-23T06:41:28.119Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"39c78597-f3db-4d54-b56e-9ed6e6b3d412","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"web-search","trial":2,"started_at":"2026-07-23T06:41:28.119Z","status":"completed","discovery":{"result_tool_ids":["linkup.search.v1","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic","cloudsway.smart_search_overseas","firecrawl.search.create.v2.a99b6f74","brave_search.images.search.list.v1","x_developer.2.news.search.retrieve.v1.19b3e678","serpapi.locations.locations.list.v1","x_developer.2.spaces.search.retrieve.v1.43a1f31c","tavily.map.execute.v1","brave_search.web.search.list.v1.f02ed09d"]},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"returned_tool_ids":["cloudsway.smart_search_overseas"],"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"execution_id":"8e870c4f-ff66-4af3-be0a-a3e57848cc37"},"finished_at":"2026-07-23T06:42:00.561Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"cf50c76f-316f-4793-877b-2f51474c6981","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"web-search","trial":3,"started_at":"2026-07-23T06:42:00.561Z","status":"completed","discovery":{"result_tool_ids":["linkup.search.v1","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic","cloudsway.smart_search_overseas","brave_search.images.search.list.v1","firecrawl.search.create.v2.a99b6f74","x_developer.2.spaces.search.retrieve.v1.43a1f31c","serpapi.locations.locations.list.v1","x_developer.2.news.search.retrieve.v1.19b3e678","tavily.map.execute.v1","brave_search.web.search.list.v1.f02ed09d"]},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"returned_tool_ids":["cloudsway.smart_search_overseas"],"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"execution_id":"17f61a10-d829-4e93-b2ee-08cd5cd5fe38"},"finished_at":"2026-07-23T06:42:34.259Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"5bf899ee-7846-44cf-8e8a-ebeb8bfbff65","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"air-quality","trial":1,"started_at":"2026-07-23T06:42:34.259Z","status":"completed","discovery":{"result_tool_ids":["openweathermap.air_pollution.current.v2_5","openweathermap.air_pollution.retrieve.v2.5","openweathermap.air_pollution.history.v2_5","openweathermap.air_pollution.history.retrieve.v2.5","openweathermap.air_pollution.forecast.retrieve.v2.5","epa_envirofacts.qaannualperformanceevaluations.bystate.retrieve.v1.61a2aa7d","epa_envirofacts.transactionsqaannualperformanceevaluations.bystate.retrieve.v1.4143b8bf","epa_envirofacts.transactionssample.bysite.retrieve.v1.93889e72","epa_envirofacts.annualdata.bybox.retrieve.v1.4b74a0d6","epa_envirofacts.dailydata.bycounty.retrieve.v1.f1ee8d1b"]},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"returned_tool_ids":["openweathermap.air_pollution.current.v2_5"],"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"execution_id":"b8834f76-74db-4431-9f9a-3e0e73012872"},"finished_at":"2026-07-23T06:43:06.453Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"673ccda1-c286-49cd-9299-75ce17706e92","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"air-quality","trial":2,"started_at":"2026-07-23T06:43:06.453Z","status":"completed","discovery":{"result_tool_ids":["openweathermap.air_pollution.current.v2_5","openweathermap.air_pollution.retrieve.v2.5","openweathermap.air_pollution.history.v2_5","openweathermap.air_pollution.history.retrieve.v2.5","openweathermap.air_pollution.forecast.retrieve.v2.5","epa_envirofacts.qaannualperformanceevaluations.bystate.retrieve.v1.61a2aa7d","epa_envirofacts.transactionsqaannualperformanceevaluations.bystate.retrieve.v1.4143b8bf","epa_envirofacts.transactionssample.bysite.retrieve.v1.93889e72","epa_envirofacts.annualdata.bybox.retrieve.v1.4b74a0d6","epa_envirofacts.dailydata.bycounty.retrieve.v1.f1ee8d1b"]},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"returned_tool_ids":["openweathermap.air_pollution.current.v2_5"],"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"execution_id":"d478235d-935b-4887-8d4c-286f524c9f3d"},"finished_at":"2026-07-23T06:43:43.650Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"859aee1a-1670-4140-b9ac-91435124e6dc","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"air-quality","trial":3,"started_at":"2026-07-23T06:43:43.650Z","status":"completed","discovery":{"result_tool_ids":["openweathermap.air_pollution.current.v2_5","openweathermap.air_pollution.retrieve.v2.5","openweathermap.air_pollution.history.v2_5","openweathermap.air_pollution.history.retrieve.v2.5","openweathermap.air_pollution.forecast.retrieve.v2.5","epa_envirofacts.qaannualperformanceevaluations.bystate.retrieve.v1.61a2aa7d","epa_envirofacts.transactionsqaannualperformanceevaluations.bystate.retrieve.v1.4143b8bf","epa_envirofacts.transactionssample.bysite.retrieve.v1.93889e72","epa_envirofacts.annualdata.bybox.retrieve.v1.4b74a0d6","epa_envirofacts.dailydata.bycounty.retrieve.v1.f1ee8d1b"]},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"returned_tool_ids":["openweathermap.air_pollution.current.v2_5"],"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"execution_id":"a0193cf7-479f-4113-8dd3-ec693b39bfbc"},"finished_at":"2026-07-23T06:44:12.213Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"70a47304-ae9e-4cb5-8331-5df6a8e83a43","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-news","trial":1,"started_at":"2026-07-23T06:44:12.213Z","status":"completed","discovery":{"result_tool_ids":["finnhub.market.news.list.v1","finnhub.news.retrieve.v1.51b2567e","finnhub.news.retrieve.v1.2e72ea5d","finnhub.news.execute.v1","finnhub.news.retrieve.v1","alphavantage.news_sentiment.query.v1.467a92c0","finnhub.companynews.retrieve.v1.4d11f47b","finnhub.companynews.retrieve.v1","finnhub.companynews.retrieve.v1.e428b704","finnhub.companynews.execute.v1"]},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"returned_tool_ids":["finnhub.market.news.list.v1"],"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"execution_id":"d9e295bf-8357-492a-92b0-6d6ccc1ad06d"},"finished_at":"2026-07-23T06:44:39.351Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"73355630-bc92-4e0a-a1c8-4f25f895a949","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-news","trial":2,"started_at":"2026-07-23T06:44:39.351Z","status":"completed","discovery":{"result_tool_ids":["finnhub.market.news.list.v1","finnhub.news.retrieve.v1.51b2567e","finnhub.news.retrieve.v1.2e72ea5d","finnhub.news.execute.v1","finnhub.news.retrieve.v1","alphavantage.news_sentiment.query.v1.467a92c0","finnhub.companynews.retrieve.v1.4d11f47b","finnhub.companynews.retrieve.v1","finnhub.companynews.retrieve.v1.e428b704","finnhub.companynews.execute.v1"]},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"returned_tool_ids":["finnhub.market.news.list.v1"],"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"execution_id":"19ddaebd-5dd6-4792-93a2-16617d3db005"},"finished_at":"2026-07-23T06:45:14.542Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"f05cea11-e23c-463a-98f1-e5c0b4f51e88","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-news","trial":3,"started_at":"2026-07-23T06:45:14.542Z","status":"completed","discovery":{"result_tool_ids":["finnhub.market.news.list.v1","finnhub.news.retrieve.v1.51b2567e","finnhub.news.retrieve.v1.2e72ea5d","finnhub.news.execute.v1","finnhub.news.retrieve.v1","alphavantage.news_sentiment.query.v1.467a92c0","finnhub.companynews.retrieve.v1.4d11f47b","finnhub.companynews.retrieve.v1","finnhub.companynews.retrieve.v1.e428b704","finnhub.companynews.execute.v1"]},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"returned_tool_ids":["finnhub.market.news.list.v1"],"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"execution_id":"11798d87-25ba-45c9-80fd-4a20871b981d"},"finished_at":"2026-07-23T06:45:40.767Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ae3e60aa-8ee9-4650-b3e2-6d906a42da8b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T06:45:40.767Z","status":"failed","discovery":{"result_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8","dataforseo.dataforseo_labs.google.domain_whois_overview.live.execute.v3","dataforseo.databases.domains.query.v3.c1720ec0","virustotal.domains.report.retrieve.v3.c5459740","bainiu_api.common.company_domain.query.v1.e978ca32","virustotal.domain_reports.retrieve.v3.c5459740","dataforseo.domain_analytics.technologies.domains_by_technology.live.create.v3.bfc3dc2d","virustotal.domains.retrieve.v3.c5459740","virustotal.domains.analyse.v3.c755c69e","dataforseo.domain_analytics.technologies.domains_by_html_terms.live.create.v3.d3760ae1"]},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"returned_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"],"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","message":"Adapter invocation failed"},"finished_at":"2026-07-23T06:46:30.257Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"cc1eebe6-e68b-45ba-8f32-827f98baa240","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T06:46:30.257Z","status":"failed","discovery":{"result_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8","dataforseo.dataforseo_labs.google.domain_whois_overview.live.execute.v3","dataforseo.databases.domains.query.v3.c1720ec0","virustotal.domains.report.retrieve.v3.c5459740","bainiu_api.common.company_domain.query.v1.e978ca32","virustotal.domain_reports.retrieve.v3.c5459740","dataforseo.domain_analytics.technologies.domains_by_technology.live.create.v3.bfc3dc2d","virustotal.domains.retrieve.v3.c5459740","virustotal.domains.analyse.v3.c755c69e","dataforseo.domain_analytics.technologies.domains_by_html_terms.live.create.v3.d3760ae1"]},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"returned_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"],"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","message":"Adapter invocation failed"},"finished_at":"2026-07-23T06:47:52.318Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"19382342-d937-46dc-ba4a-c71139081673","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T06:47:52.318Z","status":"failed","discovery":{"result_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8","dataforseo.dataforseo_labs.google.domain_whois_overview.live.execute.v3","dataforseo.databases.domains.query.v3.c1720ec0","virustotal.domains.report.retrieve.v3.c5459740","bainiu_api.common.company_domain.query.v1.e978ca32","virustotal.domain_reports.retrieve.v3.c5459740","dataforseo.domain_analytics.technologies.domains_by_technology.live.create.v3.bfc3dc2d","virustotal.domains.retrieve.v3.c5459740","virustotal.domains.analyse.v3.c755c69e","dataforseo.domain_analytics.technologies.domains_by_html_terms.live.create.v3.d3760ae1"]},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"returned_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"],"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","message":"Adapter invocation failed"},"finished_at":"2026-07-23T06:48:55.681Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"48e281c3-3a7a-4327-9911-427072bc6fcd","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"image-search","trial":1,"started_at":"2026-07-23T06:48:55.681Z","status":"completed","discovery":{"result_tool_ids":["brave_search.images.search.list.v1","shutterstock.images.search.list.v2.69059569","shutterstock.images.search.list.v2.c2b44682","dataforseo.serp.google.search_by_image.task.create.v3","jina_search.rerank.create.v2026.23dc486d","jina_search.embeddings.create.v2026.5c7cb661","linkup.search.v1","shutterstock.images.search.list.v2.15f15845","shutterstock.images.search.suggestions.list.v2.f6850c10","hailuoai.image_generation.create.v1"]},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"returned_tool_ids":["brave_search.images.search.list.v1"],"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"execution_id":"1262ba5e-f345-4ef0-a799-515d53cb2082"},"finished_at":"2026-07-23T06:49:28.588Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"d23e1fc9-03a5-4c76-8761-22c9de0e6231","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"image-search","trial":2,"started_at":"2026-07-23T06:49:28.588Z","status":"completed","discovery":{"result_tool_ids":["brave_search.images.search.list.v1","shutterstock.images.search.list.v2.69059569","shutterstock.images.search.list.v2.c2b44682","dataforseo.serp.google.search_by_image.task.create.v3","jina_search.rerank.create.v2026.23dc486d","jina_search.embeddings.create.v2026.5c7cb661","linkup.search.v1","shutterstock.images.search.list.v2.15f15845","shutterstock.images.search.suggestions.list.v2.f6850c10","hailuoai.image_generation.create.v1"]},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"returned_tool_ids":["brave_search.images.search.list.v1"],"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":10,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"execution_id":"87f346a7-38d1-4666-ae15-3e1843c5e0a5"},"finished_at":"2026-07-23T06:49:55.653Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"262b0d23-181c-4b1c-8241-8cfd530dbb96","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"image-search","trial":3,"started_at":"2026-07-23T06:49:55.653Z","status":"completed","discovery":{"result_tool_ids":["brave_search.images.search.list.v1","shutterstock.images.search.list.v2.69059569","shutterstock.images.search.list.v2.c2b44682","dataforseo.serp.google.search_by_image.task.create.v3","jina_search.rerank.create.v2026.23dc486d","jina_search.embeddings.create.v2026.5c7cb661","linkup.search.v1","shutterstock.images.search.list.v2.15f15845","shutterstock.images.search.suggestions.list.v2.f6850c10","hailuoai.image_generation.create.v1"]},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"returned_tool_ids":["brave_search.images.search.list.v1"],"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"execution_id":"4f2204fb-38c5-45c9-a415-52d2b6f3294a"},"finished_at":"2026-07-23T06:50:22.831Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"3d110e2a-4ee1-47f7-a018-f5ae9d166e46","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-holidays","trial":1,"started_at":"2026-07-23T06:50:22.831Z","status":"completed","discovery":{"result_tool_ids":["eodhd.exchange_details.retrieve.v1.7da32390","finnhub.stock.marketholiday.retrieve.v1","finnhub.stock.marketholiday.execute.v1","finnhub.stock.marketholiday.retrieve.v1.5c5bfe9a","finnhub.stock.marketholiday.retrieve.v1.75d6c1f9","massive_stocks.stocks.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.retrieve.v1","massive_stocks.options.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.list.v1","theta_data.calendar.yearholidays.retrieve.v3.125586f0"]},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"returned_tool_ids":["finnhub.stock.marketholiday.retrieve.v1"],"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"execution_id":"67e0f2b5-0ae5-4de8-8d5d-285971094a0d"},"finished_at":"2026-07-23T06:50:57.424Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"b619dc10-8cc0-4413-9219-1aea84479b72","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-holidays","trial":2,"started_at":"2026-07-23T06:50:57.424Z","status":"completed","discovery":{"result_tool_ids":["eodhd.exchange_details.retrieve.v1.7da32390","finnhub.stock.marketholiday.retrieve.v1","finnhub.stock.marketholiday.execute.v1","finnhub.stock.marketholiday.retrieve.v1.5c5bfe9a","finnhub.stock.marketholiday.retrieve.v1.75d6c1f9","massive_stocks.stocks.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.retrieve.v1","massive_stocks.options.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.list.v1","theta_data.calendar.yearholidays.retrieve.v3.125586f0"]},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"returned_tool_ids":["finnhub.stock.marketholiday.retrieve.v1"],"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"execution_id":"596fd2ea-7672-49ce-9c15-b6ca3f725589"},"finished_at":"2026-07-23T06:51:30.279Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"0ab6d79a-6510-4b11-966c-7063b6646f73","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-holidays","trial":3,"started_at":"2026-07-23T06:51:30.279Z","status":"completed","discovery":{"result_tool_ids":["eodhd.exchange_details.retrieve.v1.7da32390","finnhub.stock.marketholiday.retrieve.v1","finnhub.stock.marketholiday.execute.v1","finnhub.stock.marketholiday.retrieve.v1.5c5bfe9a","finnhub.stock.marketholiday.retrieve.v1.75d6c1f9","massive_stocks.stocks.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.retrieve.v1","massive_stocks.options.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.list.v1","theta_data.calendar.yearholidays.retrieve.v3.125586f0"]},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"returned_tool_ids":["finnhub.stock.marketholiday.retrieve.v1"],"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"execution_id":"49e1d181-d923-4d8b-9e0c-26ff778119a3"},"finished_at":"2026-07-23T06:51:58.663Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"245b91ee-988d-474f-8fd1-82da7c62183a","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-23T06:51:58.663Z","status":"completed","discovery":{"result_tool_ids":["usgs_earthquake_catalog.event.events.list.v1","usgs_earthquake_catalog.fdsnws.event.query.retrieve.v1.025549c9","usgs_earthquake_catalog.earthquakes.query.v1","usgs_earthquake_catalog.fdsnws.event.query.v1.025549c9","usgs_earthquake_catalog.events.query.v1.025549c9","usgs_earthquake.catalogs.list.v1","usgs_earthquake.contributors.list.v1","usgs-earthquake-event-api.event.query.list.v1","usgs_earthquake.parameters.list.v1","usgs_earthquake.version.retrieve.v1"]},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"returned_tool_ids":["usgs_earthquake_catalog.event.events.list.v1"],"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"execution_id":"0a2f5cc0-b3e1-4538-8cad-9f2b807e253d"},"finished_at":"2026-07-23T06:52:30.880Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"dc38691f-d8f6-40f8-9e77-9288c33a7e68","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-23T06:52:30.881Z","status":"completed","discovery":{"result_tool_ids":["usgs_earthquake_catalog.event.events.list.v1","usgs_earthquake_catalog.fdsnws.event.query.retrieve.v1.025549c9","usgs_earthquake_catalog.earthquakes.query.v1","usgs_earthquake_catalog.fdsnws.event.query.v1.025549c9","usgs_earthquake_catalog.events.query.v1.025549c9","usgs_earthquake.catalogs.list.v1","usgs_earthquake.contributors.list.v1","usgs-earthquake-event-api.event.query.list.v1","usgs_earthquake.parameters.list.v1","usgs_earthquake.version.retrieve.v1"]},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"returned_tool_ids":["usgs_earthquake_catalog.event.events.list.v1"],"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"execution_id":"1c998fc3-bda1-437c-9f78-90b00026a63d"},"finished_at":"2026-07-23T06:53:04.476Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"c1624700-5aba-4cfc-8a68-acabf3de024a","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T06:53:04.476Z","status":"failed","discovery":{"result_tool_ids":["usgs_earthquake_catalog.event.events.list.v1","usgs_earthquake_catalog.fdsnws.event.query.retrieve.v1.025549c9","usgs_earthquake_catalog.earthquakes.query.v1","usgs_earthquake_catalog.fdsnws.event.query.v1.025549c9","usgs_earthquake_catalog.events.query.v1.025549c9","usgs_earthquake.catalogs.list.v1","usgs_earthquake.contributors.list.v1","usgs-earthquake-event-api.event.query.list.v1","usgs_earthquake.parameters.list.v1","usgs_earthquake.version.retrieve.v1"]},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"returned_tool_ids":["usgs_earthquake_catalog.event.events.list.v1"],"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","message":"Adapter invocation failed"},"finished_at":"2026-07-23T06:55:18.009Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"0e485876-22f8-4265-91e0-51990c3644e9","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"company-profile","trial":1,"started_at":"2026-07-23T06:55:18.009Z","status":"completed","discovery":{"result_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2","bainiu_api.company.detail.execute.v1.f6b528e7","bainiu_api.openapi.company.alter.execute.v1.8fa090d1","tradefeeds.compinfo.retrieve.v1.c2314765","financialmodelingprep.stable.companynotes.retrieve.v1.930aa468","financialmodelingprep.search.cusip.retrieve.v1.51d0c621","bainiu_api.openapi.common.company_info_deep_search.execute.v1.8e5f009d","financialmodelingprep.stable.secfilingscompanysearch.symbol.retrieve.v1.5cf7397d","finnhub.stock.profile2.retrieve.v1.59df26f6","bainiu_api.company_detail.create.v1.8fa090d1"]},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"returned_tool_ids":["tradefeeds.compinfo.retrieve.v1.c2314765"],"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"execution_id":"59b6de87-9bf7-4cb9-beef-7dfc854746e1"},"finished_at":"2026-07-23T06:55:45.466Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ea693fd1-b596-40b1-bd16-eec213eecb12","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"company-profile","trial":2,"started_at":"2026-07-23T06:55:45.466Z","status":"completed","discovery":{"result_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2","bainiu_api.company.detail.execute.v1.f6b528e7","bainiu_api.openapi.company.alter.execute.v1.8fa090d1","tradefeeds.compinfo.retrieve.v1.c2314765","financialmodelingprep.stable.companynotes.retrieve.v1.930aa468","financialmodelingprep.search.cusip.retrieve.v1.51d0c621","bainiu_api.openapi.common.company_info_deep_search.execute.v1.8e5f009d","financialmodelingprep.stable.secfilingscompanysearch.symbol.retrieve.v1.5cf7397d","finnhub.stock.profile2.retrieve.v1.59df26f6","bainiu_api.company_detail.create.v1.8fa090d1"]},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"returned_tool_ids":["tradefeeds.compinfo.retrieve.v1.c2314765"],"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"execution_id":"a278f80c-4d27-428b-a610-39eb6993d3bb"},"finished_at":"2026-07-23T06:56:19.074Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"abf65b68-8d35-453f-89b6-d30447a432e7","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"company-profile","trial":3,"started_at":"2026-07-23T06:56:19.074Z","status":"completed","discovery":{"result_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2","bainiu_api.company.detail.execute.v1.f6b528e7","bainiu_api.openapi.company.alter.execute.v1.8fa090d1","tradefeeds.compinfo.retrieve.v1.c2314765","financialmodelingprep.stable.companynotes.retrieve.v1.930aa468","financialmodelingprep.search.cusip.retrieve.v1.51d0c621","bainiu_api.openapi.common.company_info_deep_search.execute.v1.8e5f009d","financialmodelingprep.stable.secfilingscompanysearch.symbol.retrieve.v1.5cf7397d","finnhub.stock.profile2.retrieve.v1.59df26f6","bainiu_api.company_detail.create.v1.8fa090d1"]},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"returned_tool_ids":["tradefeeds.compinfo.retrieve.v1.c2314765"],"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"execution_id":"831c21f1-85e0-450c-b5f1-22ea733384bc"},"finished_at":"2026-07-23T06:56:53.486Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"a6ccdda6-dc1f-4765-873e-bd8c22197a6f","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"sports-standings","trial":1,"started_at":"2026-07-23T06:56:53.486Z","status":"completed","discovery":{"result_tool_ids":["api_sports.standings.list.v1.98a153bd","api_sports.standings.list.v1.39568ba4","api_sports.standings.groups.list.v1.a077a680","api_sports.standings.groups.list.v1.0d8a40cf","api_sports.standings.groups.list.v1.57393fb3","api_sports.standings.retrieve.v1.ff019893","api_sports.standings.stages.list.v1.838f769a","api_sports.standings.stages.list.v1.5ba13a89","api-football.standings.list.v3.76f7703e","api_sports.standings.groups.list.v1.bed67ce0"]},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"returned_tool_ids":["api-football.standings.list.v3.76f7703e"],"required_parameters":["season"]},"parameters":{"league":39,"season":2025},"call":{"attempted":true,"success":true,"execution_id":"470514d9-7c25-4168-be3e-bfebf7fdf390"},"finished_at":"2026-07-23T06:57:26.157Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"a025004c-c94c-444b-a2c9-bb176ed51ea3","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"sports-standings","trial":2,"started_at":"2026-07-23T06:57:26.157Z","status":"completed","discovery":{"result_tool_ids":["api_sports.standings.list.v1.98a153bd","api_sports.standings.list.v1.39568ba4","api_sports.standings.groups.list.v1.a077a680","api_sports.standings.groups.list.v1.0d8a40cf","api_sports.standings.groups.list.v1.57393fb3","api_sports.standings.retrieve.v1.ff019893","api_sports.standings.stages.list.v1.838f769a","api_sports.standings.stages.list.v1.5ba13a89","api-football.standings.list.v3.76f7703e","api_sports.standings.groups.list.v1.bed67ce0"]},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"returned_tool_ids":["api-football.standings.list.v3.76f7703e"],"required_parameters":["season"]},"parameters":{"league":39,"season":2025},"call":{"attempted":true,"success":true,"execution_id":"a0359f15-d70c-4906-97b9-2faeb215a13c"},"finished_at":"2026-07-23T06:57:50.791Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"f15c0986-0c0c-4eab-8dac-99cc4f6f7dea","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"sports-standings","trial":3,"started_at":"2026-07-23T06:57:50.791Z","status":"completed","discovery":{"result_tool_ids":["api_sports.standings.list.v1.98a153bd","api_sports.standings.list.v1.39568ba4","api_sports.standings.groups.list.v1.a077a680","api_sports.standings.groups.list.v1.0d8a40cf","api_sports.standings.groups.list.v1.57393fb3","api_sports.standings.retrieve.v1.ff019893","api_sports.standings.stages.list.v1.838f769a","api_sports.standings.stages.list.v1.5ba13a89","api-football.standings.list.v3.76f7703e","api_sports.standings.groups.list.v1.bed67ce0"]},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"returned_tool_ids":["api-football.standings.list.v3.76f7703e"],"required_parameters":["season"]},"parameters":{"league":39,"season":2025},"call":{"attempted":true,"success":true,"execution_id":"056d8f80-b061-4b18-940e-ba3a460c330d"},"finished_at":"2026-07-23T06:58:17.129Z"} diff --git a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.summary.json b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.summary.json new file mode 100644 index 0000000..016316d --- /dev/null +++ b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.summary.json @@ -0,0 +1,1272 @@ +{ + "schema_version": 1, + "methodology": "discover-call-v1", + "generated_at": "2026-07-23T06:58:29.076Z", + "models": [ + { + "model": "gpt-5.6-sol", + "tasks": 18, + "trials_per_task": 3, + "runs": 54, + "completed": 50, + "executed": 50, + "selection_grounded_rate": 1, + "inspection_grounded_rate": 1, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0.7593, + "call_success_rate": 0.88, + "workflow_success_rate": 0.6481, + "workflow_success_wilson_95": [ + 0.5148, + 0.7618 + ], + "failures_by_stage": { + "adapter": 4 + } + } + ], + "records": [ + { + "run_id": "ac88304b-0662-4d2b-bb4b-778a65d4d84d", + "task_id": "weather-london", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "5b911181-4d6c-472a-a2a0-4cb12e433f06", + "task_id": "weather-london", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "a8268395-01a9-413b-b2af-ba30d2e71f9a", + "task_id": "weather-london", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "4e925856-364c-4ca7-b769-c8d4d43e85c2", + "task_id": "stock-aapl", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "9c2b558d-b803-4299-9b8d-7778e56d0fcb", + "task_id": "stock-aapl", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "6762a6af-2002-4349-9239-89c6c62a597b", + "task_id": "stock-aapl", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "aa24674f-2a54-49ce-9652-95a7afc5a139", + "task_id": "exchange-usd-eur", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "workflow_success": false, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "d038e88d-be8e-4d2e-810c-ec4f155b6871", + "task_id": "exchange-usd-eur", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "workflow_success": false, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "c54fe3bb-0b0a-4758-beec-b7a5ce2bb540", + "task_id": "exchange-usd-eur", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "workflow_success": false, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "6bede096-05fb-47c7-a2eb-5a2f12dad17d", + "task_id": "news-ai", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "workflow_success": false, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "44918a28-652b-430d-ac7e-53daadb0e69b", + "task_id": "news-ai", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "workflow_success": false, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "a306ce69-4682-4778-a8e5-47b0c598e895", + "task_id": "news-ai", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "workflow_success": false, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "9377cc54-be7e-42b4-bf14-f4d14d013c54", + "task_id": "papers-climate", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "c7815fa0-d6c8-4346-8463-1ae287470639", + "task_id": "papers-climate", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "844e8bbf-ae65-4047-9349-cbd1c1bf7b50", + "task_id": "papers-climate", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "cc32c600-4ef5-4fd8-b37c-6ae16c084f7e", + "task_id": "timezone-tokyo", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "workflow_success": false, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "de6093e2-ce59-4692-ae3c-da0801ac1675", + "task_id": "timezone-tokyo", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "workflow_success": false, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "a66e8303-b561-4d85-a2fa-baf46e0a953b", + "task_id": "timezone-tokyo", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "workflow_success": false, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "28718552-be81-486c-b313-75a5550ff3cc", + "task_id": "geocode-address", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "26e071b2-d139-4ee3-879c-5900578b1233", + "task_id": "geocode-address", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "e2f6f4c2-a8e7-4967-98b1-1a8c29aa3410", + "task_id": "geocode-address", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "fde5c3d4-8ff3-4813-a8c5-5e6a61ed3c2e", + "task_id": "ip-geolocation", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": false, + "workflow_success": false, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "cb085acd-e437-40df-9e07-e91343e2af9d", + "task_id": "ip-geolocation", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": false, + "workflow_success": false, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "143359c4-589b-437f-ae30-9a99b9f1636b", + "task_id": "ip-geolocation", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": false, + "workflow_success": false, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "58e4b1fd-8de8-4f85-8db1-478b57309b00", + "task_id": "crypto-price", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "f498a668-e254-4a1f-a109-0b11043a9a77", + "task_id": "crypto-price", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "3b48193b-6026-491e-8472-51034ae93749", + "task_id": "crypto-price", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "3d70bc1e-3063-41e6-8b57-40c6e6b963e2", + "task_id": "web-search", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "39c78597-f3db-4d54-b56e-9ed6e6b3d412", + "task_id": "web-search", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "cf50c76f-316f-4793-877b-2f51474c6981", + "task_id": "web-search", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "5bf899ee-7846-44cf-8e8a-ebeb8bfbff65", + "task_id": "air-quality", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "673ccda1-c286-49cd-9299-75ce17706e92", + "task_id": "air-quality", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "859aee1a-1670-4140-b9ac-91435124e6dc", + "task_id": "air-quality", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "70a47304-ae9e-4cb5-8331-5df6a8e83a43", + "task_id": "market-news", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "73355630-bc92-4e0a-a1c8-4f25f895a949", + "task_id": "market-news", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "f05cea11-e23c-463a-98f1-e5c0b4f51e88", + "task_id": "market-news", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "ae3e60aa-8ee9-4650-b3e2-6d906a42da8b", + "task_id": "domain-intel", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": false, + "executed": false, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": null, + "workflow_success": false, + "error_stage": "adapter", + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "cc1eebe6-e68b-45ba-8f32-827f98baa240", + "task_id": "domain-intel", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": false, + "executed": false, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": null, + "workflow_success": false, + "error_stage": "adapter", + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "19382342-d937-46dc-ba4a-c71139081673", + "task_id": "domain-intel", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": false, + "executed": false, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": null, + "workflow_success": false, + "error_stage": "adapter", + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "48e281c3-3a7a-4327-9911-427072bc6fcd", + "task_id": "image-search", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "d23e1fc9-03a5-4c76-8761-22c9de0e6231", + "task_id": "image-search", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "262b0d23-181c-4b1c-8241-8cfd530dbb96", + "task_id": "image-search", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "3d110e2a-4ee1-47f7-a018-f5ae9d166e46", + "task_id": "market-holidays", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "b619dc10-8cc0-4413-9219-1aea84479b72", + "task_id": "market-holidays", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "0ab6d79a-6510-4b11-966c-7063b6646f73", + "task_id": "market-holidays", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "245b91ee-988d-474f-8fd1-82da7c62183a", + "task_id": "earthquake-catalog", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "dc38691f-d8f6-40f8-9e77-9288c33a7e68", + "task_id": "earthquake-catalog", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "c1624700-5aba-4cfc-8a68-acabf3de024a", + "task_id": "earthquake-catalog", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": false, + "executed": false, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": null, + "workflow_success": false, + "error_stage": "adapter", + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "0e485876-22f8-4265-91e0-51990c3644e9", + "task_id": "company-profile", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": false, + "workflow_success": false, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "ea693fd1-b596-40b1-bd16-eec213eecb12", + "task_id": "company-profile", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": false, + "workflow_success": false, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "abf65b68-8d35-453f-89b6-d30447a432e7", + "task_id": "company-profile", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": false, + "workflow_success": false, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "a6ccdda6-dc1f-4765-873e-bd8c22197a6f", + "task_id": "sports-standings", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "a025004c-c94c-444b-a2c9-bb176ed51ea3", + "task_id": "sports-standings", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "f15c0986-0c0c-4eab-8dac-99cc4f6f7dea", + "task_id": "sports-standings", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "workflow_success": true, + "error_stage": null, + "metadata": { + "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", + "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", + "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + } + ] +} diff --git a/benchmarks/discover-call/results/README.md b/benchmarks/discover-call/results/README.md new file mode 100644 index 0000000..ec95c39 --- /dev/null +++ b/benchmarks/discover-call/results/README.md @@ -0,0 +1,57 @@ +# Published benchmark results + +## 2026-07-23 — `gpt-5.6-sol` + +This is the first controlled result for the versioned `tasks/v2.jsonl` contract +benchmark. It contains 18 tasks, three trials per task, and real QVeris calls. +No failed trial was removed or selectively rerun. + +| Metric | Result | +| --- | ---: | +| Runs | 54 | +| Completed and executed | 50 / 54 | +| Selection grounded | 100% | +| Inspection grounded | 100% | +| Required-parameter accuracy | 100% | +| Constraint accuracy | 75.93% | +| Call success among attempted calls | 88.00% (44 / 50) | +| Strict workflow success | 64.81% (35 / 54) | +| Workflow success, 95% Wilson interval | 51.48%–76.18% | + +Reproducibility metadata: + +- model: `gpt-5.6-sol` +- reasoning effort: `medium` +- Codex CLI: `0.144.1` +- adapter revision: + `72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium` +- toolkit revision: `fd08165e979a8ffaa030c56a4bd4853d805bf095` +- task set: `tasks/v2.jsonl` +- task-set SHA-256: + `67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba` +- API base URL: `https://qveris.ai/api/v1` +- discovery limit: `10` + +Artifacts: + +- [`2026-07-23-gpt-5.6-sol-v2.runs.jsonl`](2026-07-23-gpt-5.6-sol-v2.runs.jsonl) +- [`2026-07-23-gpt-5.6-sol-v2.summary.json`](2026-07-23-gpt-5.6-sol-v2.summary.json) + +### Interpretation + +The component metrics have different denominators. Call success is 44 of the 50 +attempted calls; strict workflow success is 35 of all 54 trials and additionally +requires grounded selection and inspection, complete required parameters, and +all task constraints. + +The four adapter-stage failures were retained: all three domain-intelligence +trials selected a catalog capability whose inspected contract exposed no usable +domain parameter, and one earthquake trial attempted prohibited tool use during +parameterization. Six well-formed calls returned `success: false`: all three IP +lookup trials and all three company-profile trials. + +Constraint misses include three valid `symbol=USD/EUR` executions that the +current v2 alias scorer does not split into separate base/quote constraints, +three URL-encoded news queries, and three timezone-list selections that executed +successfully but did not accept Tokyo as an input. These details are why the +strict workflow metric should not be interpreted as API availability alone. diff --git a/docs/en-US/discover-call-benchmark.md b/docs/en-US/discover-call-benchmark.md index c60003a..0ad51cb 100644 --- a/docs/en-US/discover-call-benchmark.md +++ b/docs/en-US/discover-call-benchmark.md @@ -39,7 +39,29 @@ provider error bodies must never be published. ## Published results -No official model result is published yet. The repository currently includes a -synthetic scorer fixture only; it is not a product-performance claim. The first -controlled run will be added with raw records and the exact methodology metadata -before any accuracy percentage is advertised. +The first controlled result was run on 2026-07-23 with `gpt-5.6-sol`, medium +reasoning effort, Codex CLI 0.144.1, `tasks/v2.jsonl`, and three trials per task. +All 54 trials used real execution and all failures remain in the denominator. + +| Metric | Result | +| --- | ---: | +| Completed and executed | 50 / 54 | +| Selection grounded | 100% | +| Inspection grounded | 100% | +| Required-parameter accuracy | 100% | +| Constraint accuracy | 75.93% | +| Call success among attempted calls | 88.00% (44 / 50) | +| Strict workflow success | 64.81% (35 / 54) | +| Workflow success, 95% Wilson interval | 51.48%–76.18% | + +Call success and workflow success have different denominators: strict workflow +success includes all 54 trials and also requires every preceding scoring +component to pass. Four trials failed during model parameterization and six +well-formed calls returned `success: false`. Additional constraint misses +include a valid combined currency pair that the current v2 alias scorer does +not split, URL-encoded news queries, and timezone-list selections that could not +accept the requested city. + +See the [result notes, immutable revisions, raw JSONL, and generated +summary](../../benchmarks/discover-call/results/README.md). The synthetic scorer +fixture remains test-only and is not a product-performance claim. diff --git a/docs/zh-CN/discover-call-benchmark.md b/docs/zh-CN/discover-call-benchmark.md index 7ee6f44..a454270 100644 --- a/docs/zh-CN/discover-call-benchmark.md +++ b/docs/zh-CN/discover-call-benchmark.md @@ -30,5 +30,25 @@ call 成功率，以及严格的端到端工作流成功率。工作流成功要 ## 已发布结果 -当前尚未发布正式模型结果。仓库中的 scorer fixture 仅用于验证评分器，不代表产品性能。首次受控运行 -会先提交原始记录与完整方法元数据，之后才会对外使用准确率数字。 +首次受控结果于 2026-07-23 运行，使用 `gpt-5.6-sol`、medium reasoning effort、 +Codex CLI 0.144.1、`tasks/v2.jsonl`，每个任务执行三个 trial。54 个 trial 均启用了真实执行， +所有失败均保留在分母中。 + +| 指标 | 结果 | +| --- | ---: | +| 完成参数化并执行 | 50 / 54 | +| 选择来自 discover | 100% | +| inspect 确认同一能力 | 100% | +| 必填参数完整率 | 100% | +| 任务约束准确率 | 75.93% | +| 已尝试 call 的成功率 | 88.00%（44 / 50） | +| 严格工作流成功率 | 64.81%（35 / 54） | +| 工作流成功率 95% Wilson 区间 | 51.48%–76.18% | + +call 成功率和工作流成功率的分母不同：严格工作流成功率包含全部 54 个 trial，并要求前置评分项 +全部通过。4 个 trial 在模型参数化阶段失败，另有 6 个参数格式有效的 call 返回 +`success: false`。其他约束未命中包括：当前 v2 别名评分器没有把有效的组合货币对拆成基础/报价 +货币、新闻查询值经过 URL 编码，以及选择了无法接收目标城市的时区列表能力。 + +完整的[结果说明、不可变 revision、原始 JSONL 与生成汇总](../../benchmarks/discover-call/results/README.md) +均已公开。scorer fixture 仍只用于测试，不代表产品性能。 From 7aade1138cb1135cdcba44e3ebb05ac9d78f218a Mon Sep 17 00:00:00 2001 From: Linfang Wang Date: Thu, 23 Jul 2026 15:24:15 +0800 Subject: [PATCH 05/19] docs: correct benchmark timeout attribution --- benchmarks/discover-call/results/README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/benchmarks/discover-call/results/README.md b/benchmarks/discover-call/results/README.md index ec95c39..e2228c7 100644 --- a/benchmarks/discover-call/results/README.md +++ b/benchmarks/discover-call/results/README.md @@ -46,9 +46,12 @@ all task constraints. The four adapter-stage failures were retained: all three domain-intelligence trials selected a catalog capability whose inspected contract exposed no usable -domain parameter, and one earthquake trial attempted prohibited tool use during -parameterization. Six well-formed calls returned `success: false`: all three IP -lookup trials and all three company-profile trials. +domain parameter, and the final earthquake trial reached the 120-second adapter +timeout during parameterization. The adapter intentionally records only a safe, +generic failure at this boundary, so no more specific model event can be +attributed to that original trial. Six well-formed calls returned +`success: false`: all three IP lookup trials and all three company-profile +trials. Constraint misses include three valid `symbol=USD/EUR` executions that the current v2 alias scorer does not split into separate base/quote constraints, From 5006c2a0789e2352944ffd87b1a154b96ec0566f Mon Sep 17 00:00:00 2001 From: Linfang Wang Date: Thu, 23 Jul 2026 15:52:27 +0800 Subject: [PATCH 06/19] benchmark: add oracle and v3 comparison lanes --- benchmarks/discover-call/README.md | 47 ++++++++++- .../discover-call/adapters/codex-cli.mjs | 35 +++++--- benchmarks/discover-call/adapters/oracle.mjs | 57 +++++++++++++ benchmarks/discover-call/src/harness.mjs | 50 +++++++++++ benchmarks/discover-call/src/run.mjs | 48 +++++++++-- benchmarks/discover-call/src/scoring.mjs | 46 +++++++++-- benchmarks/discover-call/tasks/v3.jsonl | 18 ++++ .../discover-call/test/adapter.test.mjs | 82 ++++++++++++++++++- .../discover-call/test/harness.test.mjs | 14 +++- .../discover-call/test/scoring.test.mjs | 70 ++++++++++++++++ docs/en-US/discover-call-benchmark.md | 21 ++++- docs/zh-CN/discover-call-benchmark.md | 15 +++- 12 files changed, 469 insertions(+), 34 deletions(-) create mode 100644 benchmarks/discover-call/adapters/oracle.mjs create mode 100644 benchmarks/discover-call/tasks/v3.jsonl diff --git a/benchmarks/discover-call/README.md b/benchmarks/discover-call/README.md index 77141b5..63970f1 100644 --- a/benchmarks/discover-call/README.md +++ b/benchmarks/discover-call/README.md @@ -34,7 +34,9 @@ The scorer reports these metrics separately: - **required-parameter accuracy**: fraction of required schema parameters supplied; - **constraint accuracy**: fraction of task facts represented through accepted parameter aliases; - **call success**: successful real executions among attempted calls; -- **workflow success**: all preceding checks equal 100% and the real call succeeds. +- **result validity**: successful calls returned a non-empty result; +- **workflow success**: all preceding checks equal 100%, the real call succeeds, + and its result is non-empty. The transport retries `429` and `503` responses before recording an API-stage failure, matching the public clients' transient-failure behavior. The summary @@ -58,6 +60,11 @@ Task sets are versioned and immutable once referenced by a published result: multiple parameterizable capabilities within the default discovery limit, and constraint aliases were taken from the actual parameter names of those capabilities. +- `tasks/v3.jsonl` — the 18-task comparison set for lane-based baselines. It + preserves the v2 prompts and discovery queries, adds explicit Oracle + candidates, recognizes combined parameters such as `symbol=USD/EUR`, and + applies URL decoding only where a task opts into it. An empty Oracle candidate + list records a verified discovery-coverage gap; it is not silently excluded. New tasks land as a new `tasks/vN.jsonl` revision, never by editing an existing file, and must still score deterministically at the contract level. @@ -74,6 +81,7 @@ must use separately named credentials for their model provider. ```bash node src/run.mjs \ --model provider/model-version \ + --lane pinned-model \ --adapter node \ --adapter-arg /absolute/path/to/model-adapter.mjs \ --adapter-revision adapter-git-sha-or-config-hash \ @@ -85,6 +93,43 @@ node src/run.mjs \ `--execute` performs billed calls. Omit it while validating an adapter, but do not publish dry-run records as benchmark results. +### Comparison lanes + +Use the lanes together; none is a substitute for the others: + +- `oracle` measures the current platform ceiling for the fixed discovery query. + The deterministic adapter selects only a configured candidate that appears in + the current Top 10 and supplies fixed valid parameters. A task with no suitable + returned candidate remains a strict failure. +- `pinned-model` measures longitudinal changes with one immutable model, + reasoning configuration, CLI, adapter revision, and task set. +- `current-model` measures the currently recommended model under the same task + contract. +- `model` is retained for backward-compatible records that predate explicit + lanes. + +The primary routing gap is `oracle workflow success - model workflow success`. +Compare component metrics and failure reasons alongside it so discovery +coverage, model routing, parameter construction, execution, and result validity +are not collapsed into one diagnosis. + +Run the deterministic Oracle lane with the same task file supplied both to the +harness and adapter: + +```bash +node src/run.mjs \ + --model oracle-v1 \ + --lane oracle \ + --adapter node \ + --adapter-arg "$PWD/adapters/oracle.mjs" \ + --adapter-arg "$PWD/tasks/v3.jsonl" \ + --adapter-revision toolkit-git-sha/oracle-v1 \ + --tasks tasks/v3.jsonl \ + --trials 3 \ + --execute \ + --output runs/oracle-v1.jsonl +``` + Score the records: ```bash diff --git a/benchmarks/discover-call/adapters/codex-cli.mjs b/benchmarks/discover-call/adapters/codex-cli.mjs index 4ba0a58..135e246 100644 --- a/benchmarks/discover-call/adapters/codex-cli.mjs +++ b/benchmarks/discover-call/adapters/codex-cli.mjs @@ -65,7 +65,7 @@ export function parseCodexEvents(stdout) { .split(/\r?\n/) .map((line) => line.trim()) .filter(Boolean); - if (lines.length === 0) throw new Error('Codex CLI returned no events'); + if (lines.length === 0) throw adapterFailure('Codex CLI returned no events', 'invalid_events'); let finalMessage; for (const line of lines) { @@ -73,34 +73,36 @@ export function parseCodexEvents(stdout) { try { event = JSON.parse(line); } catch { - throw new Error('Codex CLI returned invalid JSONL'); + throw adapterFailure('Codex CLI returned invalid JSONL', 'invalid_events'); } - if (!isObject(event)) throw new Error('Codex CLI returned an invalid event'); + if (!isObject(event)) throw adapterFailure('Codex CLI returned an invalid event', 'invalid_events'); if (event.type === 'error' || event.type === 'turn.failed') { - throw new Error('Codex CLI reported an unsuccessful result'); + throw adapterFailure('Codex CLI reported an unsuccessful result', 'model_failed'); } if ((event.type === 'item.started' || event.type === 'item.completed') && isObject(event.item)) { if (!ALLOWED_ITEM_TYPES.has(event.item.type)) { - throw new Error('Codex CLI attempted to use a tool'); + throw adapterFailure('Codex CLI attempted to use a tool', 'tool_use_rejected'); } if (event.type === 'item.completed' && event.item.type === 'agent_message') { if (typeof event.item.text !== 'string') { - throw new Error('Codex CLI returned an invalid agent message'); + throw adapterFailure('Codex CLI returned an invalid agent message', 'invalid_output'); } finalMessage = event.item.text; } } } - if (typeof finalMessage !== 'string') throw new Error('Codex CLI returned no structured output'); + if (typeof finalMessage !== 'string') { + throw adapterFailure('Codex CLI returned no structured output', 'invalid_output'); + } try { const result = JSON.parse(finalMessage); if (isObject(result)) return result; } catch { // Fall through to the generic, non-sensitive error below. } - throw new Error('Codex CLI returned no structured output'); + throw adapterFailure('Codex CLI returned no structured output', 'invalid_output'); } export async function invokeCodex(payload) { @@ -158,7 +160,7 @@ function runCodex(invocation) { child.stderr.resume(); child.on('error', () => { cleanup(); - reject(new Error('Codex CLI could not be started')); + reject(adapterFailure('Codex CLI could not be started', 'start_failed')); }); child.on('close', (code) => { cleanup(); @@ -166,9 +168,11 @@ function runCodex(invocation) { process.kill(process.pid, forwardedSignal); return; } - if (outputExceeded) return reject(new Error('Codex CLI output exceeded the adapter limit')); + if (outputExceeded) { + return reject(adapterFailure('Codex CLI output exceeded the adapter limit', 'output_limit')); + } if (code === 0) resolve(output); - else reject(new Error('Codex CLI invocation failed')); + else reject(adapterFailure('Codex CLI invocation failed', 'cli_failed')); }); child.stdin.end(invocation.stdin); }); @@ -178,6 +182,12 @@ function isObject(value) { return value !== null && typeof value === 'object' && !Array.isArray(value); } +function adapterFailure(message, code) { + const error = new Error(message); + error.adapterCode = code; + return error; +} + async function main() { let input = ''; for await (const chunk of process.stdin) input += chunk; @@ -187,7 +197,8 @@ async function main() { if (process.argv[1] === fileURLToPath(import.meta.url)) { main().catch((error) => { - process.stderr.write(`${error.message}\n`); + const code = typeof error?.adapterCode === 'string' ? error.adapterCode : 'invalid_output'; + process.stderr.write(`QVERIS_BENCHMARK_ADAPTER_ERROR=${code}\n`); process.exitCode = 1; }); } diff --git a/benchmarks/discover-call/adapters/oracle.mjs b/benchmarks/discover-call/adapters/oracle.mjs new file mode 100644 index 0000000..d4a1e82 --- /dev/null +++ b/benchmarks/discover-call/adapters/oracle.mjs @@ -0,0 +1,57 @@ +#!/usr/bin/env node + +import { readFile } from 'node:fs/promises'; + +try { + const tasksPath = parseTasksPath(process.argv.slice(2)); + const tasks = await readTasks(tasksPath); + const taskById = new Map(tasks.map((task) => [task.id, task])); + + let input = ''; + for await (const chunk of process.stdin) input += chunk; + const payload = JSON.parse(input); + const task = taskById.get(payload.input?.task_id); + if (!task?.oracle || !Array.isArray(task.oracle.candidates)) throw oracleFailure('missing_oracle'); + + if (payload.stage === 'select') { + const discovered = new Set( + Array.isArray(payload.input?.discovery?.results) + ? payload.input.discovery.results.map((tool) => tool?.tool_id).filter(Boolean) + : [], + ); + const candidate = task.oracle.candidates.find((item) => discovered.has(item.tool_id)); + process.stdout.write(JSON.stringify({ tool_id: candidate?.tool_id ?? null })); + } else if (payload.stage === 'parameterize') { + const selectedToolId = payload.input?.selected_tool?.tool_id; + const candidate = task.oracle.candidates.find((item) => item.tool_id === selectedToolId); + if (!candidate) throw oracleFailure('missing_oracle_candidate'); + process.stdout.write(JSON.stringify({ parameters: candidate.parameters })); + } else { + throw oracleFailure('unsupported_stage'); + } +} catch (error) { + process.stderr.write(`QVERIS_BENCHMARK_ADAPTER_ERROR=${error?.oracleCode || 'missing_oracle'}\n`); + process.exitCode = 2; +} + +async function readTasks(path) { + const text = await readFile(path, 'utf8'); + return text + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith('#')) + .map((line) => JSON.parse(line)); +} + +function parseTasksPath(argv) { + if (argv[0] && !argv[0].startsWith('--')) return argv[0]; + const index = argv.indexOf('--tasks'); + if (index === -1 || !argv[index + 1]) throw oracleFailure('missing_oracle'); + return argv[index + 1]; +} + +function oracleFailure(code) { + const error = new Error(code); + error.oracleCode = code; + return error; +} diff --git a/benchmarks/discover-call/src/harness.mjs b/benchmarks/discover-call/src/harness.mjs index 12c7741..03a5e09 100644 --- a/benchmarks/discover-call/src/harness.mjs +++ b/benchmarks/discover-call/src/harness.mjs @@ -27,6 +27,40 @@ export function validateTask(task) { if (constraint.match !== undefined && !['equals', 'contains'].includes(constraint.match)) { throw new Error(`Task ${task.id}: constraint ${constraint.id} has an unsupported match mode`); } + if ( + constraint.composite_aliases !== undefined && + (!Array.isArray(constraint.composite_aliases) || + constraint.composite_aliases.some((alias) => typeof alias !== 'string' || !alias)) + ) { + throw new Error(`Task ${task.id}: constraint ${constraint.id} has invalid composite_aliases`); + } + if ( + constraint.normalizers !== undefined && + (!Array.isArray(constraint.normalizers) || + constraint.normalizers.some((normalizer) => normalizer !== 'url_decode')) + ) { + throw new Error(`Task ${task.id}: constraint ${constraint.id} has invalid normalizers`); + } + } + if (task.oracle !== undefined) { + if (!task.oracle || typeof task.oracle !== 'object' || !Array.isArray(task.oracle.candidates)) { + throw new Error(`Task ${task.id}: oracle.candidates must be an array`); + } + if ( + task.oracle.candidates.length === 0 && + (typeof task.oracle.unavailable_reason !== 'string' || !task.oracle.unavailable_reason) + ) { + throw new Error(`Task ${task.id}: an empty oracle needs unavailable_reason`); + } + for (const candidate of task.oracle.candidates) { + if ( + typeof candidate?.tool_id !== 'string' || + !candidate.tool_id || + !isPlainObject(candidate.parameters) + ) { + throw new Error(`Task ${task.id}: every oracle candidate needs tool_id and parameters`); + } + } } return task; } @@ -127,12 +161,14 @@ async function runTrial({ task, model, trial, execute, limit, api, invokeAdapter }); record.call.success = response?.success === true; record.call.execution_id = response?.execution_id ?? null; + record.call.result_valid = response?.success === true ? hasMeaningfulResult(response?.result) : false; } record.status = 'completed'; } catch (error) { record.error = { stage: error?.benchmarkStage || 'unknown', + reason_code: safeReasonCode(error), message: safeErrorMessage(error), }; } @@ -271,6 +307,14 @@ function omitNullParameters(parameters) { return Object.fromEntries(Object.entries(parameters).filter(([, value]) => value !== null)); } +function hasMeaningfulResult(result) { + if (result === null || result === undefined) return false; + if (typeof result === 'string') return result.trim().length > 0; + if (Array.isArray(result)) return result.length > 0; + if (isPlainObject(result)) return Object.keys(result).length > 0; + return true; +} + function toolId(value) { const id = value?.tool_id; return typeof id === 'string' && id.trim() ? id.trim() : null; @@ -287,9 +331,15 @@ function isPlainObject(value) { function stageError(stage, message) { const error = new Error(message); error.benchmarkStage = stage; + error.benchmarkReason = stage === 'select' ? 'missing_tool_id' : 'invalid_parameters'; return error; } +function safeReasonCode(error) { + const reason = error?.benchmarkReason; + return typeof reason === 'string' && /^[a-z][a-z0-9_]{1,63}$/.test(reason) ? reason : null; +} + function safeErrorMessage(error) { const stage = error?.benchmarkStage || 'benchmark'; if (stage === 'api') return 'API request failed'; diff --git a/benchmarks/discover-call/src/run.mjs b/benchmarks/discover-call/src/run.mjs index 697fe7c..f6b2039 100644 --- a/benchmarks/discover-call/src/run.mjs +++ b/benchmarks/discover-call/src/run.mjs @@ -42,6 +42,7 @@ export async function main(argv = process.argv.slice(2)) { api, invokeAdapter, metadata: { + lane: options.lane, adapter_revision: options.adapterRevision, toolkit_revision: toolkitRevision, task_set_sha256: taskSetSha256, @@ -68,30 +69,44 @@ export function createProcessAdapter({ command, args = [], timeoutMs = 120_000 } child.stdin.on('error', () => {}); let stdout = ''; let stderr = ''; + let stdoutExceeded = false; + let stderrExceeded = false; const timer = setTimeout(() => { child.kill('SIGTERM'); - reject(adapterError('Adapter timed out')); + reject(adapterError('Adapter timed out', 'timeout')); }, timeoutMs); child.stdout.on('data', (chunk) => { stdout += chunk; - if (stdout.length > 1_000_000) child.kill('SIGTERM'); + if (stdout.length > 1_000_000 && !stdoutExceeded) { + stdoutExceeded = true; + child.kill('SIGTERM'); + } }); child.stderr.on('data', (chunk) => { stderr += chunk; - if (stderr.length > 100_000) child.kill('SIGTERM'); + if (stderr.length > 100_000 && !stderrExceeded) { + stderrExceeded = true; + child.kill('SIGTERM'); + } }); child.on('error', () => { clearTimeout(timer); - reject(adapterError('Adapter could not be started')); + reject(adapterError('Adapter could not be started', 'start_failed')); }); child.on('close', (code) => { clearTimeout(timer); - if (code !== 0) return reject(adapterError('Adapter exited unsuccessfully')); + if (stdoutExceeded) return reject(adapterError('Adapter output exceeded the limit', 'stdout_limit')); + if (stderrExceeded) return reject(adapterError('Adapter error output exceeded the limit', 'stderr_limit')); + if (code !== 0) { + return reject( + adapterError('Adapter exited unsuccessfully', adapterReasonFromStderr(stderr) || 'process_exit'), + ); + } try { resolvePromise(JSON.parse(stdout.trim())); } catch { - reject(adapterError('Adapter returned invalid JSON')); + reject(adapterError('Adapter returned invalid JSON', 'invalid_json')); } }); child.stdin.end(JSON.stringify(payload)); @@ -105,6 +120,7 @@ function parseArgs(argv) { trials: 3, execute: false, limit: 10, + lane: 'model', adapterArgs: [], adapterTimeoutMs: 120_000, }; @@ -121,6 +137,7 @@ function parseArgs(argv) { else if (arg === '--output') options.output = nextValue(argv, ++index, arg); else if (arg === '--trials') options.trials = integer(nextValue(argv, ++index, arg), arg); else if (arg === '--limit') options.limit = integer(nextValue(argv, ++index, arg), arg); + else if (arg === '--lane') options.lane = lane(nextValue(argv, ++index, arg)); else if (arg === '--adapter-timeout-ms') options.adapterTimeoutMs = integer(nextValue(argv, ++index, arg), arg); else throw new Error(`Unknown argument: ${arg}`); } @@ -142,12 +159,27 @@ function integer(value, flag) { return parsed; } -function adapterError(message) { +function lane(value) { + if (!['model', 'oracle', 'pinned-model', 'current-model'].includes(value)) { + throw new Error('--lane must be model, oracle, pinned-model, or current-model'); + } + return value; +} + +function adapterError(message, reason) { const error = new Error(message); error.benchmarkStage = 'adapter'; + error.benchmarkReason = reason; return error; } +function adapterReasonFromStderr(stderr) { + const match = stderr.match( + /QVERIS_BENCHMARK_ADAPTER_ERROR=(tool_use_rejected|model_failed|invalid_events|invalid_output|cli_failed|start_failed|output_limit|missing_oracle|missing_oracle_candidate|unsupported_stage)/, + ); + return match?.[1] ?? null; +} + function resolveToolkitRevision(explicit) { if (explicit) return explicit; if (process.env.GITHUB_SHA) return process.env.GITHUB_SHA; @@ -159,7 +191,7 @@ function resolveToolkitRevision(explicit) { } function helpText() { - return `QVeris discover→call benchmark\n\nUsage:\n node src/run.mjs --model MODEL --adapter COMMAND --adapter-revision REVISION [options]\n\nOptions:\n --adapter-arg VALUE Repeatable adapter argument (no shell parsing)\n --adapter-revision VALUE Immutable adapter source/config revision (required)\n --toolkit-revision VALUE Toolkit revision (auto-detected in git/GitHub Actions)\n --tasks PATH Task-set JSONL (default: tasks/v1.jsonl)\n --output PATH Run-record JSONL (default: benchmark-runs.jsonl)\n --trials N Trials per task (default: 3)\n --limit N Discover result limit (default: 10)\n --execute Perform billed call requests (required for workflow success)\n --adapter-timeout-ms N Per-stage adapter timeout (default: 120000)\n`; + return `QVeris discover→call benchmark\n\nUsage:\n node src/run.mjs --model MODEL --adapter COMMAND --adapter-revision REVISION [options]\n\nOptions:\n --adapter-arg VALUE Repeatable adapter argument (no shell parsing)\n --adapter-revision VALUE Immutable adapter source/config revision (required)\n --toolkit-revision VALUE Toolkit revision (auto-detected in git/GitHub Actions)\n --tasks PATH Task-set JSONL (default: tasks/v1.jsonl)\n --output PATH Run-record JSONL (default: benchmark-runs.jsonl)\n --trials N Trials per task (default: 3)\n --limit N Discover result limit (default: 10)\n --lane VALUE model, oracle, pinned-model, or current-model\n --execute Perform billed call requests (required for workflow success)\n --adapter-timeout-ms N Per-stage adapter timeout (default: 120000)\n`; } if (process.argv[1] === fileURLToPath(import.meta.url)) { diff --git a/benchmarks/discover-call/src/scoring.mjs b/benchmarks/discover-call/src/scoring.mjs index 6daf431..cda224e 100644 --- a/benchmarks/discover-call/src/scoring.mjs +++ b/benchmarks/discover-call/src/scoring.mjs @@ -53,6 +53,7 @@ function validateCompleteModelRun(model, tasks, records) { // Undefined is a valid shared value for old/synthetic fixtures, but mixing // it with runner metadata is rejected. const comparableMetadata = [ + 'lane', 'adapter_revision', 'toolkit_revision', 'task_set_sha256', @@ -86,13 +87,15 @@ export function scoreRecord(task, record) { const constraintAccuracy = mean(task.constraints.map((constraint) => constraintSatisfied(parameters, constraint))); const executed = record.call?.attempted === true; const callSuccess = executed ? record.call?.success === true : null; + const resultValid = executed && callSuccess ? record.call?.result_valid !== false : executed ? false : null; const workflowSuccess = completed && selectionGrounded && inspectionGrounded && requiredParameterAccuracy === 1 && constraintAccuracy === 1 && - callSuccess === true; + callSuccess === true && + resultValid === true; return { run_id: record.run_id, @@ -106,8 +109,10 @@ export function scoreRecord(task, record) { required_parameter_accuracy: round(requiredParameterAccuracy), constraint_accuracy: round(constraintAccuracy), call_success: callSuccess, + result_valid: resultValid, workflow_success: workflowSuccess, error_stage: record.error?.stage ?? null, + error_reason: record.error?.reason_code ?? null, metadata: record.metadata, }; } @@ -118,11 +123,16 @@ function aggregateModel(model, records) { const interval = wilsonInterval(workflowWins, records.length); const taskCount = new Set(records.map((record) => record.task_id)).size; const failuresByStage = {}; + const failuresByReason = {}; for (const record of records) { if (record.error_stage) failuresByStage[record.error_stage] = (failuresByStage[record.error_stage] || 0) + 1; + if (record.error_reason) { + failuresByReason[record.error_reason] = (failuresByReason[record.error_reason] || 0) + 1; + } } return { model, + lane: records[0]?.metadata?.lane ?? 'model', tasks: taskCount, trials_per_task: records.length / taskCount, runs: records.length, @@ -133,9 +143,11 @@ function aggregateModel(model, records) { required_parameter_accuracy: round(mean(records.map((record) => record.required_parameter_accuracy))), constraint_accuracy: round(mean(records.map((record) => record.constraint_accuracy))), call_success_rate: executed.length ? round(mean(executed.map((record) => record.call_success))) : null, + result_valid_rate: executed.length ? round(mean(executed.map((record) => record.result_valid))) : null, workflow_success_rate: round(workflowWins / records.length), workflow_success_wilson_95: interval.map(round), failures_by_stage: failuresByStage, + failures_by_reason: failuresByReason, }; } @@ -145,20 +157,44 @@ function constraintSatisfied(parameters, constraint) { const actual = parameters[alias]; const expected = constraint.value; if (constraint.match === 'contains') { - if (normalize(actual).includes(normalize(expected))) return 1; - } else if (normalize(actual) === normalize(expected)) { + if (normalize(actual, constraint).includes(normalize(expected, constraint))) return 1; + } else if (normalize(actual, constraint) === normalize(expected, constraint)) { return 1; } } + for (const alias of array(constraint.composite_aliases)) { + if (!hasValue(parameters[alias])) continue; + if (normalize(parameters[alias], constraint).includes(normalize(constraint.value, constraint))) return 1; + } return 0; } -function normalize(value) { - if (typeof value === 'string') return value.trim().toLowerCase(); +function normalize(value, constraint = {}) { + if (typeof value === 'string') { + let normalized = value.trim().toLowerCase(); + for (const normalizer of array(constraint.normalizers)) { + if (normalizer === 'url_decode') normalized = safeUrlDecode(normalized); + } + return normalized; + } if (typeof value === 'number' || typeof value === 'boolean') return String(value); return JSON.stringify(value); } +function safeUrlDecode(value) { + let decoded = value.replace(/\+/g, ' '); + for (let attempt = 0; attempt < 2; attempt++) { + try { + const next = decodeURIComponent(decoded); + if (next === decoded) break; + decoded = next; + } catch { + break; + } + } + return decoded; +} + function hasValue(value) { return value !== undefined && value !== null && value !== ''; } diff --git a/benchmarks/discover-call/tasks/v3.jsonl b/benchmarks/discover-call/tasks/v3.jsonl new file mode 100644 index 0000000..d919866 --- /dev/null +++ b/benchmarks/discover-call/tasks/v3.jsonl @@ -0,0 +1,18 @@ +{"id":"weather-london","prompt":"Find a weather forecast capability and request the forecast for London.","discover_query":"weather forecast API","constraints":[{"id":"location","aliases":["city","location","place","query"],"value":"London"}],"oracle":{"candidates":[{"tool_id":"visualcrossing.timeline.retrieve.v1","parameters":{"location":"London","unitGroup":"metric","contentType":"json"}}]}} +{"id":"stock-aapl","prompt":"Find a market-data capability and request the latest price for AAPL.","discover_query":"stock quote market data API","constraints":[{"id":"symbol","aliases":["symbol","ticker","code"],"value":"AAPL"}],"oracle":{"candidates":[{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3","parameters":{"symbol":"AAPL","format":"JSON"}}]}} +{"id":"exchange-usd-eur","prompt":"Find a foreign-exchange capability and request a USD to EUR conversion.","discover_query":"currency exchange conversion API","constraints":[{"id":"base_currency","aliases":["base","base_currency","from","from_currency"],"composite_aliases":["symbol","pair"],"value":"USD"},{"id":"quote_currency","aliases":["quote","quote_currency","to","to_currency","target_currency"],"composite_aliases":["symbol","pair"],"value":"EUR"}],"oracle":{"candidates":[{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4","parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"}}]}} +{"id":"news-ai","prompt":"Find a news-search capability and search for recent artificial intelligence news.","discover_query":"recent news search API","constraints":[{"id":"query","aliases":["query","q","keyword","keywords","topic"],"value":"artificial intelligence","match":"contains","normalizers":["url_decode"]}],"oracle":{"candidates":[{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df","parameters":{"q":"artificial intelligence","language":"en","sortBy":"publishedAt","pageSize":20,"page":1}}]}} +{"id":"papers-climate","prompt":"Find a scholarly-search capability and search for papers about climate risk.","discover_query":"academic paper search API","constraints":[{"id":"query","aliases":["query","q","keyword","keywords","topic"],"value":"climate risk","match":"contains"}],"oracle":{"candidates":[{"tool_id":"qveris_academic.paper_aggregation_v1","parameters":{"query":"climate risk","limit":10}}]}} +{"id":"timezone-tokyo","prompt":"Find a timezone capability and request timezone information for Tokyo.","discover_query":"timezone lookup API","constraints":[{"id":"location","aliases":["city","location","place","query"],"value":"Tokyo"}],"oracle":{"candidates":[],"unavailable_reason":"The fixed discovery query currently returns no top-10 capability that accepts a city or coordinates for a Tokyo timezone lookup."}} +{"id":"geocode-address","prompt":"Find a geocoding capability and convert the street address '北京市海淀区上地十街10号' to coordinates.","discover_query":"geocoding convert address to coordinates API","constraints":[{"id":"address","aliases":["address","q","query","location"],"value":"上地十街","match":"contains"}],"oracle":{"candidates":[{"tool_id":"amap_webservice.geocode.geo.retrieve.v3","parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"}}]}} +{"id":"ip-geolocation","prompt":"Find an IP geolocation capability and look up details for the IP address 8.8.8.8.","discover_query":"IP address geolocation lookup API","constraints":[{"id":"ip","aliases":["ip","q","query","ip_address","address"],"value":"8.8.8.8"}],"oracle":{"candidates":[{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1","parameters":{"ip":"8.8.8.8"}},{"tool_id":"virustotal.ip.report.retrieve.v3.ab34b0d1","parameters":{"ip":"8.8.8.8"}}]}} +{"id":"crypto-price","prompt":"Find a cryptocurrency market-data capability and request the latest price for Bitcoin (BTC).","discover_query":"cryptocurrency price API","constraints":[{"id":"symbol","aliases":["symbol","symbols","slug","id","pair","base_currency"],"value":"BTC","match":"contains"}],"oracle":{"candidates":[{"tool_id":"binance.ticker.price.list.v3.8675eca0","parameters":{"symbol":"BTCUSDT"}}]}} +{"id":"web-search","prompt":"Find a web search capability and search the web for 'model context protocol'.","discover_query":"web search API","constraints":[{"id":"query","aliases":["q","query","keyword","keywords","text"],"value":"model context protocol","match":"contains","normalizers":["url_decode"]}],"oracle":{"candidates":[{"tool_id":"cloudsway.smart_search_overseas","parameters":{"q":"model context protocol","count":"10"}}]}} +{"id":"air-quality","prompt":"Find an air quality capability and request the current air pollution for latitude 51.5, longitude -0.12.","discover_query":"air quality index API","constraints":[{"id":"latitude","aliases":["lat","latitude"],"value":51.5},{"id":"longitude","aliases":["lon","lng","longitude"],"value":-0.12}],"oracle":{"candidates":[{"tool_id":"openweathermap.air_pollution.current.v2_5","parameters":{"lat":51.5,"lon":-0.12}}]}} +{"id":"market-news","prompt":"Find a financial news capability and request general market news.","discover_query":"stock market news API","constraints":[{"id":"category","aliases":["category","topic","q","query"],"value":"general","match":"contains"}],"oracle":{"candidates":[{"tool_id":"finnhub.market.news.list.v1","parameters":{"category":"general"}}]}} +{"id":"domain-intel","prompt":"Find a domain intelligence capability and look up the domain example.com.","discover_query":"domain whois lookup API","constraints":[{"id":"domain","aliases":["domain","domain_name","q","query","target","host"],"value":"example.com","match":"contains"}],"oracle":{"candidates":[{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740","parameters":{"domain":"example.com"}},{"tool_id":"virustotal.domain_reports.retrieve.v3.c5459740","parameters":{"domain":"example.com"}}]}} +{"id":"image-search","prompt":"Find an image search capability and search for images of the Golden Gate Bridge.","discover_query":"image search API","constraints":[{"id":"query","aliases":["q","query","keyword","keywords"],"value":"golden gate","match":"contains","normalizers":["url_decode"]}],"oracle":{"candidates":[{"tool_id":"brave_search.images.search.list.v1","parameters":{"q":"Golden Gate Bridge"}}]}} +{"id":"market-holidays","prompt":"Find a market calendar capability and request stock market holidays for the exchange code US.","discover_query":"stock market holiday calendar API","constraints":[{"id":"exchange","aliases":["exchange","market","exchange_code"],"value":"US"}],"oracle":{"candidates":[{"tool_id":"finnhub.stock.marketholiday.retrieve.v1","parameters":{"exchange":"US"}}]}} +{"id":"earthquake-catalog","prompt":"Find an earthquake catalog capability and query recent events in geojson format.","discover_query":"recent earthquakes data API","constraints":[{"id":"format","aliases":["format","output","outputformat"],"value":"geojson"}],"oracle":{"candidates":[{"tool_id":"usgs_earthquake_catalog.event.events.list.v1","parameters":{"format":"geojson"}}]}} +{"id":"company-profile","prompt":"Find a company fundamentals capability and look up the company profile for CIK 0000320193.","discover_query":"company information lookup API","constraints":[{"id":"cik","aliases":["cik","symbol","company","q","query"],"value":"320193","match":"contains"}],"oracle":{"candidates":[{"tool_id":"mcp_gildata.uscompanybasicinfo.v1","parameters":{"query":"0000320193"}},{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2","parameters":{"cik":"0000320193"}}]}} +{"id":"sports-standings","prompt":"Find a sports data capability and request league standings for league 39 in season 2025.","discover_query":"sports league standings API","constraints":[{"id":"league","aliases":["league","league_id"],"value":39},{"id":"season","aliases":["season","year"],"value":2025}],"oracle":{"candidates":[{"tool_id":"api_sports.standings.list.v1.39568ba4","parameters":{"league":39,"season":"2025"}}]}} diff --git a/benchmarks/discover-call/test/adapter.test.mjs b/benchmarks/discover-call/test/adapter.test.mjs index 574d13d..9e50de0 100644 --- a/benchmarks/discover-call/test/adapter.test.mjs +++ b/benchmarks/discover-call/test/adapter.test.mjs @@ -12,6 +12,8 @@ import { } from '../adapters/codex-cli.mjs'; const adapterPath = resolve(fileURLToPath(new URL('../adapters/first-result.mjs', import.meta.url))); +const oracleAdapterPath = resolve(fileURLToPath(new URL('../adapters/oracle.mjs', import.meta.url))); +const v3TasksPath = resolve(fileURLToPath(new URL('../tasks/v3.jsonl', import.meta.url))); test('process adapter exchanges one JSON object without shell parsing', async () => { const invoke = createProcessAdapter({ command: process.execPath, args: [adapterPath], timeoutMs: 5_000 }); @@ -56,7 +58,81 @@ test('process adapter reports startup failures without an unhandled stdin error' command: `missing-adapter-${process.pid}`, timeoutMs: 5_000, }); - await assert.rejects(invoke({ stage: 'select' }), /could not be started/); + await assert.rejects(invoke({ stage: 'select' }), (error) => { + assert.equal(error.benchmarkReason, 'start_failed'); + return /could not be started/.test(error.message); + }); +}); + +test('process adapter records safe failure reasons without provider stderr', async () => { + const invoke = createProcessAdapter({ + command: process.execPath, + args: [ + '-e', + "process.stdin.resume(); process.stdin.on('end', () => { process.stderr.write('private body\\nQVERIS_BENCHMARK_ADAPTER_ERROR=tool_use_rejected\\n'); process.exitCode = 2; })", + ], + timeoutMs: 5_000, + }); + await assert.rejects(invoke({ stage: 'select' }), (error) => { + assert.equal(error.benchmarkReason, 'tool_use_rejected'); + assert.equal(error.message.includes('private body'), false); + return true; + }); +}); + +test('process adapter classifies timeouts', async () => { + const invoke = createProcessAdapter({ + command: process.execPath, + args: ['-e', "process.stdin.resume(); setInterval(() => {}, 1000)"], + timeoutMs: 25, + }); + await assert.rejects(invoke({ stage: 'select' }), (error) => { + assert.equal(error.benchmarkReason, 'timeout'); + return true; + }); +}); + +test('Oracle adapter selects only configured candidates present in discovery', async () => { + const invoke = createProcessAdapter({ + command: process.execPath, + args: [oracleAdapterPath, v3TasksPath], + timeoutMs: 5_000, + }); + assert.deepEqual( + await invoke({ + stage: 'select', + input: { + task_id: 'weather-london', + discovery: { + results: [ + { tool_id: 'unrelated.tool' }, + { tool_id: 'visualcrossing.timeline.retrieve.v1' }, + ], + }, + }, + }), + { tool_id: 'visualcrossing.timeline.retrieve.v1' }, + ); + assert.deepEqual( + await invoke({ + stage: 'parameterize', + input: { + task_id: 'weather-london', + selected_tool: { tool_id: 'visualcrossing.timeline.retrieve.v1' }, + }, + }), + { parameters: { location: 'London', unitGroup: 'metric', contentType: 'json' } }, + ); + assert.deepEqual( + await invoke({ + stage: 'select', + input: { + task_id: 'timezone-tokyo', + discovery: { results: [{ tool_id: 'api_sports.timezone.retrieve.v1.e993615d' }] }, + }, + }), + { tool_id: null }, + ); }); test('Claude adapter preserves canonical messages and response schema', () => { @@ -167,11 +243,11 @@ test('Codex adapter extracts the final structured message and rejects tool use', item: { type: 'command_execution', command: 'pwd' }, }), ), - /attempted to use a tool/, + (error) => error.adapterCode === 'tool_use_rejected' && /attempted to use a tool/.test(error.message), ); assert.throws( () => parseCodexEvents(JSON.stringify({ type: 'turn.failed', error: { message: 'private' } })), - /unsuccessful result/, + (error) => error.adapterCode === 'model_failed' && /unsuccessful result/.test(error.message), ); assert.throws( () => diff --git a/benchmarks/discover-call/test/harness.test.mjs b/benchmarks/discover-call/test/harness.test.mjs index 19711df..450512c 100644 --- a/benchmarks/discover-call/test/harness.test.mjs +++ b/benchmarks/discover-call/test/harness.test.mjs @@ -38,7 +38,7 @@ test('orchestrates discover, select, inspect, parameterize, and call', async () }, async call(input) { events.push(['call', input]); - return { success: true, execution_id: 'exec-1' }; + return { success: true, execution_id: 'exec-1', result: { forecast: [{ temperature: 20 }] } }; }, }, async invokeAdapter(payload) { @@ -64,6 +64,7 @@ test('orchestrates discover, select, inspect, parameterize, and call', async () assert.deepEqual(records[0].inspection.required_parameters, ['city']); assert.deepEqual(records[0].parameters, { city: 'London' }); assert.equal(records[0].call.success, true); + assert.equal(records[0].call.result_valid, true); assert.equal(records[0].call.execution_id, 'exec-1'); assert.equal(records[0].metadata.adapter_revision, 'adapter-sha'); }); @@ -136,6 +137,7 @@ test('records adapter failures without copying their error text', async () => { async invokeAdapter() { const error = new Error('adapter failed with secret-token'); error.benchmarkStage = 'adapter'; + error.benchmarkReason = 'tool_use_rejected'; throw error; }, }); @@ -143,11 +145,21 @@ test('records adapter failures without copying their error text', async () => { assert.equal(records[0].status, 'failed'); assert.equal(records[0].error.stage, 'adapter'); assert.equal(records[0].error.message, 'Adapter invocation failed'); + assert.equal(records[0].error.reason_code, 'tool_use_rejected'); assert.equal(JSON.stringify(records[0]).includes('secret-token'), false); }); test('validates task constraints and trial bounds', async () => { assert.throws(() => validateTask({ id: 'bad', prompt: 'x', constraints: [] }), /constraints/); + assert.throws( + () => + validateTask({ + ...task, + constraints: [{ ...task.constraints[0], normalizers: ['unknown'] }], + }), + /normalizers/, + ); + assert.throws(() => validateTask({ ...task, oracle: { candidates: [] } }), /unavailable_reason/); await assert.rejects( runBenchmark({ tasks: [task], model: 'model-a', trials: 0, api: {}, invokeAdapter() {} }), /trials/, diff --git a/benchmarks/discover-call/test/scoring.test.mjs b/benchmarks/discover-call/test/scoring.test.mjs index 55667c5..3895925 100644 --- a/benchmarks/discover-call/test/scoring.test.mjs +++ b/benchmarks/discover-call/test/scoring.test.mjs @@ -49,6 +49,74 @@ test('does not count dry runs as workflow success', () => { assert.equal(result.workflow_success, false); }); +test('v3 constraints support composite parameters and explicit URL decoding', () => { + const exchange = { + id: 'exchange', + prompt: 'USD to EUR', + constraints: [ + { id: 'base', aliases: ['base'], composite_aliases: ['symbol'], value: 'USD' }, + { id: 'quote', aliases: ['quote'], composite_aliases: ['symbol'], value: 'EUR' }, + ], + }; + const news = { + id: 'news', + prompt: 'AI news', + constraints: [ + { + id: 'query', + aliases: ['q'], + value: 'artificial intelligence', + match: 'contains', + normalizers: ['url_decode'], + }, + ], + }; + const baseRecord = { + run_id: 'run', + model: 'model-a', + trial: 1, + status: 'completed', + discovery: { result_tool_ids: ['tool'] }, + selection: { tool_id: 'tool' }, + inspection: { returned_tool_ids: ['tool'], required_parameters: [] }, + call: { attempted: true, success: true, result_valid: true }, + }; + + assert.equal( + scoreRecord(exchange, { ...baseRecord, task_id: exchange.id, parameters: { symbol: 'USD/EUR' } }) + .constraint_accuracy, + 1, + ); + assert.equal( + scoreRecord(news, { + ...baseRecord, + run_id: 'run-news', + task_id: news.id, + parameters: { q: '%22artificial%20intelligence%22' }, + }).constraint_accuracy, + 1, + ); +}); + +test('requires a meaningful result when result validity is recorded', () => { + const record = { + run_id: 'run-empty', + task_id: task.id, + model: 'model-a', + trial: 1, + status: 'completed', + discovery: { result_tool_ids: ['weather.forecast'] }, + selection: { tool_id: 'weather.forecast' }, + inspection: { returned_tool_ids: ['weather.forecast'], required_parameters: ['city'] }, + parameters: { city: 'London' }, + call: { attempted: true, success: true, result_valid: false }, + }; + const result = scoreRecord(task, record); + assert.equal(result.call_success, true); + assert.equal(result.result_valid, false); + assert.equal(result.workflow_success, false); +}); + test('aggregates per model with parameter and workflow rates', () => { const records = [ { @@ -90,6 +158,7 @@ test('aggregates per model with parameter and workflow rates', () => { calls: summary.models[0].call_success_rate, workflow: summary.models[0].workflow_success_rate, failures: summary.models[0].failures_by_stage, + failureReasons: summary.models[0].failures_by_reason, }, { tasks: 1, @@ -101,6 +170,7 @@ test('aggregates per model with parameter and workflow rates', () => { calls: 0.5, workflow: 0.5, failures: {}, + failureReasons: {}, }, ); assert.equal(summary.models[0].workflow_success_wilson_95.length, 2); diff --git a/docs/en-US/discover-call-benchmark.md b/docs/en-US/discover-call-benchmark.md index 0ad51cb..1625d47 100644 --- a/docs/en-US/discover-call-benchmark.md +++ b/docs/en-US/discover-call-benchmark.md @@ -19,8 +19,10 @@ execution is enabled. The scorer publishes grounded selection, grounded inspection, required-parameter accuracy, task-constraint accuracy, call success, and strict end-to-end workflow -success. Workflow success requires every component to pass and therefore cannot -be reported from dry runs. The aggregate includes a 95% Wilson interval. +success. A successful call must also return a non-empty result to satisfy the +result-validity and strict-workflow gates. Workflow success requires every +component to pass and therefore cannot be reported from dry runs. The aggregate +includes a 95% Wilson interval and safe failure counts by stage and reason. Transient `429` and `503` responses are retried first; exhausted API failures remain in the denominator and are reported separately by failure stage. @@ -30,6 +32,21 @@ the same task while still requiring its choice to come from the actual discovery response. Model adapters receive canonical messages and a response schema but never receive the scorer's ground-truth constraints. +The v3 task set adds explicit handling for combined parameters such as +`symbol=USD/EUR` and opt-in URL decoding, without changing the historical v2 +scoring contract. It also supports three complementary comparison lanes: + +- the deterministic Oracle lane measures the current platform ceiling by + selecting a fixed valid candidate only when it appears in the query's Top 10; +- the pinned-model lane measures changes over time with an immutable model and + adapter configuration; +- the current-model lane measures the currently recommended model. + +The difference between Oracle and model strict-workflow success is the routing +gap. Component metrics and failure reasons identify whether the remaining loss +comes from discovery coverage, model routing, parameter construction, +execution, or result validity. + ## Reproducibility requirements Published results must retain failures, use at least three trials per task, name diff --git a/docs/zh-CN/discover-call-benchmark.md b/docs/zh-CN/discover-call-benchmark.md index a454270..04585f6 100644 --- a/docs/zh-CN/discover-call-benchmark.md +++ b/docs/zh-CN/discover-call-benchmark.md @@ -13,8 +13,9 @@ QVeris 评测完整的 Agent 工作流，而不是只使用无法验证的形容 `inspect`，让模型根据当前参数 schema 构造参数，并在启用执行时完成真实 `call`。 评分分别报告选择是否来自 discover、inspect 是否确认同一能力、必填参数完整率、任务约束准确率、 -call 成功率，以及严格的端到端工作流成功率。工作流成功要求全部分项通过，因此不能用 dry run -生成。汇总同时给出 95% Wilson 区间。 +call 成功率、结果有效率，以及严格的端到端工作流成功率。成功的 call 还必须返回非空结果，才能 +通过结果有效性与严格工作流门槛。工作流成功要求全部分项通过，因此不能用 dry run 生成。汇总 +同时给出 95% Wilson 区间，以及经过安全归类的失败阶段和原因计数。 `429` 和 `503` 瞬时响应会先按客户端语义重试；重试耗尽的 API 失败仍保留在分母中，并按失败阶段 单独报告。 @@ -22,6 +23,16 @@ call 成功率，以及严格的端到端工作流成功率。工作流成功要 误判为失败，同时仍要求模型的选择来自真实 discovery 响应。Model adapter 只会收到固定 messages 与 response schema，不会得到 scorer 使用的 ground-truth constraints。 +v3 任务集在不改变历史 v2 评分契约的前提下，明确支持 `symbol=USD/EUR` 这类组合参数，以及由任务 +显式启用的 URL 解码，并提供三条互补的对比基线： + +- 确定性的 Oracle lane 只会在固定查询 Top 10 中出现预设有效候选时选择它，用于衡量当前平台上限； +- pinned-model lane 固定不可变模型和 adapter 配置，用于纵向观察版本变化； +- current-model lane 使用当前推荐模型，用于观察当前能力。 + +Oracle 与模型的严格工作流成功率之差就是 routing gap。还需要结合各分项指标和失败原因，判断损失 +来自 discovery 覆盖、模型路由、参数构造、执行，还是结果有效性。 + ## 可复现要求 公开结果必须保留失败样本，每个任务至少执行三个 trial，注明不可变的模型版本与 adapter revision， From be3e0449d6434bd139e9b7373c14fb8b34f3f27c Mon Sep 17 00:00:00 2001 From: Linfang Wang Date: Thu, 23 Jul 2026 18:12:02 +0800 Subject: [PATCH 07/19] docs: publish v3 benchmark baselines --- .../2026-07-23-gpt-5.6-sol-v3.runs.jsonl | 54 + .../2026-07-23-gpt-5.6-sol-v3.summary.json | 1439 +++++++++++++++++ .../2026-07-23-oracle-v1-v3.runs.jsonl | 54 + .../2026-07-23-oracle-v1-v3.summary.json | 1439 +++++++++++++++++ benchmarks/discover-call/results/README.md | 79 +- docs/en-US/discover-call-benchmark.md | 47 +- docs/zh-CN/discover-call-benchmark.md | 38 +- 7 files changed, 3106 insertions(+), 44 deletions(-) create mode 100644 benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.runs.jsonl create mode 100644 benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.summary.json create mode 100644 benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.runs.jsonl create mode 100644 benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.summary.json diff --git a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.runs.jsonl b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.runs.jsonl new file mode 100644 index 0000000..6a66e29 --- /dev/null +++ b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.runs.jsonl @@ -0,0 +1,54 @@ +{"schema_version":1,"benchmark_version":"v1","run_id":"23187609-52d7-44b6-b60e-91f68d32777c","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"weather-london","trial":1,"started_at":"2026-07-23T09:40:25.828Z","status":"completed","discovery":{"result_tool_ids":["weather_gov.gridpoints.forecast.retrieve.v1.40a9a3ce","weather_gov.gridpoints.forecast.retrieve.v3.29bbec80","amap_webservice.weather.weatherinfo.retrieve.v3","weather_gov.gridpoints.forecast.hourly.retrieve.v3.808ea240","weather_gov.icons.retrieve.v3.ad0b4d80","weather_gov.products.retrieve.v3.43ee57b8","visualcrossing.timeline.retrieve.v1","weather_gov.gridpoints.retrieve.v3.a0c89365","weather_gov.gridpoints.retrieve.v3.2cf46664","weather_gov.products.retrieve.v3.511d7df1"]},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"returned_tool_ids":["visualcrossing.timeline.retrieve.v1"],"required_parameters":["location"]},"parameters":{"location":"London"},"call":{"attempted":true,"success":true,"execution_id":"2cb5aa8e-91fa-49b8-a383-880e16acda3e","result_valid":true},"finished_at":"2026-07-23T09:40:51.455Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"d2bfdd97-1ac6-4388-aab6-dc04bd389b4d","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"weather-london","trial":2,"started_at":"2026-07-23T09:40:51.455Z","status":"completed","discovery":{"result_tool_ids":["weather_gov.gridpoints.forecast.retrieve.v1.40a9a3ce","weather_gov.gridpoints.forecast.retrieve.v3.29bbec80","amap_webservice.weather.weatherinfo.retrieve.v3","weather_gov.gridpoints.forecast.hourly.retrieve.v3.808ea240","weather_gov.icons.retrieve.v3.ad0b4d80","weather_gov.products.retrieve.v3.43ee57b8","visualcrossing.timeline.retrieve.v1","weather_gov.gridpoints.retrieve.v3.a0c89365","weather_gov.gridpoints.retrieve.v3.2cf46664","weather_gov.products.retrieve.v3.511d7df1"]},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"returned_tool_ids":["visualcrossing.timeline.retrieve.v1"],"required_parameters":["location"]},"parameters":{"location":"London"},"call":{"attempted":true,"success":true,"execution_id":"cc681d1e-1538-4244-b3ac-ed123a0e37c4","result_valid":true},"finished_at":"2026-07-23T09:41:19.367Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"9dc77ec7-6fea-42e6-8786-f8970a33da5d","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"weather-london","trial":3,"started_at":"2026-07-23T09:41:19.367Z","status":"completed","discovery":{"result_tool_ids":["weather_gov.gridpoints.forecast.retrieve.v1.40a9a3ce","weather_gov.gridpoints.forecast.retrieve.v3.29bbec80","amap_webservice.weather.weatherinfo.retrieve.v3","weather_gov.gridpoints.forecast.hourly.retrieve.v3.808ea240","weather_gov.icons.retrieve.v3.ad0b4d80","weather_gov.products.retrieve.v3.43ee57b8","visualcrossing.timeline.retrieve.v1","weather_gov.gridpoints.retrieve.v3.a0c89365","weather_gov.gridpoints.retrieve.v3.2cf46664","weather_gov.products.retrieve.v3.511d7df1"]},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"returned_tool_ids":["visualcrossing.timeline.retrieve.v1"],"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"execution_id":"5c4cd3ab-c6c5-4cbe-b494-c60d8e290c44","result_valid":true},"finished_at":"2026-07-23T09:41:45.578Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"98d4488a-8f0e-48fe-8743-ed08df8940f8","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-23T09:41:45.578Z","status":"completed","discovery":{"result_tool_ids":["eodhd.live_v2.us_quote_delayed.retrieve.v1.f0e13d45","twelvedata.quote.retrieve.v1.affbefe3","massive_stocks.options.reference.conditions.list.v3.41a82e07","alphavantage.realtime_bulk_quotes.retrieve.v1","fiu_mcp_server.postv1stockquoteextend.create.v2.a30e86e7","hangseng_polysource.a_shares_live_quote.query.v2.10fe0581","mcp_gildata.usstockdailyquotes.v1","financialmodelingprep.cusip.search.v1.ad19189f","fiu_mcp_server.postv3stockquoteextend.create.v2.b35a7f02","massive_stocks.stocks.marketstatus.retrieve.v1.3045ae6e"]},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"returned_tool_ids":["twelvedata.quote.retrieve.v1.affbefe3"],"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"20ac0b50-e5e7-45fe-94da-dae66e4faa18","result_valid":true},"finished_at":"2026-07-23T09:42:10.683Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"450cf5b6-8404-4be2-b73e-c28a6ae3227d","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-23T09:42:10.683Z","status":"completed","discovery":{"result_tool_ids":["eodhd.live_v2.us_quote_delayed.retrieve.v1.f0e13d45","twelvedata.quote.retrieve.v1.affbefe3","massive_stocks.options.reference.conditions.list.v3.41a82e07","alphavantage.realtime_bulk_quotes.retrieve.v1","fiu_mcp_server.postv1stockquoteextend.create.v2.a30e86e7","hangseng_polysource.a_shares_live_quote.query.v2.10fe0581","mcp_gildata.usstockdailyquotes.v1","financialmodelingprep.cusip.search.v1.ad19189f","fiu_mcp_server.postv3stockquoteextend.create.v2.b35a7f02","massive_stocks.stocks.marketstatus.retrieve.v1.3045ae6e"]},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"returned_tool_ids":["twelvedata.quote.retrieve.v1.affbefe3"],"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"a8db7a5c-7f32-4569-ab8d-2f7e913ab169","result_valid":true},"finished_at":"2026-07-23T09:42:35.790Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"456b1477-ec0a-4772-887c-8971ac437a95","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-23T09:42:35.790Z","status":"completed","discovery":{"result_tool_ids":["eodhd.live_v2.us_quote_delayed.retrieve.v1.f0e13d45","twelvedata.quote.retrieve.v1.affbefe3","massive_stocks.options.reference.conditions.list.v3.41a82e07","alphavantage.realtime_bulk_quotes.retrieve.v1","fiu_mcp_server.postv1stockquoteextend.create.v2.a30e86e7","hangseng_polysource.a_shares_live_quote.query.v2.10fe0581","mcp_gildata.usstockdailyquotes.v1","financialmodelingprep.cusip.search.v1.ad19189f","fiu_mcp_server.postv3stockquoteextend.create.v2.b35a7f02","massive_stocks.stocks.marketstatus.retrieve.v1.3045ae6e"]},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"returned_tool_ids":["twelvedata.quote.retrieve.v1.affbefe3"],"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"6bc11716-c8cd-4a51-b753-21e5c544fd76","result_valid":true},"finished_at":"2026-07-23T09:43:04.020Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"850f6ae9-de46-4e92-a722-81edd34dd191","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-23T09:43:04.020Z","status":"completed","discovery":{"result_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4","twelvedata.exchangerate.retrieve.v1.9eeb3b0d","coinmarketcap.tools.priceconversion.retrieve.v2.24e991d9","bybitp2p.payment.fx.convert.v5.7883b755","coingecko_pro.exchangerates.retrieve.v3.0f6133ea","bybitp2p.pay.fx.convert.v5.9221b0c4","coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","coinmarketcap.tools.priceconversion.retrieve.v2.52e4cb2d","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d","nbp_pl.exchangerates.rates.last.list.v1"]},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"returned_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4"],"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":100,"format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"47709f24-24dc-4a0e-a4d1-693fdd193e2f","result_valid":true},"finished_at":"2026-07-23T09:43:40.323Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"378c4fcc-1e86-4149-8b38-82d8230e5d11","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-23T09:43:40.323Z","status":"completed","discovery":{"result_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4","twelvedata.exchangerate.retrieve.v1.9eeb3b0d","coinmarketcap.tools.priceconversion.retrieve.v2.24e991d9","bybitp2p.payment.fx.convert.v5.7883b755","coingecko_pro.exchangerates.retrieve.v3.0f6133ea","bybitp2p.pay.fx.convert.v5.9221b0c4","coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","coinmarketcap.tools.priceconversion.retrieve.v2.52e4cb2d","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d","nbp_pl.exchangerates.rates.last.list.v1"]},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"returned_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4"],"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"8564bc40-9c95-4f99-9ef3-1388a755e461","result_valid":true},"finished_at":"2026-07-23T09:44:16.949Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"beb58cdc-281f-42cf-a388-9d9f30fa74be","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-23T09:44:16.949Z","status":"completed","discovery":{"result_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4","twelvedata.exchangerate.retrieve.v1.9eeb3b0d","coinmarketcap.tools.priceconversion.retrieve.v2.24e991d9","bybitp2p.payment.fx.convert.v5.7883b755","coingecko_pro.exchangerates.retrieve.v3.0f6133ea","bybitp2p.pay.fx.convert.v5.9221b0c4","coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","coinmarketcap.tools.priceconversion.retrieve.v2.52e4cb2d","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d","nbp_pl.exchangerates.rates.last.list.v1"]},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"returned_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4"],"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"ef1b196f-77fb-4f9b-b49f-504c5ff96748","result_valid":true},"finished_at":"2026-07-23T09:44:53.121Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"b602eea9-1fa5-4b43-adf2-59adff282a87","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"news-ai","trial":1,"started_at":"2026-07-23T09:44:53.121Z","status":"completed","discovery":{"result_tool_ids":["x_developer.2.news.search.retrieve.v1.19b3e678","newsdata.news.search.v1.b65ccc56","x_developer.2.tweets.search.recent.retrieve.v2.f424c8f9","qveris_finance.finance_news_aggregation_v1","jina_search.rerank.create.v2026.23dc486d","linkup.search.v1","newsapi.top_headlines.list.v2.38face94","newsapi.everything.retrieve.v2.e77ab1df","brave_search.news.search.list.v1.6ab52eaa","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic"]},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"returned_tool_ids":["newsapi.everything.retrieve.v2.e77ab1df"],"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20AI","from":"2026-07-16","to":"2026-07-23T23:59:59+08:00","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"execution_id":"f9669577-aab3-42e1-acbe-a311c92d0040","result_valid":true},"finished_at":"2026-07-23T09:45:25.145Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"bfc0ab6d-9a15-4768-a672-e466f0d23414","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"news-ai","trial":2,"started_at":"2026-07-23T09:45:25.145Z","status":"completed","discovery":{"result_tool_ids":["x_developer.2.news.search.retrieve.v1.19b3e678","newsdata.news.search.v1.b65ccc56","x_developer.2.tweets.search.recent.retrieve.v2.f424c8f9","qveris_finance.finance_news_aggregation_v1","jina_search.rerank.create.v2026.23dc486d","linkup.search.v1","newsapi.top_headlines.list.v2.38face94","newsapi.everything.retrieve.v2.e77ab1df","brave_search.news.search.list.v1.6ab52eaa","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic"]},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"returned_tool_ids":["newsapi.everything.retrieve.v2.e77ab1df"],"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20AI","from":"2026-07-16","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"execution_id":"60c01932-2aeb-4bf1-89d0-afb8ada449a8","result_valid":true},"finished_at":"2026-07-23T09:46:05.132Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"c2bab891-e16c-49ce-af35-9ba5220ef929","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"news-ai","trial":3,"started_at":"2026-07-23T09:46:05.132Z","status":"completed","discovery":{"result_tool_ids":["x_developer.2.news.search.retrieve.v1.19b3e678","newsdata.news.search.v1.b65ccc56","x_developer.2.tweets.search.recent.retrieve.v2.f424c8f9","qveris_finance.finance_news_aggregation_v1","jina_search.rerank.create.v2026.23dc486d","linkup.search.v1","newsapi.top_headlines.list.v2.38face94","newsapi.everything.retrieve.v2.e77ab1df","brave_search.news.search.list.v1.6ab52eaa","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic"]},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"returned_tool_ids":["newsapi.everything.retrieve.v2.e77ab1df"],"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20%22generative%20AI%22","from":"2026-07-20","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"execution_id":"ea44ee78-ac4d-42c2-957a-a9cfe5f5d904","result_valid":true},"finished_at":"2026-07-23T09:46:35.462Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"7c3b1d86-cdf3-42aa-94d0-bfa2024284ec","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T09:46:35.462Z","status":"completed","discovery":{"result_tool_ids":["semantic_scholar.paper.search.bulk.execute.v1.239dfd66","qveris_academic.paper_aggregation_v1","semantic_scholar.paper.search.bulk.retrieve.v1.096de627","semantic_scholar.paper.search.match.retrieve.v1.d3fd06f6","semantic_scholar.paper.search.match.retrieve.v1.b9f166f3","semantic_scholar.paper.autocomplete.retrieve.v1.e16c9798","semantic_scholar.paper.autocomplete.list.v1.99d870f6","xiaosu.scholar.search.v1.f79d2374","semantic_scholar.author.search.query.v1.1c428051","semantic_scholar.author.search.list.v1.ea3671fb"]},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"returned_tool_ids":["qveris_academic.paper_aggregation_v1"],"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"execution_id":"c78b5f36-0aa8-4732-945f-0c7ae03e33d0","result_valid":true},"finished_at":"2026-07-23T09:47:02.518Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"0dfd6d1c-ca90-4916-8b8f-4ae4d3e7f273","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T09:47:02.518Z","status":"completed","discovery":{"result_tool_ids":["semantic_scholar.paper.search.bulk.execute.v1.239dfd66","qveris_academic.paper_aggregation_v1","semantic_scholar.paper.search.bulk.retrieve.v1.096de627","semantic_scholar.paper.search.match.retrieve.v1.d3fd06f6","semantic_scholar.paper.search.match.retrieve.v1.b9f166f3","semantic_scholar.paper.autocomplete.retrieve.v1.e16c9798","semantic_scholar.paper.autocomplete.list.v1.99d870f6","xiaosu.scholar.search.v1.f79d2374","semantic_scholar.author.search.query.v1.1c428051","semantic_scholar.author.search.list.v1.ea3671fb"]},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"returned_tool_ids":["qveris_academic.paper_aggregation_v1"],"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"execution_id":"c93cbab8-c8de-4eb6-8352-6ad43723e538","result_valid":true},"finished_at":"2026-07-23T09:47:32.328Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ce6fbc71-c894-461a-adb1-4a607e7d4bdc","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T09:47:32.328Z","status":"completed","discovery":{"result_tool_ids":["semantic_scholar.paper.search.bulk.execute.v1.239dfd66","qveris_academic.paper_aggregation_v1","semantic_scholar.paper.search.bulk.retrieve.v1.096de627","semantic_scholar.paper.search.match.retrieve.v1.d3fd06f6","semantic_scholar.paper.search.match.retrieve.v1.b9f166f3","semantic_scholar.paper.autocomplete.retrieve.v1.e16c9798","semantic_scholar.paper.autocomplete.list.v1.99d870f6","xiaosu.scholar.search.v1.f79d2374","semantic_scholar.author.search.query.v1.1c428051","semantic_scholar.author.search.list.v1.ea3671fb"]},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"returned_tool_ids":["qveris_academic.paper_aggregation_v1"],"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"execution_id":"785d5fb3-a7ae-44ad-902b-a09ad5db2659","result_valid":true},"finished_at":"2026-07-23T09:47:58.422Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"b43508b4-6a54-487e-b3fd-dd511201af19","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T09:47:58.422Z","status":"completed","discovery":{"result_tool_ids":["api_sports.timezone.retrieve.v1.e993615d","api_sports.timezone.retrieve.v1.6792e6aa","api_sports.timezone.retrieve.v1.9b882d11","api_sports.timezone.retrieve.v1.00ab54ff","api_sports.timezone.retrieve.v1.ae6ae8d7","api_sports.timezone.retrieve.v1.64cfc345","api_sports.timezone.list.v1.bd878c58","api_sports.timezone.list.v1.fafec612","api_sports.timezone.list.v1.58366f46","api_sports.timezone.retrieve.v1.895e2257"]},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"returned_tool_ids":["api_sports.timezone.retrieve.v1.e993615d"],"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"execution_id":"499009d1-ebcc-470c-af61-581ca41f69ff","result_valid":true},"finished_at":"2026-07-23T09:48:24.635Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"704ee017-af2e-469b-a055-de7b367c3ffb","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T09:48:24.635Z","status":"completed","discovery":{"result_tool_ids":["api_sports.timezone.retrieve.v1.e993615d","api_sports.timezone.retrieve.v1.6792e6aa","api_sports.timezone.retrieve.v1.9b882d11","api_sports.timezone.retrieve.v1.00ab54ff","api_sports.timezone.retrieve.v1.ae6ae8d7","api_sports.timezone.retrieve.v1.64cfc345","api_sports.timezone.list.v1.bd878c58","api_sports.timezone.list.v1.fafec612","api_sports.timezone.list.v1.58366f46","api_sports.timezone.retrieve.v1.895e2257"]},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"returned_tool_ids":["api_sports.timezone.retrieve.v1.e993615d"],"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"execution_id":"527fbd2d-6eac-4400-8274-f30654f8f0f2","result_valid":true},"finished_at":"2026-07-23T09:48:55.300Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"495bda63-87f8-4ada-bcd2-a2a4076f6431","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T09:48:55.300Z","status":"completed","discovery":{"result_tool_ids":["api_sports.timezone.retrieve.v1.e993615d","api_sports.timezone.retrieve.v1.6792e6aa","api_sports.timezone.retrieve.v1.9b882d11","api_sports.timezone.retrieve.v1.00ab54ff","api_sports.timezone.retrieve.v1.ae6ae8d7","api_sports.timezone.retrieve.v1.64cfc345","api_sports.timezone.list.v1.bd878c58","api_sports.timezone.list.v1.fafec612","api_sports.timezone.list.v1.58366f46","api_sports.timezone.retrieve.v1.895e2257"]},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"returned_tool_ids":["api_sports.timezone.retrieve.v1.e993615d"],"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"execution_id":"18d67dfc-1fcd-417a-9892-44647cd21c74","result_valid":true},"finished_at":"2026-07-23T09:49:26.689Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"10a88d8d-1053-4d50-bab5-4b7114dd0ee7","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T09:49:26.689Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.geocoder.reverse.v2","baidu_map.geocoder.query.v2","baidu_map.geocoding.reverse_geocode.v1","baidu_map.reversegeocoding.create.v1","baidu_map.reverse_geocoding.query.v2","amap_webservice.geocode.geo.retrieve.v3","baidu_map.geocoder.wgeoreverse.create.v1","baidu_map.geocoder.wgeobatch.create.v1","arcgis_ocean.geocoding.reverse_geocode.v1.4d40d93e","baidu_map.geoconv.convert.v1"]},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"returned_tool_ids":["amap_webservice.geocode.geo.retrieve.v3"],"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京","output":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"ad2703a5-83ed-49b6-a202-1d4023478acd","result_valid":true},"finished_at":"2026-07-23T09:49:55.414Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"252d6b99-8605-4a66-aaec-d2b63a4fe36f","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T09:49:55.414Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.geocoder.reverse.v2","baidu_map.geocoder.query.v2","baidu_map.geocoding.reverse_geocode.v1","baidu_map.reversegeocoding.create.v1","baidu_map.reverse_geocoding.query.v2","amap_webservice.geocode.geo.retrieve.v3","baidu_map.geocoder.wgeoreverse.create.v1","baidu_map.geocoder.wgeobatch.create.v1","arcgis_ocean.geocoding.reverse_geocode.v1.4d40d93e","baidu_map.geoconv.convert.v1"]},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"returned_tool_ids":["amap_webservice.geocode.geo.retrieve.v3"],"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京","output":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"ce188a4a-8f86-4f1b-8152-d0a5a1e621b1","result_valid":true},"finished_at":"2026-07-23T09:50:31.198Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"3b2b6bf9-ad89-4298-92a7-3622fb115a13","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T09:50:31.198Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.geocoder.reverse.v2","baidu_map.geocoder.query.v2","baidu_map.geocoding.reverse_geocode.v1","baidu_map.reversegeocoding.create.v1","baidu_map.reverse_geocoding.query.v2","amap_webservice.geocode.geo.retrieve.v3","baidu_map.geocoder.wgeoreverse.create.v1","baidu_map.geocoder.wgeobatch.create.v1","arcgis_ocean.geocoding.reverse_geocode.v1.4d40d93e","baidu_map.geoconv.convert.v1"]},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"returned_tool_ids":["amap_webservice.geocode.geo.retrieve.v3"],"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京","output":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"27e3f46b-504e-4d9e-8ba5-a6413bb3abf4","result_valid":true},"finished_at":"2026-07-23T09:51:04.973Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"aaac2eb2-646e-42ff-ae32-1326ee9682b4","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-23T09:51:04.973Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.location.ip.retrieve.v1","decodo.isp_pay_per_ip.ip.retrieve.v1.bc2c6371","baidu_map.location.ip.execute.v1","amap_webservice.ip.retrieve.v3","virustotal.ip.info.retrieve.v3.ab34b0d1","weather_api.ip_lookup.retrieve.v1.e095d904","virustotal.ip.report.retrieve.v3.ab34b0d1","baidu_map.ip_location.retrieve.v1","weather_api.search.list.v1.73de2282","virustotal.ip-address.report.retrieve.v2.79606e33"]},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"returned_tool_ids":["weather_api.ip_lookup.retrieve.v1.e095d904"],"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"execution_id":"0dfbbf4f-3249-4340-9c3b-fdf6a09fb1f4","result_valid":false},"finished_at":"2026-07-23T09:51:37.056Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"3e947ba6-ca5a-47dc-abe3-404e9464239b","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-23T09:51:37.056Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.location.ip.retrieve.v1","decodo.isp_pay_per_ip.ip.retrieve.v1.bc2c6371","baidu_map.location.ip.execute.v1","amap_webservice.ip.retrieve.v3","virustotal.ip.info.retrieve.v3.ab34b0d1","weather_api.ip_lookup.retrieve.v1.e095d904","baidu_map.ip_location.retrieve.v1","virustotal.ip-address.report.retrieve.v2.79606e33","virustotal.ip_addresses.retrieve.v3.ab34b0d1","weather_api.search.list.v1.73de2282"]},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"returned_tool_ids":["weather_api.ip_lookup.retrieve.v1.e095d904"],"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"execution_id":"29664b6b-1928-4a74-9e22-7d3213840dd1","result_valid":false},"finished_at":"2026-07-23T09:52:06.521Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"69e303d3-d9ea-4f86-9fa3-0b4bd9c3087c","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-23T09:52:06.521Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.location.ip.retrieve.v1","decodo.isp_pay_per_ip.ip.retrieve.v1.bc2c6371","baidu_map.location.ip.execute.v1","amap_webservice.ip.retrieve.v3","virustotal.ip.info.retrieve.v3.ab34b0d1","weather_api.ip_lookup.retrieve.v1.e095d904","baidu_map.ip_location.retrieve.v1","virustotal.ip-address.report.retrieve.v2.79606e33","virustotal.ip_addresses.retrieve.v3.ab34b0d1","weather_api.search.list.v1.73de2282"]},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"returned_tool_ids":["weather_api.ip_lookup.retrieve.v1.e095d904"],"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"execution_id":"885aef06-36b7-4c20-af1d-57100415e97e","result_valid":false},"finished_at":"2026-07-23T09:52:38.206Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"16873a4e-2cd3-47d9-845e-0067aeb15a42","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"crypto-price","trial":1,"started_at":"2026-07-23T09:52:38.206Z","status":"completed","discovery":{"result_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","binance.ticker.price.list.v3.8675eca0","binance.avg_price.retrieve.v1.aef0de4f","coinmarketcap.cryptocurrency.listings.latest.retrieve.v1.8dddc12f","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.9dc24be4","etherscan.api.stats.ethprice.retrieve.v2.8eeca59e","binance.avgPrice.retrieve.v3.7b0852b4","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.251242d7","deribit.public.get_index_price.retrieve.v2.d47dcdcb","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d"]},"selection":{"tool_id":"coinmarketcap.tools.price-conversion.execute.v1.2b99a4df"},"inspection":{"returned_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df"],"required_parameters":["amount"]},"parameters":{"amount":1,"id":"1","convert":"USD"},"call":{"attempted":true,"success":true,"execution_id":"7f24abeb-87ee-4715-b069-c50153111792","result_valid":true},"finished_at":"2026-07-23T09:53:13.658Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"190b85e5-c226-468b-91d5-99a08a96768d","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"crypto-price","trial":2,"started_at":"2026-07-23T09:53:13.658Z","status":"completed","discovery":{"result_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","binance.ticker.price.list.v3.8675eca0","binance.avg_price.retrieve.v1.aef0de4f","coinmarketcap.cryptocurrency.listings.latest.retrieve.v1.8dddc12f","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.9dc24be4","etherscan.api.stats.ethprice.retrieve.v2.8eeca59e","binance.avgPrice.retrieve.v3.7b0852b4","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.251242d7","deribit.public.get_index_price.retrieve.v2.d47dcdcb","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d"]},"selection":{"tool_id":"coinmarketcap.tools.price-conversion.execute.v1.2b99a4df"},"inspection":{"returned_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df"],"required_parameters":["amount"]},"parameters":{"amount":1,"id":"1","convert":"USD"},"call":{"attempted":true,"success":true,"execution_id":"98f54c1a-21f0-490e-97d2-2e96781fae9a","result_valid":true},"finished_at":"2026-07-23T09:53:52.934Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"533afeba-6917-4bdb-83f5-6314d89c1207","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"crypto-price","trial":3,"started_at":"2026-07-23T09:53:52.934Z","status":"completed","discovery":{"result_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","binance.ticker.price.list.v3.8675eca0","binance.avg_price.retrieve.v1.aef0de4f","coinmarketcap.cryptocurrency.listings.latest.retrieve.v1.8dddc12f","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.9dc24be4","etherscan.api.stats.ethprice.retrieve.v2.8eeca59e","binance.avgPrice.retrieve.v3.7b0852b4","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.251242d7","deribit.public.get_index_price.retrieve.v2.d47dcdcb","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d"]},"selection":{"tool_id":"coinmarketcap.tools.price-conversion.execute.v1.2b99a4df"},"inspection":{"returned_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df"],"required_parameters":["amount"]},"parameters":{"amount":1,"id":"1","convert":"USD"},"call":{"attempted":true,"success":true,"execution_id":"04fb61ae-dccb-4ddb-a0ff-c8d0056f702a","result_valid":true},"finished_at":"2026-07-23T09:54:29.273Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"4e7bf26c-1907-441c-8103-49071ebcef84","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"web-search","trial":1,"started_at":"2026-07-23T09:54:29.273Z","status":"completed","discovery":{"result_tool_ids":["linkup.search.v1","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic","firecrawl.search.create.v2.a99b6f74","brave_search.images.search.list.v1","cloudsway.smart_search_overseas","serpapi.locations.locations.list.v1","x_developer.2.news.search.retrieve.v1.19b3e678","x_developer.2.spaces.search.retrieve.v1.43a1f31c","tavily.map.execute.v1","brave_search.web.search.list.v1.f02ed09d"]},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"returned_tool_ids":["cloudsway.smart_search_overseas"],"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"execution_id":"83b31afa-4453-4d1f-8c68-cc9caffa3277","result_valid":true},"finished_at":"2026-07-23T09:54:58.710Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"7c57619b-bf15-4504-8587-08a778c4c502","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"web-search","trial":2,"started_at":"2026-07-23T09:54:58.710Z","status":"completed","discovery":{"result_tool_ids":["linkup.search.v1","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic","firecrawl.search.create.v2.a99b6f74","brave_search.images.search.list.v1","cloudsway.smart_search_overseas","serpapi.locations.locations.list.v1","x_developer.2.news.search.retrieve.v1.19b3e678","x_developer.2.spaces.search.retrieve.v1.43a1f31c","tavily.map.execute.v1","brave_search.web.search.list.v1.f02ed09d"]},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"returned_tool_ids":["cloudsway.smart_search_overseas"],"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"execution_id":"1ecaa507-b75e-45ba-84da-475c507d2e2d","result_valid":true},"finished_at":"2026-07-23T09:55:30.042Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"b0c765a8-abef-4d88-b1d1-583d7a7083f5","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"web-search","trial":3,"started_at":"2026-07-23T09:55:30.042Z","status":"completed","discovery":{"result_tool_ids":["linkup.search.v1","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic","firecrawl.search.create.v2.a99b6f74","brave_search.images.search.list.v1","cloudsway.smart_search_overseas","serpapi.locations.locations.list.v1","x_developer.2.news.search.retrieve.v1.19b3e678","x_developer.2.spaces.search.retrieve.v1.43a1f31c","tavily.map.execute.v1","brave_search.web.search.list.v1.f02ed09d"]},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"returned_tool_ids":["cloudsway.smart_search_overseas"],"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"execution_id":"edc9ab7b-457d-4ed5-96a2-9179cf35a6a5","result_valid":true},"finished_at":"2026-07-23T09:56:01.974Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"3573e6ae-8c2b-4c45-9b9f-a7dbdaf459d8","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"air-quality","trial":1,"started_at":"2026-07-23T09:56:01.974Z","status":"completed","discovery":{"result_tool_ids":["openweathermap.air_pollution.current.v2_5","openweathermap.air_pollution.retrieve.v2.5","openweathermap.air_pollution.history.v2_5","openweathermap.air_pollution.forecast.retrieve.v2.5","openweathermap.air_pollution.history.retrieve.v2.5","epa_envirofacts.qaannualperformanceevaluations.bystate.retrieve.v1.61a2aa7d","epa_envirofacts.transactionsqaannualperformanceevaluations.bystate.retrieve.v1.4143b8bf","epa_envirofacts.transactionssample.bysite.retrieve.v1.93889e72","epa_envirofacts.annualdata.bybox.retrieve.v1.4b74a0d6","epa_envirofacts.dailydata.bycounty.retrieve.v1.f1ee8d1b"]},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"returned_tool_ids":["openweathermap.air_pollution.current.v2_5"],"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"execution_id":"2f2b2ad6-dd18-4f13-b8b4-b6729820ac07","result_valid":true},"finished_at":"2026-07-23T09:56:42.264Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"7835af93-e41c-428a-995f-4447571133fe","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"air-quality","trial":2,"started_at":"2026-07-23T09:56:42.264Z","status":"completed","discovery":{"result_tool_ids":["openweathermap.air_pollution.current.v2_5","openweathermap.air_pollution.retrieve.v2.5","openweathermap.air_pollution.history.v2_5","openweathermap.air_pollution.forecast.retrieve.v2.5","openweathermap.air_pollution.history.retrieve.v2.5","epa_envirofacts.qaannualperformanceevaluations.bystate.retrieve.v1.61a2aa7d","epa_envirofacts.transactionsqaannualperformanceevaluations.bystate.retrieve.v1.4143b8bf","epa_envirofacts.transactionssample.bysite.retrieve.v1.93889e72","epa_envirofacts.annualdata.bybox.retrieve.v1.4b74a0d6","epa_envirofacts.dailydata.bycounty.retrieve.v1.f1ee8d1b"]},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"returned_tool_ids":["openweathermap.air_pollution.current.v2_5"],"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"execution_id":"8cd90627-f7bc-4483-9bf9-729efe21d1a9","result_valid":true},"finished_at":"2026-07-23T09:57:10.056Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"73d62bf3-09c9-4c6f-af68-cbe62631e4c3","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"air-quality","trial":3,"started_at":"2026-07-23T09:57:10.056Z","status":"completed","discovery":{"result_tool_ids":["openweathermap.air_pollution.current.v2_5","openweathermap.air_pollution.retrieve.v2.5","openweathermap.air_pollution.history.v2_5","openweathermap.air_pollution.forecast.retrieve.v2.5","openweathermap.air_pollution.history.retrieve.v2.5","epa_envirofacts.qaannualperformanceevaluations.bystate.retrieve.v1.61a2aa7d","epa_envirofacts.transactionsqaannualperformanceevaluations.bystate.retrieve.v1.4143b8bf","epa_envirofacts.transactionssample.bysite.retrieve.v1.93889e72","epa_envirofacts.annualdata.bybox.retrieve.v1.4b74a0d6","epa_envirofacts.dailydata.bycounty.retrieve.v1.f1ee8d1b"]},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"returned_tool_ids":["openweathermap.air_pollution.current.v2_5"],"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"execution_id":"1532ee19-92f8-4ba5-995b-1a29d2abb90f","result_valid":true},"finished_at":"2026-07-23T09:57:36.396Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"0e125c12-7a7b-4c87-9e64-54a8be79f774","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-news","trial":1,"started_at":"2026-07-23T09:57:36.396Z","status":"completed","discovery":{"result_tool_ids":["finnhub.market.news.list.v1","finnhub.news.retrieve.v1.51b2567e","finnhub.news.retrieve.v1.2e72ea5d","finnhub.news.retrieve.v1","finnhub.news.execute.v1","finnhub.companynews.retrieve.v1","alphavantage.news_sentiment.query.v1.467a92c0","finnhub.companynews.retrieve.v1.4d11f47b","finnhub.companynews.retrieve.v1.e428b704","finnhub.companynews.execute.v1"]},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"returned_tool_ids":["finnhub.market.news.list.v1"],"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"execution_id":"c1e2d58e-6c82-446f-bfad-7421f700489e","result_valid":true},"finished_at":"2026-07-23T09:58:04.308Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"7a2ca54b-b346-4343-b6fb-54a264c5b778","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-news","trial":2,"started_at":"2026-07-23T09:58:04.308Z","status":"completed","discovery":{"result_tool_ids":["finnhub.market.news.list.v1","finnhub.news.retrieve.v1.51b2567e","finnhub.news.retrieve.v1.2e72ea5d","finnhub.news.retrieve.v1","finnhub.news.execute.v1","finnhub.companynews.retrieve.v1","alphavantage.news_sentiment.query.v1.467a92c0","finnhub.companynews.retrieve.v1.4d11f47b","finnhub.companynews.retrieve.v1.e428b704","finnhub.companynews.execute.v1"]},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"returned_tool_ids":["finnhub.market.news.list.v1"],"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"execution_id":"f8c8716d-b880-4871-9df6-fe9040f8ddd1","result_valid":true},"finished_at":"2026-07-23T09:58:29.894Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"e98dc7fd-0ed4-4487-a753-c6c197254603","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-news","trial":3,"started_at":"2026-07-23T09:58:29.894Z","status":"completed","discovery":{"result_tool_ids":["finnhub.market.news.list.v1","finnhub.news.retrieve.v1.51b2567e","finnhub.news.retrieve.v1.2e72ea5d","finnhub.news.retrieve.v1","finnhub.news.execute.v1","finnhub.companynews.retrieve.v1","alphavantage.news_sentiment.query.v1.467a92c0","finnhub.companynews.retrieve.v1.4d11f47b","finnhub.companynews.retrieve.v1.e428b704","finnhub.companynews.execute.v1"]},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"returned_tool_ids":["finnhub.market.news.list.v1"],"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"execution_id":"95bd80bb-dbd4-4137-873a-2d3307eabf77","result_valid":true},"finished_at":"2026-07-23T09:58:58.074Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"a22d10aa-0f23-4f76-b2a2-692576b4b2a2","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T09:58:58.074Z","status":"failed","discovery":{"result_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8","dataforseo.dataforseo_labs.google.domain_whois_overview.live.execute.v3","dataforseo.databases.domains.query.v3.c1720ec0","virustotal.domains.report.retrieve.v3.c5459740","bainiu_api.common.company_domain.query.v1.e978ca32","virustotal.domain_reports.retrieve.v3.c5459740","dataforseo.domain_analytics.technologies.domains_by_technology.live.create.v3.bfc3dc2d","virustotal.domains.retrieve.v3.c5459740","virustotal.domains.analyse.v3.c755c69e","dataforseo.domain_analytics.technologies.domains_by_html_terms.live.create.v3.d3760ae1"]},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"returned_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"],"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected","message":"Adapter invocation failed"},"finished_at":"2026-07-23T09:59:40.997Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"97361151-f794-478a-8b90-e8e544c484f7","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T09:59:40.997Z","status":"completed","discovery":{"result_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8","dataforseo.dataforseo_labs.google.domain_whois_overview.live.execute.v3","dataforseo.databases.domains.query.v3.c1720ec0","virustotal.domains.report.retrieve.v3.c5459740","bainiu_api.common.company_domain.query.v1.e978ca32","virustotal.domain_reports.retrieve.v3.c5459740","dataforseo.domain_analytics.technologies.domains_by_technology.live.create.v3.bfc3dc2d","virustotal.domains.retrieve.v3.c5459740","virustotal.domains.analyse.v3.c755c69e","dataforseo.domain_analytics.technologies.domains_by_html_terms.live.create.v3.d3760ae1"]},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"returned_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"],"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"execution_id":"d3182215-e0d8-4571-b0c8-e7192d34d59a","result_valid":true},"finished_at":"2026-07-23T10:00:15.027Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"e877975e-2fff-4ec9-a487-0bddba738e4f","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T10:00:15.027Z","status":"failed","discovery":{"result_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8","dataforseo.dataforseo_labs.google.domain_whois_overview.live.execute.v3","dataforseo.databases.domains.query.v3.c1720ec0","virustotal.domains.report.retrieve.v3.c5459740","bainiu_api.common.company_domain.query.v1.e978ca32","virustotal.domain_reports.retrieve.v3.c5459740","dataforseo.domain_analytics.technologies.domains_by_technology.live.create.v3.bfc3dc2d","virustotal.domains.retrieve.v3.c5459740","virustotal.domains.analyse.v3.c755c69e","dataforseo.domain_analytics.technologies.domains_by_html_terms.live.create.v3.d3760ae1"]},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"returned_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"],"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected","message":"Adapter invocation failed"},"finished_at":"2026-07-23T10:00:55.710Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"d9fe796d-cd3d-4d0c-8bb3-0cd1fc7e128d","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"image-search","trial":1,"started_at":"2026-07-23T10:00:55.710Z","status":"completed","discovery":{"result_tool_ids":["brave_search.images.search.list.v1","shutterstock.images.search.list.v2.69059569","dataforseo.serp.google.search_by_image.task.create.v3","shutterstock.images.search.list.v2.c2b44682","jina_search.embeddings.create.v2026.5c7cb661","jina_search.rerank.create.v2026.23dc486d","linkup.search.v1","shutterstock.images.search.list.v2.15f15845","shutterstock.images.search.suggestions.list.v2.f6850c10","shutterstock.images.search.suggestions.list.v2"]},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"returned_tool_ids":["brave_search.images.search.list.v1"],"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"execution_id":"fa985665-5752-4080-9878-40d7480d003e","result_valid":true},"finished_at":"2026-07-23T10:01:21.601Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"e3d0d1fd-e32c-4701-b87a-7d2907aae773","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"image-search","trial":2,"started_at":"2026-07-23T10:01:21.601Z","status":"completed","discovery":{"result_tool_ids":["brave_search.images.search.list.v1","shutterstock.images.search.list.v2.69059569","dataforseo.serp.google.search_by_image.task.create.v3","shutterstock.images.search.list.v2.c2b44682","jina_search.embeddings.create.v2026.5c7cb661","jina_search.rerank.create.v2026.23dc486d","linkup.search.v1","shutterstock.images.search.list.v2.15f15845","shutterstock.images.search.suggestions.list.v2.f6850c10","shutterstock.images.search.suggestions.list.v2"]},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"returned_tool_ids":["brave_search.images.search.list.v1"],"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"execution_id":"89c4a370-2374-4ecc-8132-095b9ba65e48","result_valid":true},"finished_at":"2026-07-23T10:01:46.789Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"109d0ded-433f-4ef6-a7ed-f8cf19b897be","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"image-search","trial":3,"started_at":"2026-07-23T10:01:46.789Z","status":"completed","discovery":{"result_tool_ids":["brave_search.images.search.list.v1","shutterstock.images.search.list.v2.69059569","dataforseo.serp.google.search_by_image.task.create.v3","shutterstock.images.search.list.v2.c2b44682","jina_search.rerank.create.v2026.23dc486d","jina_search.embeddings.create.v2026.5c7cb661","linkup.search.v1","shutterstock.images.search.list.v2.15f15845","shutterstock.images.search.suggestions.list.v2.f6850c10","shutterstock.images.search.suggestions.list.v2"]},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"returned_tool_ids":["brave_search.images.search.list.v1"],"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"execution_id":"1a25b138-27e5-45df-a6a9-683259e805c3","result_valid":true},"finished_at":"2026-07-23T10:02:17.125Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"e55b1799-5e11-481c-a18f-e4be01b4e744","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-holidays","trial":1,"started_at":"2026-07-23T10:02:17.125Z","status":"completed","discovery":{"result_tool_ids":["finnhub.stock.marketholiday.retrieve.v1.75d6c1f9","finnhub.stock.marketholiday.execute.v1","eodhd.exchange_details.retrieve.v1.7da32390","finnhub.stock.marketholiday.retrieve.v1","finnhub.stock.marketholiday.retrieve.v1.5c5bfe9a","massive_stocks.stocks.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.retrieve.v1","massive_stocks.options.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.list.v1","massive_stocks.options.marketstatus.retrieve.v1.3045ae6e"]},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"returned_tool_ids":["finnhub.stock.marketholiday.retrieve.v1"],"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"execution_id":"c1b4c819-ac15-449f-8408-3944918ff86b","result_valid":true},"finished_at":"2026-07-23T10:02:44.222Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"a88a3226-9842-4032-ac67-fcae35d59a03","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-holidays","trial":2,"started_at":"2026-07-23T10:02:44.222Z","status":"completed","discovery":{"result_tool_ids":["finnhub.stock.marketholiday.retrieve.v1.75d6c1f9","finnhub.stock.marketholiday.execute.v1","eodhd.exchange_details.retrieve.v1.7da32390","finnhub.stock.marketholiday.retrieve.v1","finnhub.stock.marketholiday.retrieve.v1.5c5bfe9a","massive_stocks.stocks.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.retrieve.v1","massive_stocks.options.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.list.v1","massive_stocks.options.marketstatus.retrieve.v1.3045ae6e"]},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"returned_tool_ids":["finnhub.stock.marketholiday.retrieve.v1"],"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"execution_id":"c53b47ab-dab7-4f70-a114-e1e32ff91bbf","result_valid":true},"finished_at":"2026-07-23T10:03:10.533Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"bd6bfb47-687b-4d08-9a70-f019316e623a","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-holidays","trial":3,"started_at":"2026-07-23T10:03:10.533Z","status":"completed","discovery":{"result_tool_ids":["finnhub.stock.marketholiday.retrieve.v1.75d6c1f9","finnhub.stock.marketholiday.execute.v1","eodhd.exchange_details.retrieve.v1.7da32390","finnhub.stock.marketholiday.retrieve.v1","finnhub.stock.marketholiday.retrieve.v1.5c5bfe9a","massive_stocks.stocks.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.retrieve.v1","massive_stocks.options.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.list.v1","massive_stocks.options.marketstatus.retrieve.v1.3045ae6e"]},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"returned_tool_ids":["finnhub.stock.marketholiday.retrieve.v1"],"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"execution_id":"ee0ee3a7-f306-42ce-ac38-4b9d2471d2da","result_valid":true},"finished_at":"2026-07-23T10:03:37.293Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"c28bc1c8-94d0-4bed-a562-af06253fc011","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-23T10:03:37.293Z","status":"completed","discovery":{"result_tool_ids":["usgs_earthquake_catalog.event.events.list.v1","usgs_earthquake_catalog.fdsnws.event.query.retrieve.v1.025549c9","usgs_earthquake_catalog.earthquakes.query.v1","usgs_earthquake_catalog.fdsnws.event.query.v1.025549c9","usgs_earthquake_catalog.events.query.v1.025549c9","usgs_earthquake.catalogs.list.v1","usgs_earthquake.contributors.list.v1","usgs-earthquake-event-api.event.query.list.v1","usgs_earthquake.parameters.list.v1","usgs_earthquake.version.retrieve.v1"]},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"returned_tool_ids":["usgs_earthquake_catalog.event.events.list.v1"],"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"execution_id":"20baa4c2-30c0-4c54-873d-29b623d63766","result_valid":true},"finished_at":"2026-07-23T10:04:19.806Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"9eff8a69-cb9a-4931-ae30-4230a29c6b2c","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-23T10:04:19.806Z","status":"completed","discovery":{"result_tool_ids":["usgs_earthquake_catalog.event.events.list.v1","usgs_earthquake_catalog.fdsnws.event.query.retrieve.v1.025549c9","usgs_earthquake_catalog.earthquakes.query.v1","usgs_earthquake_catalog.fdsnws.event.query.v1.025549c9","usgs_earthquake_catalog.events.query.v1.025549c9","usgs_earthquake.catalogs.list.v1","usgs_earthquake.contributors.list.v1","usgs-earthquake-event-api.event.query.list.v1","usgs_earthquake.parameters.list.v1","usgs_earthquake.version.retrieve.v1"]},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"returned_tool_ids":["usgs_earthquake_catalog.event.events.list.v1"],"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"execution_id":"1953bab8-8a42-4e52-acf5-4043deadc1d3","result_valid":true},"finished_at":"2026-07-23T10:04:54.838Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"81d9ac77-2d8f-4363-be1a-09193951a1b1","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T10:04:54.838Z","status":"completed","discovery":{"result_tool_ids":["usgs_earthquake_catalog.event.events.list.v1","usgs_earthquake_catalog.fdsnws.event.query.retrieve.v1.025549c9","usgs_earthquake_catalog.earthquakes.query.v1","usgs_earthquake_catalog.fdsnws.event.query.v1.025549c9","usgs_earthquake_catalog.events.query.v1.025549c9","usgs_earthquake.catalogs.list.v1","usgs_earthquake.contributors.list.v1","usgs-earthquake-event-api.event.query.list.v1","usgs_earthquake.parameters.list.v1","usgs_earthquake.version.retrieve.v1"]},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"returned_tool_ids":["usgs_earthquake_catalog.event.events.list.v1"],"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"execution_id":"29f479b7-a1f9-45fa-8539-fb5ebbd208dc","result_valid":true},"finished_at":"2026-07-23T10:05:33.311Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"abc19032-2dc3-46fc-a861-05132c7eda65","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"company-profile","trial":1,"started_at":"2026-07-23T10:05:33.311Z","status":"completed","discovery":{"result_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2","bainiu_api.company.detail.execute.v1.f6b528e7","bainiu_api.openapi.company.alter.execute.v1.8fa090d1","tradefeeds.compinfo.retrieve.v1.c2314765","financialmodelingprep.stable.companynotes.retrieve.v1.930aa468","financialmodelingprep.stable.secfilingscompanysearch.symbol.retrieve.v1.5cf7397d","bainiu_api.openapi.common.company_info_deep_search.execute.v1.8e5f009d","financialmodelingprep.search.cusip.retrieve.v1.51d0c621","bainiu_api.openapi.common.company_lianxi_public.execute.v1.bd6261e8","financialmodelingprep.stable.secfilingscompanysearch.cik.retrieve.v1.a1087fe5"]},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"returned_tool_ids":["tradefeeds.compinfo.retrieve.v1.c2314765"],"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"execution_id":"c2cc2904-3341-4f5a-b3b5-33471d8bab04","result_valid":false},"finished_at":"2026-07-23T10:06:00.664Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"66977b1e-3302-469e-9e30-590d4b8cd542","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"company-profile","trial":2,"started_at":"2026-07-23T10:06:00.664Z","status":"completed","discovery":{"result_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2","bainiu_api.company.detail.execute.v1.f6b528e7","bainiu_api.openapi.company.alter.execute.v1.8fa090d1","tradefeeds.compinfo.retrieve.v1.c2314765","financialmodelingprep.stable.companynotes.retrieve.v1.930aa468","financialmodelingprep.stable.secfilingscompanysearch.symbol.retrieve.v1.5cf7397d","bainiu_api.openapi.common.company_info_deep_search.execute.v1.8e5f009d","financialmodelingprep.search.cusip.retrieve.v1.51d0c621","bainiu_api.openapi.common.company_lianxi_public.execute.v1.bd6261e8","financialmodelingprep.stable.secfilingscompanysearch.cik.retrieve.v1.a1087fe5"]},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"returned_tool_ids":["tradefeeds.compinfo.retrieve.v1.c2314765"],"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"execution_id":"9162f0c8-7d8e-41a5-bc0f-3c8670164b72","result_valid":false},"finished_at":"2026-07-23T10:06:50.608Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"b3001ec6-f8da-4dcc-b420-a40d0991337b","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"company-profile","trial":3,"started_at":"2026-07-23T10:06:50.608Z","status":"completed","discovery":{"result_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2","bainiu_api.company.detail.execute.v1.f6b528e7","bainiu_api.openapi.company.alter.execute.v1.8fa090d1","tradefeeds.compinfo.retrieve.v1.c2314765","financialmodelingprep.stable.companynotes.retrieve.v1.930aa468","financialmodelingprep.stable.secfilingscompanysearch.symbol.retrieve.v1.5cf7397d","bainiu_api.openapi.common.company_info_deep_search.execute.v1.8e5f009d","financialmodelingprep.search.cusip.retrieve.v1.51d0c621","bainiu_api.openapi.common.company_lianxi_public.execute.v1.bd6261e8","financialmodelingprep.stable.secfilingscompanysearch.cik.retrieve.v1.a1087fe5"]},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"returned_tool_ids":["tradefeeds.compinfo.retrieve.v1.c2314765"],"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"execution_id":"85544350-ce6a-4431-9e82-c4e5665bf8ab","result_valid":false},"finished_at":"2026-07-23T10:07:33.561Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"7e26fd5b-b2f6-40bb-90dc-0d32665ffadd","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"sports-standings","trial":1,"started_at":"2026-07-23T10:07:33.561Z","status":"completed","discovery":{"result_tool_ids":["api_sports.standings.list.v1.98a153bd","api_sports.standings.list.v1.39568ba4","api_sports.standings.groups.list.v1.a077a680","api_sports.standings.groups.list.v1.0d8a40cf","api_sports.standings.stages.list.v1.5ba13a89","api_sports.standings.stages.list.v1.838f769a","api_sports.standings.retrieve.v1.ff019893","api_sports.standings.groups.list.v1.bed67ce0","api_sports.standings.groups.list.v1.57393fb3","api_sports.standings.stages.list.v1.607034e1"]},"selection":{"tool_id":"api_sports.standings.list.v1.98a153bd"},"inspection":{"returned_tool_ids":["api_sports.standings.list.v1.98a153bd"],"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"execution_id":"de1481bd-672a-4d93-a07b-1896bcd48cc0","result_valid":true},"finished_at":"2026-07-23T10:08:06.053Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"936fb637-9c45-40c3-bfe5-33fb0bf5bc9a","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"sports-standings","trial":2,"started_at":"2026-07-23T10:08:06.053Z","status":"completed","discovery":{"result_tool_ids":["api_sports.standings.list.v1.98a153bd","api_sports.standings.list.v1.39568ba4","api_sports.standings.groups.list.v1.a077a680","api_sports.standings.groups.list.v1.0d8a40cf","api_sports.standings.stages.list.v1.5ba13a89","api_sports.standings.stages.list.v1.838f769a","api_sports.standings.retrieve.v1.ff019893","api_sports.standings.groups.list.v1.bed67ce0","api_sports.standings.groups.list.v1.57393fb3","api_sports.standings.stages.list.v1.607034e1"]},"selection":{"tool_id":"api_sports.standings.list.v1.98a153bd"},"inspection":{"returned_tool_ids":["api_sports.standings.list.v1.98a153bd"],"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"execution_id":"5610b885-0407-4350-b26e-fc772427fc32","result_valid":true},"finished_at":"2026-07-23T10:08:47.521Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"85b215db-7bde-4512-aad4-781a905c7c38","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"sports-standings","trial":3,"started_at":"2026-07-23T10:08:47.521Z","status":"completed","discovery":{"result_tool_ids":["api_sports.standings.list.v1.98a153bd","api_sports.standings.list.v1.39568ba4","api_sports.standings.groups.list.v1.a077a680","api_sports.standings.groups.list.v1.0d8a40cf","api_sports.standings.stages.list.v1.5ba13a89","api_sports.standings.stages.list.v1.838f769a","api_sports.standings.retrieve.v1.ff019893","api_sports.standings.groups.list.v1.bed67ce0","api_sports.standings.groups.list.v1.57393fb3","api_sports.standings.stages.list.v1.607034e1"]},"selection":{"tool_id":"api_sports.standings.list.v1.98a153bd"},"inspection":{"returned_tool_ids":["api_sports.standings.list.v1.98a153bd"],"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"execution_id":"1cd42924-35d4-40e0-930a-d7d56a26587c","result_valid":true},"finished_at":"2026-07-23T10:09:22.117Z"} diff --git a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.summary.json b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.summary.json new file mode 100644 index 0000000..32a309d --- /dev/null +++ b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.summary.json @@ -0,0 +1,1439 @@ +{ + "schema_version": 1, + "methodology": "discover-call-v1", + "generated_at": "2026-07-23T10:09:22.150Z", + "models": [ + { + "model": "gpt-5.6-sol", + "lane": "pinned-model", + "tasks": 18, + "trials_per_task": 3, + "runs": 54, + "completed": 52, + "executed": 52, + "selection_grounded_rate": 1, + "inspection_grounded_rate": 1, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0.8333, + "call_success_rate": 0.8846, + "result_valid_rate": 0.8846, + "workflow_success_rate": 0.7222, + "workflow_success_wilson_95": [ + 0.5911, + 0.8238 + ], + "failures_by_stage": { + "adapter": 2 + }, + "failures_by_reason": { + "tool_use_rejected": 2 + } + } + ], + "records": [ + { + "run_id": "23187609-52d7-44b6-b60e-91f68d32777c", + "task_id": "weather-london", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "d2bfdd97-1ac6-4388-aab6-dc04bd389b4d", + "task_id": "weather-london", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "9dc77ec7-6fea-42e6-8786-f8970a33da5d", + "task_id": "weather-london", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "98d4488a-8f0e-48fe-8743-ed08df8940f8", + "task_id": "stock-aapl", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "450cf5b6-8404-4be2-b73e-c28a6ae3227d", + "task_id": "stock-aapl", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "456b1477-ec0a-4772-887c-8971ac437a95", + "task_id": "stock-aapl", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "850f6ae9-de46-4e92-a722-81edd34dd191", + "task_id": "exchange-usd-eur", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "378c4fcc-1e86-4149-8b38-82d8230e5d11", + "task_id": "exchange-usd-eur", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "beb58cdc-281f-42cf-a388-9d9f30fa74be", + "task_id": "exchange-usd-eur", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "b602eea9-1fa5-4b43-adf2-59adff282a87", + "task_id": "news-ai", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "bfc0ab6d-9a15-4768-a672-e466f0d23414", + "task_id": "news-ai", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "c2bab891-e16c-49ce-af35-9ba5220ef929", + "task_id": "news-ai", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "7c3b1d86-cdf3-42aa-94d0-bfa2024284ec", + "task_id": "papers-climate", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "0dfd6d1c-ca90-4916-8b8f-4ae4d3e7f273", + "task_id": "papers-climate", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "ce6fbc71-c894-461a-adb1-4a607e7d4bdc", + "task_id": "papers-climate", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "b43508b4-6a54-487e-b3fd-dd511201af19", + "task_id": "timezone-tokyo", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "result_valid": true, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "704ee017-af2e-469b-a055-de7b367c3ffb", + "task_id": "timezone-tokyo", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "result_valid": true, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "495bda63-87f8-4ada-bcd2-a2a4076f6431", + "task_id": "timezone-tokyo", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "result_valid": true, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "10a88d8d-1053-4d50-bab5-4b7114dd0ee7", + "task_id": "geocode-address", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "252d6b99-8605-4a66-aaec-d2b63a4fe36f", + "task_id": "geocode-address", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "3b2b6bf9-ad89-4298-92a7-3622fb115a13", + "task_id": "geocode-address", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "aaac2eb2-646e-42ff-ae32-1326ee9682b4", + "task_id": "ip-geolocation", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": false, + "result_valid": false, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "3e947ba6-ca5a-47dc-abe3-404e9464239b", + "task_id": "ip-geolocation", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": false, + "result_valid": false, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "69e303d3-d9ea-4f86-9fa3-0b4bd9c3087c", + "task_id": "ip-geolocation", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": false, + "result_valid": false, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "16873a4e-2cd3-47d9-845e-0067aeb15a42", + "task_id": "crypto-price", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "result_valid": true, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "190b85e5-c226-468b-91d5-99a08a96768d", + "task_id": "crypto-price", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "result_valid": true, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "533afeba-6917-4bdb-83f5-6314d89c1207", + "task_id": "crypto-price", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "result_valid": true, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "4e7bf26c-1907-441c-8103-49071ebcef84", + "task_id": "web-search", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "7c57619b-bf15-4504-8587-08a778c4c502", + "task_id": "web-search", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "b0c765a8-abef-4d88-b1d1-583d7a7083f5", + "task_id": "web-search", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "3573e6ae-8c2b-4c45-9b9f-a7dbdaf459d8", + "task_id": "air-quality", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "7835af93-e41c-428a-995f-4447571133fe", + "task_id": "air-quality", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "73d62bf3-09c9-4c6f-af68-cbe62631e4c3", + "task_id": "air-quality", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "0e125c12-7a7b-4c87-9e64-54a8be79f774", + "task_id": "market-news", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "7a2ca54b-b346-4343-b6fb-54a264c5b778", + "task_id": "market-news", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "e98dc7fd-0ed4-4487-a753-c6c197254603", + "task_id": "market-news", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "a22d10aa-0f23-4f76-b2a2-692576b4b2a2", + "task_id": "domain-intel", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": false, + "executed": false, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": null, + "result_valid": null, + "workflow_success": false, + "error_stage": "adapter", + "error_reason": "tool_use_rejected", + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "97361151-f794-478a-8b90-e8e544c484f7", + "task_id": "domain-intel", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "result_valid": true, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "e877975e-2fff-4ec9-a487-0bddba738e4f", + "task_id": "domain-intel", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": false, + "executed": false, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": null, + "result_valid": null, + "workflow_success": false, + "error_stage": "adapter", + "error_reason": "tool_use_rejected", + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "d9fe796d-cd3d-4d0c-8bb3-0cd1fc7e128d", + "task_id": "image-search", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "e3d0d1fd-e32c-4701-b87a-7d2907aae773", + "task_id": "image-search", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "109d0ded-433f-4ef6-a7ed-f8cf19b897be", + "task_id": "image-search", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "e55b1799-5e11-481c-a18f-e4be01b4e744", + "task_id": "market-holidays", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "a88a3226-9842-4032-ac67-fcae35d59a03", + "task_id": "market-holidays", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "bd6bfb47-687b-4d08-9a70-f019316e623a", + "task_id": "market-holidays", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "c28bc1c8-94d0-4bed-a562-af06253fc011", + "task_id": "earthquake-catalog", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "9eff8a69-cb9a-4931-ae30-4230a29c6b2c", + "task_id": "earthquake-catalog", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "81d9ac77-2d8f-4363-be1a-09193951a1b1", + "task_id": "earthquake-catalog", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "abc19032-2dc3-46fc-a861-05132c7eda65", + "task_id": "company-profile", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": false, + "result_valid": false, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "66977b1e-3302-469e-9e30-590d4b8cd542", + "task_id": "company-profile", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": false, + "result_valid": false, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "b3001ec6-f8da-4dcc-b420-a40d0991337b", + "task_id": "company-profile", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": false, + "result_valid": false, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "7e26fd5b-b2f6-40bb-90dc-0d32665ffadd", + "task_id": "sports-standings", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "936fb637-9c45-40c3-bfe5-33fb0bf5bc9a", + "task_id": "sports-standings", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "85b215db-7bde-4512-aad4-781a905c7c38", + "task_id": "sports-standings", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "pinned-model", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + } + ] +} diff --git a/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.runs.jsonl b/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.runs.jsonl new file mode 100644 index 0000000..05ce48b --- /dev/null +++ b/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.runs.jsonl @@ -0,0 +1,54 @@ +{"schema_version":1,"benchmark_version":"v1","run_id":"8ba500d0-14cb-46a7-a347-c9ba9be3a3a8","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"weather-london","trial":1,"started_at":"2026-07-23T07:52:46.344Z","status":"completed","discovery":{"result_tool_ids":["weather_gov.gridpoints.forecast.retrieve.v1.40a9a3ce","weather_gov.gridpoints.forecast.retrieve.v3.29bbec80","weather_gov.gridpoints.forecast.hourly.retrieve.v3.808ea240","amap_webservice.weather.weatherinfo.retrieve.v3","weather_gov.icons.retrieve.v3.ad0b4d80","visualcrossing.timeline.retrieve.v1","weather_gov.products.retrieve.v3.43ee57b8","weather_gov.products.retrieve.v1.18657145","weather_gov.gridpoints.retrieve.v1.5bc9e3c1","weather_gov.gridpoints.retrieve.v3.a0c89365"]},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"returned_tool_ids":["visualcrossing.timeline.retrieve.v1"],"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"execution_id":"f3dfb93b-4c10-43b3-983e-124d7ab9942c","result_valid":true},"finished_at":"2026-07-23T07:52:51.394Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"1f240f6b-6c51-4970-94c8-86a99dd6a36e","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"weather-london","trial":2,"started_at":"2026-07-23T07:52:51.394Z","status":"completed","discovery":{"result_tool_ids":["weather_gov.gridpoints.forecast.retrieve.v1.40a9a3ce","weather_gov.gridpoints.forecast.retrieve.v3.29bbec80","weather_gov.gridpoints.forecast.hourly.retrieve.v3.808ea240","amap_webservice.weather.weatherinfo.retrieve.v3","weather_gov.icons.retrieve.v3.ad0b4d80","visualcrossing.timeline.retrieve.v1","weather_gov.products.retrieve.v3.43ee57b8","weather_gov.products.retrieve.v1.18657145","weather_gov.gridpoints.retrieve.v1.5bc9e3c1","weather_gov.gridpoints.retrieve.v3.a0c89365"]},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"returned_tool_ids":["visualcrossing.timeline.retrieve.v1"],"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"execution_id":"853f33f6-5dec-4427-97e5-9edb5ed803c4","result_valid":true},"finished_at":"2026-07-23T07:52:55.748Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"891080de-9c5f-45d3-897f-94db7557d1c1","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"weather-london","trial":3,"started_at":"2026-07-23T07:52:55.748Z","status":"completed","discovery":{"result_tool_ids":["weather_gov.gridpoints.forecast.retrieve.v1.40a9a3ce","weather_gov.gridpoints.forecast.retrieve.v3.29bbec80","weather_gov.gridpoints.forecast.hourly.retrieve.v3.808ea240","amap_webservice.weather.weatherinfo.retrieve.v3","weather_gov.icons.retrieve.v3.ad0b4d80","visualcrossing.timeline.retrieve.v1","weather_gov.products.retrieve.v3.43ee57b8","weather_gov.products.retrieve.v1.18657145","weather_gov.gridpoints.retrieve.v1.5bc9e3c1","weather_gov.gridpoints.retrieve.v3.a0c89365"]},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"returned_tool_ids":["visualcrossing.timeline.retrieve.v1"],"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"execution_id":"265e970c-9989-4a41-bbd1-e39afe60000e","result_valid":true},"finished_at":"2026-07-23T07:52:59.070Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"f32129b5-3004-4733-90c2-6137c3a74d0f","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-23T07:52:59.070Z","status":"completed","discovery":{"result_tool_ids":["eodhd.live_v2.us_quote_delayed.retrieve.v1.f0e13d45","twelvedata.quote.retrieve.v1.affbefe3","massive_stocks.options.reference.conditions.list.v3.41a82e07","alphavantage.realtime_bulk_quotes.retrieve.v1","fiu_mcp_server.postv1stockquoteextend.create.v2.a30e86e7","hangseng_polysource.a_shares_live_quote.query.v2.10fe0581","mcp_gildata.usstockdailyquotes.v1","massive_stocks.stocks.marketstatus.retrieve.v1.3045ae6e","fiu_mcp_server.postv3stockquoteextend.create.v2.b35a7f02","financialmodelingprep.cusip.search.v1.ad19189f"]},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"returned_tool_ids":["twelvedata.quote.retrieve.v1.affbefe3"],"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"00eacb0a-ca6d-4379-b706-4b8ad85dd2e6","result_valid":true},"finished_at":"2026-07-23T07:53:02.034Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"28f35db3-bab5-4b00-9aa3-e29371acf680","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-23T07:53:02.035Z","status":"completed","discovery":{"result_tool_ids":["eodhd.live_v2.us_quote_delayed.retrieve.v1.f0e13d45","twelvedata.quote.retrieve.v1.affbefe3","massive_stocks.options.reference.conditions.list.v3.41a82e07","alphavantage.realtime_bulk_quotes.retrieve.v1","fiu_mcp_server.postv1stockquoteextend.create.v2.a30e86e7","hangseng_polysource.a_shares_live_quote.query.v2.10fe0581","mcp_gildata.usstockdailyquotes.v1","massive_stocks.stocks.marketstatus.retrieve.v1.3045ae6e","fiu_mcp_server.postv3stockquoteextend.create.v2.b35a7f02","financialmodelingprep.cusip.search.v1.ad19189f"]},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"returned_tool_ids":["twelvedata.quote.retrieve.v1.affbefe3"],"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"42dbe5fc-0e7b-4358-b908-36183fabc92b","result_valid":true},"finished_at":"2026-07-23T07:53:05.383Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"770c7467-c2b6-4090-8826-0182ccfae9da","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-23T07:53:05.383Z","status":"completed","discovery":{"result_tool_ids":["eodhd.live_v2.us_quote_delayed.retrieve.v1.f0e13d45","twelvedata.quote.retrieve.v1.affbefe3","massive_stocks.options.reference.conditions.list.v3.41a82e07","alphavantage.realtime_bulk_quotes.retrieve.v1","fiu_mcp_server.postv1stockquoteextend.create.v2.a30e86e7","hangseng_polysource.a_shares_live_quote.query.v2.10fe0581","mcp_gildata.usstockdailyquotes.v1","massive_stocks.stocks.marketstatus.retrieve.v1.3045ae6e","fiu_mcp_server.postv3stockquoteextend.create.v2.b35a7f02","financialmodelingprep.cusip.search.v1.ad19189f"]},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"returned_tool_ids":["twelvedata.quote.retrieve.v1.affbefe3"],"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"d8a6652f-f68e-49a6-b36f-e3acfd0409f9","result_valid":true},"finished_at":"2026-07-23T07:53:08.429Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"c4624c8d-fb7b-4d9e-9248-209a6679336b","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-23T07:53:08.429Z","status":"completed","discovery":{"result_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4","twelvedata.exchangerate.retrieve.v1.9eeb3b0d","coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","coinmarketcap.tools.priceconversion.retrieve.v2.24e991d9","bybitp2p.pay.fx.convert.v5.9221b0c4","coingecko_pro.exchangerates.retrieve.v3.0f6133ea","bybitp2p.payment.fx.convert.v5.7883b755","nbp_pl.exchangerates.rates.last.list.v1","coinmarketcap.tools.priceconversion.retrieve.v2.52e4cb2d","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d"]},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"returned_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4"],"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"835c61d6-148f-4a93-91d7-cdd862feb1e4","result_valid":true},"finished_at":"2026-07-23T07:53:11.737Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"4d565c9f-3a0c-4dc2-ad7c-4848b1e3f561","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-23T07:53:11.737Z","status":"completed","discovery":{"result_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4","twelvedata.exchangerate.retrieve.v1.9eeb3b0d","coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","coinmarketcap.tools.priceconversion.retrieve.v2.24e991d9","bybitp2p.pay.fx.convert.v5.9221b0c4","coingecko_pro.exchangerates.retrieve.v3.0f6133ea","bybitp2p.payment.fx.convert.v5.7883b755","nbp_pl.exchangerates.rates.last.list.v1","coinmarketcap.tools.priceconversion.retrieve.v2.52e4cb2d","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d"]},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"returned_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4"],"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"fa50a78b-1fdd-47ad-965a-1630fc88bafb","result_valid":true},"finished_at":"2026-07-23T07:53:14.963Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"c8ab76aa-1e1b-4c55-8f77-f583d283d344","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-23T07:53:14.963Z","status":"completed","discovery":{"result_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4","twelvedata.exchangerate.retrieve.v1.9eeb3b0d","coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","coinmarketcap.tools.priceconversion.retrieve.v2.24e991d9","bybitp2p.pay.fx.convert.v5.9221b0c4","coingecko_pro.exchangerates.retrieve.v3.0f6133ea","bybitp2p.payment.fx.convert.v5.7883b755","nbp_pl.exchangerates.rates.last.list.v1","coinmarketcap.tools.priceconversion.retrieve.v2.52e4cb2d","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d"]},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"returned_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4"],"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"a1ddb43f-7eb0-48aa-9ed8-f469952a4f62","result_valid":true},"finished_at":"2026-07-23T07:53:18.126Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"3a5d9a59-34e8-4209-b700-633819ddcedf","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"news-ai","trial":1,"started_at":"2026-07-23T07:53:18.126Z","status":"completed","discovery":{"result_tool_ids":["x_developer.2.news.search.retrieve.v1.19b3e678","newsdata.news.search.v1.b65ccc56","x_developer.2.tweets.search.recent.retrieve.v2.f424c8f9","qveris_finance.finance_news_aggregation_v1","newsapi.top_headlines.list.v2.38face94","linkup.search.v1","newsapi.everything.retrieve.v2.e77ab1df","brave_search.news.search.list.v1.6ab52eaa","jina_search.rerank.create.v2026.23dc486d","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic"]},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"returned_tool_ids":["newsapi.everything.retrieve.v2.e77ab1df"],"required_parameters":["q"]},"parameters":{"q":"artificial intelligence","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"execution_id":"9d02d147-e450-4fb9-ac2c-da720aba6f1d","result_valid":true},"finished_at":"2026-07-23T07:53:20.737Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"1190b998-aceb-484b-8e96-8f711b737082","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"news-ai","trial":2,"started_at":"2026-07-23T07:53:20.737Z","status":"completed","discovery":{"result_tool_ids":["x_developer.2.news.search.retrieve.v1.19b3e678","newsdata.news.search.v1.b65ccc56","x_developer.2.tweets.search.recent.retrieve.v2.f424c8f9","qveris_finance.finance_news_aggregation_v1","newsapi.top_headlines.list.v2.38face94","linkup.search.v1","newsapi.everything.retrieve.v2.e77ab1df","brave_search.news.search.list.v1.6ab52eaa","jina_search.rerank.create.v2026.23dc486d","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic"]},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"returned_tool_ids":["newsapi.everything.retrieve.v2.e77ab1df"],"required_parameters":["q"]},"parameters":{"q":"artificial intelligence","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"execution_id":"4eabeb8c-81d1-47fe-8885-a9514512ae19","result_valid":true},"finished_at":"2026-07-23T07:53:23.488Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"9e1336ba-27ab-4d05-b4e2-72f24e2d9fd2","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"news-ai","trial":3,"started_at":"2026-07-23T07:53:23.488Z","status":"completed","discovery":{"result_tool_ids":["x_developer.2.news.search.retrieve.v1.19b3e678","newsdata.news.search.v1.b65ccc56","x_developer.2.tweets.search.recent.retrieve.v2.f424c8f9","qveris_finance.finance_news_aggregation_v1","newsapi.top_headlines.list.v2.38face94","linkup.search.v1","newsapi.everything.retrieve.v2.e77ab1df","brave_search.news.search.list.v1.6ab52eaa","jina_search.rerank.create.v2026.23dc486d","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic"]},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"returned_tool_ids":["newsapi.everything.retrieve.v2.e77ab1df"],"required_parameters":["q"]},"parameters":{"q":"artificial intelligence","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"execution_id":"e139ec85-de0b-4fe1-b788-760f4474fe52","result_valid":true},"finished_at":"2026-07-23T07:53:26.274Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"7ddaaf21-9eb5-4cbf-9e9e-ad0cf2b4da30","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T07:53:26.275Z","status":"completed","discovery":{"result_tool_ids":["semantic_scholar.paper.search.bulk.execute.v1.239dfd66","qveris_academic.paper_aggregation_v1","semantic_scholar.paper.search.bulk.retrieve.v1.096de627","semantic_scholar.paper.search.match.retrieve.v1.b9f166f3","semantic_scholar.paper.search.match.retrieve.v1.d3fd06f6","semantic_scholar.paper.autocomplete.retrieve.v1.e16c9798","semantic_scholar.paper.autocomplete.list.v1.99d870f6","xiaosu.scholar.search.v1.f79d2374","semantic_scholar.author.search.query.v1.1c428051","semantic_scholar.author.search.list.v1.ea3671fb"]},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"returned_tool_ids":["qveris_academic.paper_aggregation_v1"],"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"execution_id":"4fdca88a-862e-4fa8-9ee1-79ae131b52e3","result_valid":true},"finished_at":"2026-07-23T07:53:30.277Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"aa7da099-31bf-4f14-94b5-04b148b81bde","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T07:53:30.278Z","status":"completed","discovery":{"result_tool_ids":["semantic_scholar.paper.search.bulk.execute.v1.239dfd66","qveris_academic.paper_aggregation_v1","semantic_scholar.paper.search.bulk.retrieve.v1.096de627","semantic_scholar.paper.search.match.retrieve.v1.b9f166f3","semantic_scholar.paper.search.match.retrieve.v1.d3fd06f6","semantic_scholar.paper.autocomplete.retrieve.v1.e16c9798","semantic_scholar.paper.autocomplete.list.v1.99d870f6","xiaosu.scholar.search.v1.f79d2374","semantic_scholar.author.search.query.v1.1c428051","semantic_scholar.author.search.list.v1.ea3671fb"]},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"returned_tool_ids":["qveris_academic.paper_aggregation_v1"],"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"execution_id":"99f73406-fcb5-46ba-b898-684c1ad6b735","result_valid":true},"finished_at":"2026-07-23T07:53:34.089Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"b42f18f4-c0fe-443f-9d07-cb4870e88af6","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T07:53:34.089Z","status":"completed","discovery":{"result_tool_ids":["semantic_scholar.paper.search.bulk.execute.v1.239dfd66","qveris_academic.paper_aggregation_v1","semantic_scholar.paper.search.bulk.retrieve.v1.096de627","semantic_scholar.paper.search.match.retrieve.v1.b9f166f3","semantic_scholar.paper.search.match.retrieve.v1.d3fd06f6","semantic_scholar.paper.autocomplete.retrieve.v1.e16c9798","semantic_scholar.paper.autocomplete.list.v1.99d870f6","xiaosu.scholar.search.v1.f79d2374","semantic_scholar.author.search.query.v1.1c428051","semantic_scholar.author.search.list.v1.ea3671fb"]},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"returned_tool_ids":["qveris_academic.paper_aggregation_v1"],"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"execution_id":"230b7448-b12b-4ce1-951e-3aef321fcd97","result_valid":true},"finished_at":"2026-07-23T07:53:37.784Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ab798371-3cf3-4945-87c6-4240b8b9e36f","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T07:53:37.784Z","status":"failed","discovery":{"result_tool_ids":["api_sports.timezone.retrieve.v1.e993615d","api_sports.timezone.retrieve.v1.6792e6aa","api_sports.timezone.retrieve.v1.9b882d11","api_sports.timezone.list.v1.fafec612","api_sports.timezone.retrieve.v1.00ab54ff","api_sports.timezone.list.v1.bd878c58","api_sports.timezone.retrieve.v1.64cfc345","api_sports.timezone.list.v1.58366f46","api_sports.timezone.retrieve.v1.ae6ae8d7","api_sports.timezone.retrieve.v1.895e2257"]},"selection":{"tool_id":null},"inspection":{"returned_tool_ids":[],"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id","message":"Adapter did not return a non-empty tool_id"},"finished_at":"2026-07-23T07:53:39.282Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"3879f2c1-7e03-4819-b2b4-4dc353c9b6b6","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T07:53:39.282Z","status":"failed","discovery":{"result_tool_ids":["api_sports.timezone.retrieve.v1.e993615d","api_sports.timezone.retrieve.v1.6792e6aa","api_sports.timezone.retrieve.v1.9b882d11","api_sports.timezone.list.v1.fafec612","api_sports.timezone.retrieve.v1.00ab54ff","api_sports.timezone.list.v1.bd878c58","api_sports.timezone.retrieve.v1.64cfc345","api_sports.timezone.list.v1.58366f46","api_sports.timezone.retrieve.v1.ae6ae8d7","api_sports.timezone.retrieve.v1.895e2257"]},"selection":{"tool_id":null},"inspection":{"returned_tool_ids":[],"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id","message":"Adapter did not return a non-empty tool_id"},"finished_at":"2026-07-23T07:53:41.087Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"cb7160d0-c6ea-4e8b-a60e-bffc3e6747df","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T07:53:41.087Z","status":"failed","discovery":{"result_tool_ids":["api_sports.timezone.retrieve.v1.e993615d","api_sports.timezone.retrieve.v1.6792e6aa","api_sports.timezone.retrieve.v1.9b882d11","api_sports.timezone.list.v1.fafec612","api_sports.timezone.retrieve.v1.00ab54ff","api_sports.timezone.list.v1.bd878c58","api_sports.timezone.retrieve.v1.64cfc345","api_sports.timezone.list.v1.58366f46","api_sports.timezone.retrieve.v1.ae6ae8d7","api_sports.timezone.retrieve.v1.895e2257"]},"selection":{"tool_id":null},"inspection":{"returned_tool_ids":[],"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id","message":"Adapter did not return a non-empty tool_id"},"finished_at":"2026-07-23T07:53:42.486Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"c004662d-e17c-441a-8439-d18a4a1988c5","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T07:53:42.486Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.geocoder.reverse.v2","baidu_map.geocoder.query.v2","baidu_map.geocoding.reverse_geocode.v1","baidu_map.reversegeocoding.create.v1","baidu_map.reverse_geocoding.query.v2","baidu_map.geocoder.wgeoreverse.create.v1","amap_webservice.geocode.geo.retrieve.v3","baidu_map.geocoder.wgeobatch.create.v1","arcgis_ocean.geocoding.reverse_geocode.v1.4d40d93e","baidu_map.geoconv.convert.v1"]},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"returned_tool_ids":["amap_webservice.geocode.geo.retrieve.v3"],"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"06b8694c-6ab1-4f7b-8544-54d418e655fe","result_valid":true},"finished_at":"2026-07-23T07:53:46.409Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"aa9437b4-4a68-4467-9195-e421a33d242d","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T07:53:46.409Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.geocoder.reverse.v2","baidu_map.geocoder.query.v2","baidu_map.geocoding.reverse_geocode.v1","baidu_map.reversegeocoding.create.v1","baidu_map.reverse_geocoding.query.v2","baidu_map.geocoder.wgeoreverse.create.v1","amap_webservice.geocode.geo.retrieve.v3","baidu_map.geocoder.wgeobatch.create.v1","arcgis_ocean.geocoding.reverse_geocode.v1.4d40d93e","baidu_map.geoconv.convert.v1"]},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"returned_tool_ids":["amap_webservice.geocode.geo.retrieve.v3"],"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"81c8f609-3484-47ac-927a-cff6bb17c021","result_valid":true},"finished_at":"2026-07-23T07:53:51.677Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"c3632840-0ea8-46f3-9c0b-d721c14211ed","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T07:53:51.677Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.geocoder.reverse.v2","baidu_map.geocoder.query.v2","baidu_map.geocoding.reverse_geocode.v1","baidu_map.reversegeocoding.create.v1","baidu_map.reverse_geocoding.query.v2","baidu_map.geocoder.wgeoreverse.create.v1","amap_webservice.geocode.geo.retrieve.v3","baidu_map.geocoder.wgeobatch.create.v1","arcgis_ocean.geocoding.reverse_geocode.v1.4d40d93e","baidu_map.geoconv.convert.v1"]},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"returned_tool_ids":["amap_webservice.geocode.geo.retrieve.v3"],"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"98d922a3-7e54-40d6-8305-de4c6130323e","result_valid":true},"finished_at":"2026-07-23T07:53:55.919Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"a0d8e047-f8fa-4579-bec2-de6526299e1a","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-23T07:53:55.919Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.location.ip.retrieve.v1","decodo.isp_pay_per_ip.ip.retrieve.v1.bc2c6371","baidu_map.location.ip.execute.v1","virustotal.ip.info.retrieve.v3.ab34b0d1","amap_webservice.ip.retrieve.v3","weather_api.ip_lookup.retrieve.v1.e095d904","virustotal.ip_addresses.retrieve.v3.ab34b0d1","virustotal.ip.report.retrieve.v3.ab34b0d1","baidu_map.ip_location.retrieve.v1","virustotal.ip-address.report.retrieve.v2.79606e33"]},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"returned_tool_ids":["virustotal.ip.info.retrieve.v3.ab34b0d1"],"required_parameters":["ip"]},"parameters":{"ip":"8.8.8.8"},"call":{"attempted":true,"success":true,"execution_id":"85c9f4d0-b112-47f3-b500-8809ccedd99f","result_valid":true},"finished_at":"2026-07-23T07:53:58.844Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"bc8d8e9b-d160-465f-bda4-3b185ef33115","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-23T07:53:58.844Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.location.ip.retrieve.v1","decodo.isp_pay_per_ip.ip.retrieve.v1.bc2c6371","baidu_map.location.ip.execute.v1","virustotal.ip.info.retrieve.v3.ab34b0d1","amap_webservice.ip.retrieve.v3","weather_api.ip_lookup.retrieve.v1.e095d904","virustotal.ip_addresses.retrieve.v3.ab34b0d1","virustotal.ip.report.retrieve.v3.ab34b0d1","baidu_map.ip_location.retrieve.v1","virustotal.ip-address.report.retrieve.v2.79606e33"]},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"returned_tool_ids":["virustotal.ip.info.retrieve.v3.ab34b0d1"],"required_parameters":["ip"]},"parameters":{"ip":"8.8.8.8"},"call":{"attempted":true,"success":true,"execution_id":"a0581017-05ca-4296-818f-7f00105ca92f","result_valid":true},"finished_at":"2026-07-23T07:54:02.027Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"cd7516ab-c10d-4af9-8063-273a79f68af3","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-23T07:54:02.027Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.location.ip.retrieve.v1","decodo.isp_pay_per_ip.ip.retrieve.v1.bc2c6371","baidu_map.location.ip.execute.v1","virustotal.ip.info.retrieve.v3.ab34b0d1","amap_webservice.ip.retrieve.v3","weather_api.ip_lookup.retrieve.v1.e095d904","virustotal.ip_addresses.retrieve.v3.ab34b0d1","virustotal.ip.report.retrieve.v3.ab34b0d1","baidu_map.ip_location.retrieve.v1","virustotal.ip-address.report.retrieve.v2.79606e33"]},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"returned_tool_ids":["virustotal.ip.info.retrieve.v3.ab34b0d1"],"required_parameters":["ip"]},"parameters":{"ip":"8.8.8.8"},"call":{"attempted":true,"success":true,"execution_id":"e1bcc9d4-9f48-45c2-8bd8-acdc868eb913","result_valid":true},"finished_at":"2026-07-23T07:54:05.637Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"be7b9cb2-88b4-4148-b1aa-4a17f035c27a","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"crypto-price","trial":1,"started_at":"2026-07-23T07:54:05.637Z","status":"completed","discovery":{"result_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","binance.avg_price.retrieve.v1.aef0de4f","etherscan.api.stats.ethprice.retrieve.v2.8eeca59e","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.251242d7","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.9dc24be4","binance.ticker.price.list.v3.8675eca0","binance.avgPrice.retrieve.v3.7b0852b4","deribit.public.get_index_price.retrieve.v2.d47dcdcb","binance.ticker.query.v1.0437113d","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d"]},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"returned_tool_ids":["binance.ticker.price.list.v3.8675eca0"],"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"execution_id":"6cab41f3-55c0-4bf9-9bd5-c99de6753874","result_valid":true},"finished_at":"2026-07-23T07:54:09.422Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"fa26dcf3-d647-434e-8fc4-cfb726806704","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"crypto-price","trial":2,"started_at":"2026-07-23T07:54:09.422Z","status":"completed","discovery":{"result_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","binance.avg_price.retrieve.v1.aef0de4f","etherscan.api.stats.ethprice.retrieve.v2.8eeca59e","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.251242d7","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.9dc24be4","binance.ticker.price.list.v3.8675eca0","binance.avgPrice.retrieve.v3.7b0852b4","deribit.public.get_index_price.retrieve.v2.d47dcdcb","binance.ticker.query.v1.0437113d","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d"]},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"returned_tool_ids":["binance.ticker.price.list.v3.8675eca0"],"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"execution_id":"3606598a-880a-486b-a236-0b07cae220a3","result_valid":true},"finished_at":"2026-07-23T07:54:12.396Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"5a70425d-4ab1-48a2-a484-b91463c77e34","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"crypto-price","trial":3,"started_at":"2026-07-23T07:54:12.396Z","status":"completed","discovery":{"result_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","binance.avg_price.retrieve.v1.aef0de4f","etherscan.api.stats.ethprice.retrieve.v2.8eeca59e","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.251242d7","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.9dc24be4","binance.ticker.price.list.v3.8675eca0","binance.avgPrice.retrieve.v3.7b0852b4","deribit.public.get_index_price.retrieve.v2.d47dcdcb","binance.ticker.query.v1.0437113d","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d"]},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"returned_tool_ids":["binance.ticker.price.list.v3.8675eca0"],"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"execution_id":"2fde8e07-7939-4545-8321-6ce7d557f37a","result_valid":true},"finished_at":"2026-07-23T07:54:15.174Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"cc44232f-8d5d-40e4-87d0-914004721dfc","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"web-search","trial":1,"started_at":"2026-07-23T07:54:15.174Z","status":"completed","discovery":{"result_tool_ids":["linkup.search.v1","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic","cloudsway.smart_search_overseas","brave_search.images.search.list.v1","firecrawl.search.create.v2.a99b6f74","serpapi.locations.locations.list.v1","x_developer.2.news.search.retrieve.v1.19b3e678","x_developer.2.spaces.search.retrieve.v1.43a1f31c","tavily.map.execute.v1","brave_search.web.search.list.v1.f02ed09d"]},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"returned_tool_ids":["cloudsway.smart_search_overseas"],"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"execution_id":"263066a3-72ae-4a8d-9422-aaa015590464","result_valid":true},"finished_at":"2026-07-23T07:54:17.456Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"0ecbcede-88b3-403b-934f-8e1ce9909259","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"web-search","trial":2,"started_at":"2026-07-23T07:54:17.456Z","status":"completed","discovery":{"result_tool_ids":["linkup.search.v1","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic","cloudsway.smart_search_overseas","brave_search.images.search.list.v1","firecrawl.search.create.v2.a99b6f74","serpapi.locations.locations.list.v1","x_developer.2.news.search.retrieve.v1.19b3e678","x_developer.2.spaces.search.retrieve.v1.43a1f31c","tavily.map.execute.v1","brave_search.web.search.list.v1.f02ed09d"]},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"returned_tool_ids":["cloudsway.smart_search_overseas"],"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"execution_id":"99ba7ee3-2175-4e5b-b887-fd486bf4f03b","result_valid":true},"finished_at":"2026-07-23T07:54:20.037Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"7b1f0b31-d0a7-4fb7-b807-d8633bd42387","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"web-search","trial":3,"started_at":"2026-07-23T07:54:20.037Z","status":"completed","discovery":{"result_tool_ids":["linkup.search.v1","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic","cloudsway.smart_search_overseas","brave_search.images.search.list.v1","firecrawl.search.create.v2.a99b6f74","serpapi.locations.locations.list.v1","x_developer.2.news.search.retrieve.v1.19b3e678","x_developer.2.spaces.search.retrieve.v1.43a1f31c","tavily.map.execute.v1","brave_search.web.search.list.v1.f02ed09d"]},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"returned_tool_ids":["cloudsway.smart_search_overseas"],"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"execution_id":"e8756117-a9dc-4130-8f78-ab65b1f14b72","result_valid":true},"finished_at":"2026-07-23T07:54:22.508Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"e85eed29-7b7c-4c24-875b-7c4ec6f2afc6","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"air-quality","trial":1,"started_at":"2026-07-23T07:54:22.508Z","status":"completed","discovery":{"result_tool_ids":["openweathermap.air_pollution.current.v2_5","openweathermap.air_pollution.retrieve.v2.5","openweathermap.air_pollution.history.v2_5","openweathermap.air_pollution.history.retrieve.v2.5","openweathermap.air_pollution.forecast.retrieve.v2.5","epa_envirofacts.qaannualperformanceevaluations.bystate.retrieve.v1.61a2aa7d","epa_envirofacts.transactionsqaannualperformanceevaluations.bystate.retrieve.v1.4143b8bf","epa_envirofacts.transactionssample.bysite.retrieve.v1.93889e72","epa_envirofacts.annualdata.bybox.retrieve.v1.4b74a0d6","epa_envirofacts.dailydata.bycounty.retrieve.v1.f1ee8d1b"]},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"returned_tool_ids":["openweathermap.air_pollution.current.v2_5"],"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"execution_id":"e73a69f2-623b-4c65-9994-cbfe8248e4d9","result_valid":true},"finished_at":"2026-07-23T07:54:25.490Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"203b9a62-2140-4edb-b40a-61c95d9c3146","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"air-quality","trial":2,"started_at":"2026-07-23T07:54:25.490Z","status":"completed","discovery":{"result_tool_ids":["openweathermap.air_pollution.current.v2_5","openweathermap.air_pollution.retrieve.v2.5","openweathermap.air_pollution.history.v2_5","openweathermap.air_pollution.history.retrieve.v2.5","openweathermap.air_pollution.forecast.retrieve.v2.5","epa_envirofacts.qaannualperformanceevaluations.bystate.retrieve.v1.61a2aa7d","epa_envirofacts.transactionsqaannualperformanceevaluations.bystate.retrieve.v1.4143b8bf","epa_envirofacts.transactionssample.bysite.retrieve.v1.93889e72","epa_envirofacts.annualdata.bybox.retrieve.v1.4b74a0d6","epa_envirofacts.dailydata.bycounty.retrieve.v1.f1ee8d1b"]},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"returned_tool_ids":["openweathermap.air_pollution.current.v2_5"],"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"execution_id":"8bddf105-bf44-485e-901a-77235d5de868","result_valid":true},"finished_at":"2026-07-23T07:54:27.929Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"95854287-bfed-486a-9aff-20d839999229","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"air-quality","trial":3,"started_at":"2026-07-23T07:54:27.929Z","status":"completed","discovery":{"result_tool_ids":["openweathermap.air_pollution.current.v2_5","openweathermap.air_pollution.retrieve.v2.5","openweathermap.air_pollution.history.v2_5","openweathermap.air_pollution.history.retrieve.v2.5","openweathermap.air_pollution.forecast.retrieve.v2.5","epa_envirofacts.qaannualperformanceevaluations.bystate.retrieve.v1.61a2aa7d","epa_envirofacts.transactionsqaannualperformanceevaluations.bystate.retrieve.v1.4143b8bf","epa_envirofacts.transactionssample.bysite.retrieve.v1.93889e72","epa_envirofacts.annualdata.bybox.retrieve.v1.4b74a0d6","epa_envirofacts.dailydata.bycounty.retrieve.v1.f1ee8d1b"]},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"returned_tool_ids":["openweathermap.air_pollution.current.v2_5"],"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"execution_id":"720179b8-bbdb-4eea-b58c-10c52d7da23a","result_valid":true},"finished_at":"2026-07-23T07:54:30.101Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ff1c9565-ffe8-4a70-adc2-b5215c3410ff","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-news","trial":1,"started_at":"2026-07-23T07:54:30.101Z","status":"completed","discovery":{"result_tool_ids":["finnhub.news.retrieve.v1.2e72ea5d","finnhub.market.news.list.v1","finnhub.news.retrieve.v1","finnhub.news.retrieve.v1.51b2567e","finnhub.news.execute.v1","finnhub.companynews.retrieve.v1","alphavantage.news_sentiment.query.v1.467a92c0","finnhub.companynews.retrieve.v1.4d11f47b","finnhub.companynews.retrieve.v1.e428b704","finnhub.companynews.execute.v1"]},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"returned_tool_ids":["finnhub.market.news.list.v1"],"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"execution_id":"3672275c-75aa-4a0b-b4d2-036c94fbbaa2","result_valid":true},"finished_at":"2026-07-23T07:54:33.694Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"56a668ab-8a69-4277-8729-4a1717af668c","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-news","trial":2,"started_at":"2026-07-23T07:54:33.695Z","status":"completed","discovery":{"result_tool_ids":["finnhub.news.retrieve.v1.2e72ea5d","finnhub.market.news.list.v1","finnhub.news.retrieve.v1","finnhub.news.retrieve.v1.51b2567e","finnhub.news.execute.v1","finnhub.companynews.retrieve.v1","alphavantage.news_sentiment.query.v1.467a92c0","finnhub.companynews.retrieve.v1.4d11f47b","finnhub.companynews.retrieve.v1.e428b704","finnhub.companynews.execute.v1"]},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"returned_tool_ids":["finnhub.market.news.list.v1"],"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"execution_id":"0db2574e-d68c-4fd6-87d7-d2683b157b26","result_valid":true},"finished_at":"2026-07-23T07:54:37.509Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"2fd403d8-a11c-41e1-9390-d076ca5dfdb0","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-news","trial":3,"started_at":"2026-07-23T07:54:37.510Z","status":"completed","discovery":{"result_tool_ids":["finnhub.news.retrieve.v1.2e72ea5d","finnhub.market.news.list.v1","finnhub.news.retrieve.v1","finnhub.news.retrieve.v1.51b2567e","finnhub.news.execute.v1","finnhub.companynews.retrieve.v1","alphavantage.news_sentiment.query.v1.467a92c0","finnhub.companynews.retrieve.v1.4d11f47b","finnhub.companynews.retrieve.v1.e428b704","finnhub.companynews.execute.v1"]},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"returned_tool_ids":["finnhub.market.news.list.v1"],"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"execution_id":"262940d0-2688-431c-9ea7-9e19e83d7ba9","result_valid":true},"finished_at":"2026-07-23T07:54:41.063Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"8b3f5092-fa39-4426-a908-35e2af2c9312","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T07:54:41.063Z","status":"completed","discovery":{"result_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8","dataforseo.dataforseo_labs.google.domain_whois_overview.live.execute.v3","dataforseo.databases.domains.query.v3.c1720ec0","virustotal.domains.report.retrieve.v3.c5459740","bainiu_api.common.company_domain.query.v1.e978ca32","virustotal.domain_reports.retrieve.v3.c5459740","dataforseo.domain_analytics.technologies.domains_by_technology.live.create.v3.bfc3dc2d","virustotal.domains.retrieve.v3.c5459740","virustotal.domains.analyse.v3.c755c69e","dataforseo.domain_analytics.technologies.domains_by_html_terms.live.create.v3.d3760ae1"]},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"returned_tool_ids":["virustotal.domains.report.retrieve.v3.c5459740"],"required_parameters":["domain"]},"parameters":{"domain":"example.com"},"call":{"attempted":true,"success":true,"execution_id":"2aaef5cc-ccf1-4eff-96f9-07eaa355d05a","result_valid":true},"finished_at":"2026-07-23T07:54:44.215Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"d454cf08-3fbd-4a50-bb4a-9af533a5675d","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T07:54:44.215Z","status":"completed","discovery":{"result_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8","dataforseo.dataforseo_labs.google.domain_whois_overview.live.execute.v3","dataforseo.databases.domains.query.v3.c1720ec0","virustotal.domains.report.retrieve.v3.c5459740","bainiu_api.common.company_domain.query.v1.e978ca32","virustotal.domain_reports.retrieve.v3.c5459740","dataforseo.domain_analytics.technologies.domains_by_technology.live.create.v3.bfc3dc2d","virustotal.domains.retrieve.v3.c5459740","virustotal.domains.analyse.v3.c755c69e","dataforseo.domain_analytics.technologies.domains_by_html_terms.live.create.v3.d3760ae1"]},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"returned_tool_ids":["virustotal.domains.report.retrieve.v3.c5459740"],"required_parameters":["domain"]},"parameters":{"domain":"example.com"},"call":{"attempted":true,"success":true,"execution_id":"f90745cc-a050-4e1f-a2f5-bbc689b2721a","result_valid":true},"finished_at":"2026-07-23T07:54:47.714Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"d24fe5b1-5ffb-427b-a45a-37baffde5d36","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T07:54:47.714Z","status":"completed","discovery":{"result_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8","dataforseo.dataforseo_labs.google.domain_whois_overview.live.execute.v3","dataforseo.databases.domains.query.v3.c1720ec0","virustotal.domains.report.retrieve.v3.c5459740","bainiu_api.common.company_domain.query.v1.e978ca32","virustotal.domain_reports.retrieve.v3.c5459740","dataforseo.domain_analytics.technologies.domains_by_technology.live.create.v3.bfc3dc2d","virustotal.domains.retrieve.v3.c5459740","virustotal.domains.analyse.v3.c755c69e","dataforseo.domain_analytics.technologies.domains_by_html_terms.live.create.v3.d3760ae1"]},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"returned_tool_ids":["virustotal.domains.report.retrieve.v3.c5459740"],"required_parameters":["domain"]},"parameters":{"domain":"example.com"},"call":{"attempted":true,"success":true,"execution_id":"8f66f75e-1ccb-48fe-b37f-580d0e6d86f7","result_valid":true},"finished_at":"2026-07-23T07:54:50.804Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"66cecd06-6fce-4084-94a3-4997791e2303","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"image-search","trial":1,"started_at":"2026-07-23T07:54:50.804Z","status":"completed","discovery":{"result_tool_ids":["brave_search.images.search.list.v1","shutterstock.images.search.list.v2.69059569","shutterstock.images.search.list.v2.c2b44682","dataforseo.serp.google.search_by_image.task.create.v3","jina_search.embeddings.create.v2026.5c7cb661","jina_search.rerank.create.v2026.23dc486d","linkup.search.v1","shutterstock.images.search.list.v2.15f15845","shutterstock.images.search.suggestions.list.v2.f6850c10","shutterstock.images.search.suggestions.list.v2"]},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"returned_tool_ids":["brave_search.images.search.list.v1"],"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge"},"call":{"attempted":true,"success":true,"execution_id":"399c7b71-6f72-425b-89f4-96486128d89b","result_valid":true},"finished_at":"2026-07-23T07:54:54.158Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ffdc3990-5991-4e84-bfb9-4997434ec7e8","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"image-search","trial":2,"started_at":"2026-07-23T07:54:54.158Z","status":"completed","discovery":{"result_tool_ids":["brave_search.images.search.list.v1","shutterstock.images.search.list.v2.69059569","shutterstock.images.search.list.v2.c2b44682","dataforseo.serp.google.search_by_image.task.create.v3","jina_search.embeddings.create.v2026.5c7cb661","jina_search.rerank.create.v2026.23dc486d","linkup.search.v1","shutterstock.images.search.list.v2.15f15845","shutterstock.images.search.suggestions.list.v2.f6850c10","shutterstock.images.search.suggestions.list.v2"]},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"returned_tool_ids":["brave_search.images.search.list.v1"],"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge"},"call":{"attempted":true,"success":true,"execution_id":"c632bc27-e454-4809-b1b9-76abb2c24b3f","result_valid":true},"finished_at":"2026-07-23T07:54:57.764Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"8fc69d41-be93-424e-ae30-754fb957f63b","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"image-search","trial":3,"started_at":"2026-07-23T07:54:57.764Z","status":"completed","discovery":{"result_tool_ids":["brave_search.images.search.list.v1","shutterstock.images.search.list.v2.69059569","shutterstock.images.search.list.v2.c2b44682","dataforseo.serp.google.search_by_image.task.create.v3","jina_search.embeddings.create.v2026.5c7cb661","jina_search.rerank.create.v2026.23dc486d","linkup.search.v1","shutterstock.images.search.list.v2.15f15845","shutterstock.images.search.suggestions.list.v2.f6850c10","shutterstock.images.search.suggestions.list.v2"]},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"returned_tool_ids":["brave_search.images.search.list.v1"],"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge"},"call":{"attempted":true,"success":true,"execution_id":"53c4f6b7-26d5-4fa8-a424-41dbee950069","result_valid":true},"finished_at":"2026-07-23T07:55:01.885Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"5cd557cc-6800-41f3-b397-f194865cc601","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-holidays","trial":1,"started_at":"2026-07-23T07:55:01.885Z","status":"completed","discovery":{"result_tool_ids":["finnhub.stock.marketholiday.retrieve.v1.75d6c1f9","finnhub.stock.marketholiday.retrieve.v1","finnhub.stock.marketholiday.execute.v1","eodhd.exchange_details.retrieve.v1.7da32390","finnhub.stock.marketholiday.retrieve.v1.5c5bfe9a","massive_stocks.stocks.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.retrieve.v1","massive_stocks.options.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.list.v1","massive_stocks.options.marketstatus.retrieve.v1.3045ae6e"]},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"returned_tool_ids":["finnhub.stock.marketholiday.retrieve.v1"],"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"execution_id":"8ee1342e-41a1-4127-874f-91a442534efe","result_valid":true},"finished_at":"2026-07-23T07:55:04.949Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"5c5c80cb-d46e-42c9-bbb7-c173a487851d","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-holidays","trial":2,"started_at":"2026-07-23T07:55:04.949Z","status":"completed","discovery":{"result_tool_ids":["finnhub.stock.marketholiday.retrieve.v1.75d6c1f9","finnhub.stock.marketholiday.retrieve.v1","finnhub.stock.marketholiday.execute.v1","eodhd.exchange_details.retrieve.v1.7da32390","finnhub.stock.marketholiday.retrieve.v1.5c5bfe9a","massive_stocks.stocks.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.retrieve.v1","massive_stocks.options.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.list.v1","massive_stocks.options.marketstatus.retrieve.v1.3045ae6e"]},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"returned_tool_ids":["finnhub.stock.marketholiday.retrieve.v1"],"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"execution_id":"9db6a044-57ff-4216-8d0e-84893a01cfd8","result_valid":true},"finished_at":"2026-07-23T07:55:07.898Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"e0657299-6f28-4851-8554-1b4d50126bfb","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-holidays","trial":3,"started_at":"2026-07-23T07:55:07.898Z","status":"completed","discovery":{"result_tool_ids":["finnhub.stock.marketholiday.retrieve.v1.75d6c1f9","finnhub.stock.marketholiday.retrieve.v1","finnhub.stock.marketholiday.execute.v1","eodhd.exchange_details.retrieve.v1.7da32390","finnhub.stock.marketholiday.retrieve.v1.5c5bfe9a","massive_stocks.stocks.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.retrieve.v1","massive_stocks.options.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.list.v1","massive_stocks.options.marketstatus.retrieve.v1.3045ae6e"]},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"returned_tool_ids":["finnhub.stock.marketholiday.retrieve.v1"],"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"execution_id":"735442ad-1d5a-46a7-92b3-7166f78e1c8f","result_valid":true},"finished_at":"2026-07-23T07:55:11.714Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"53649f10-7f15-439b-b58e-c8e01a811995","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-23T07:55:11.714Z","status":"completed","discovery":{"result_tool_ids":["usgs_earthquake_catalog.event.events.list.v1","usgs_earthquake_catalog.fdsnws.event.query.retrieve.v1.025549c9","usgs_earthquake_catalog.earthquakes.query.v1","usgs_earthquake_catalog.fdsnws.event.query.v1.025549c9","usgs_earthquake_catalog.events.query.v1.025549c9","usgs_earthquake.catalogs.list.v1","usgs_earthquake.contributors.list.v1","usgs-earthquake-event-api.event.query.list.v1","usgs_earthquake.parameters.list.v1","usgs_earthquake.version.retrieve.v1"]},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"returned_tool_ids":["usgs_earthquake_catalog.event.events.list.v1"],"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"execution_id":"8db7bed3-07ad-48e8-ae80-b5ca5235bb49","result_valid":true},"finished_at":"2026-07-23T07:55:20.097Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"cdefdbdd-5005-472b-b3e0-d33e3ecc51c5","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-23T07:55:20.097Z","status":"completed","discovery":{"result_tool_ids":["usgs_earthquake_catalog.event.events.list.v1","usgs_earthquake_catalog.fdsnws.event.query.retrieve.v1.025549c9","usgs_earthquake_catalog.earthquakes.query.v1","usgs_earthquake_catalog.fdsnws.event.query.v1.025549c9","usgs_earthquake_catalog.events.query.v1.025549c9","usgs_earthquake.catalogs.list.v1","usgs_earthquake.contributors.list.v1","usgs-earthquake-event-api.event.query.list.v1","usgs_earthquake.parameters.list.v1","usgs_earthquake.version.retrieve.v1"]},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"returned_tool_ids":["usgs_earthquake_catalog.event.events.list.v1"],"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"execution_id":"1b7c3dc5-3ada-4646-ac8c-735dc5251824","result_valid":true},"finished_at":"2026-07-23T07:55:28.789Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"929db9db-2ef4-466d-a80b-2e14af1f4c7c","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T07:55:28.789Z","status":"completed","discovery":{"result_tool_ids":["usgs_earthquake_catalog.event.events.list.v1","usgs_earthquake_catalog.fdsnws.event.query.retrieve.v1.025549c9","usgs_earthquake_catalog.earthquakes.query.v1","usgs_earthquake_catalog.fdsnws.event.query.v1.025549c9","usgs_earthquake_catalog.events.query.v1.025549c9","usgs_earthquake.catalogs.list.v1","usgs_earthquake.contributors.list.v1","usgs-earthquake-event-api.event.query.list.v1","usgs_earthquake.parameters.list.v1","usgs_earthquake.version.retrieve.v1"]},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"returned_tool_ids":["usgs_earthquake_catalog.event.events.list.v1"],"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"execution_id":"d61b9ae7-e2ef-4632-b542-9a61aaa061f2","result_valid":true},"finished_at":"2026-07-23T07:55:36.177Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"711a0f3a-7f40-4b3d-ab18-0fd661803757","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"company-profile","trial":1,"started_at":"2026-07-23T07:55:36.177Z","status":"completed","discovery":{"result_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2","bainiu_api.company.detail.execute.v1.f6b528e7","bainiu_api.openapi.company.alter.execute.v1.8fa090d1","financialmodelingprep.stable.companynotes.retrieve.v1.930aa468","tradefeeds.compinfo.retrieve.v1.c2314765","financialmodelingprep.stable.secfilingscompanysearch.symbol.retrieve.v1.5cf7397d","financialmodelingprep.search.cusip.retrieve.v1.51d0c621","bainiu_api.openapi.common.company_info_deep_search.execute.v1.8e5f009d","finnhub.stock.profile2.retrieve.v1.59df26f6","bainiu_api.openapi.common.company_lianxi_public.execute.v1.bd6261e8"]},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"returned_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"],"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":true,"execution_id":"0938428f-f710-49e3-92d2-7aaf488c788b","result_valid":true},"finished_at":"2026-07-23T07:55:40.433Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"342d9249-b2e3-43e1-9db2-9d6584889277","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"company-profile","trial":2,"started_at":"2026-07-23T07:55:40.433Z","status":"completed","discovery":{"result_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2","bainiu_api.company.detail.execute.v1.f6b528e7","bainiu_api.openapi.company.alter.execute.v1.8fa090d1","financialmodelingprep.stable.companynotes.retrieve.v1.930aa468","tradefeeds.compinfo.retrieve.v1.c2314765","financialmodelingprep.stable.secfilingscompanysearch.symbol.retrieve.v1.5cf7397d","financialmodelingprep.search.cusip.retrieve.v1.51d0c621","bainiu_api.openapi.common.company_info_deep_search.execute.v1.8e5f009d","finnhub.stock.profile2.retrieve.v1.59df26f6","bainiu_api.openapi.common.company_lianxi_public.execute.v1.bd6261e8"]},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"returned_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"],"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":true,"execution_id":"174e2b74-0f18-4300-8dec-ecb8775cde8e","result_valid":true},"finished_at":"2026-07-23T07:55:43.901Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"063b8a50-9c2c-41dd-89df-4a639c354c74","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"company-profile","trial":3,"started_at":"2026-07-23T07:55:43.901Z","status":"completed","discovery":{"result_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2","bainiu_api.company.detail.execute.v1.f6b528e7","bainiu_api.openapi.company.alter.execute.v1.8fa090d1","financialmodelingprep.stable.companynotes.retrieve.v1.930aa468","tradefeeds.compinfo.retrieve.v1.c2314765","financialmodelingprep.stable.secfilingscompanysearch.symbol.retrieve.v1.5cf7397d","financialmodelingprep.search.cusip.retrieve.v1.51d0c621","bainiu_api.openapi.common.company_info_deep_search.execute.v1.8e5f009d","finnhub.stock.profile2.retrieve.v1.59df26f6","bainiu_api.openapi.common.company_lianxi_public.execute.v1.bd6261e8"]},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"returned_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"],"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":true,"execution_id":"fa6ff70f-0e6d-49de-8bae-23b085cb48fc","result_valid":true},"finished_at":"2026-07-23T07:55:47.413Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"34788c2f-c12d-46b8-9a88-d4285ab363de","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"sports-standings","trial":1,"started_at":"2026-07-23T07:55:47.413Z","status":"completed","discovery":{"result_tool_ids":["api_sports.standings.list.v1.98a153bd","api_sports.standings.list.v1.39568ba4","api_sports.standings.groups.list.v1.0d8a40cf","api_sports.standings.groups.list.v1.a077a680","api_sports.standings.stages.list.v1.5ba13a89","api_sports.standings.retrieve.v1.ff019893","api_sports.standings.groups.list.v1.57393fb3","api_sports.standings.stages.list.v1.838f769a","api-football.standings.list.v3.76f7703e","api_sports.standings.list.v1.5956be03"]},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"returned_tool_ids":["api_sports.standings.list.v1.39568ba4"],"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"execution_id":"b8c1f968-3226-47f9-8ba1-9e99406099ee","result_valid":true},"finished_at":"2026-07-23T07:55:51.371Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"1ae754a2-5fe1-4baa-85b8-e5c6e8664576","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"sports-standings","trial":2,"started_at":"2026-07-23T07:55:51.371Z","status":"completed","discovery":{"result_tool_ids":["api_sports.standings.list.v1.98a153bd","api_sports.standings.list.v1.39568ba4","api_sports.standings.groups.list.v1.0d8a40cf","api_sports.standings.groups.list.v1.a077a680","api_sports.standings.stages.list.v1.5ba13a89","api_sports.standings.retrieve.v1.ff019893","api_sports.standings.groups.list.v1.57393fb3","api_sports.standings.stages.list.v1.838f769a","api-football.standings.list.v3.76f7703e","api_sports.standings.list.v1.5956be03"]},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"returned_tool_ids":["api_sports.standings.list.v1.39568ba4"],"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"execution_id":"13f07f7b-c843-4cb8-abd3-cd10aca5ae09","result_valid":true},"finished_at":"2026-07-23T07:55:54.947Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ddfc24d1-5ac5-4861-b959-7013b1160520","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"sports-standings","trial":3,"started_at":"2026-07-23T07:55:54.947Z","status":"completed","discovery":{"result_tool_ids":["api_sports.standings.list.v1.98a153bd","api_sports.standings.list.v1.39568ba4","api_sports.standings.groups.list.v1.0d8a40cf","api_sports.standings.groups.list.v1.a077a680","api_sports.standings.stages.list.v1.5ba13a89","api_sports.standings.retrieve.v1.ff019893","api_sports.standings.groups.list.v1.57393fb3","api_sports.standings.stages.list.v1.838f769a","api-football.standings.list.v3.76f7703e","api_sports.standings.list.v1.5956be03"]},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"returned_tool_ids":["api_sports.standings.list.v1.39568ba4"],"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"execution_id":"6bdc5e3b-9d36-49d7-805d-8a5536dfc0a8","result_valid":true},"finished_at":"2026-07-23T07:55:58.608Z"} diff --git a/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.summary.json b/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.summary.json new file mode 100644 index 0000000..9b58557 --- /dev/null +++ b/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.summary.json @@ -0,0 +1,1439 @@ +{ + "schema_version": 1, + "methodology": "discover-call-v1", + "generated_at": "2026-07-23T07:55:58.643Z", + "models": [ + { + "model": "oracle-v1", + "lane": "oracle", + "tasks": 18, + "trials_per_task": 3, + "runs": 54, + "completed": 51, + "executed": 51, + "selection_grounded_rate": 0.9444, + "inspection_grounded_rate": 0.9444, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0.9444, + "call_success_rate": 1, + "result_valid_rate": 1, + "workflow_success_rate": 0.9444, + "workflow_success_wilson_95": [ + 0.8489, + 0.9809 + ], + "failures_by_stage": { + "select": 3 + }, + "failures_by_reason": { + "missing_tool_id": 3 + } + } + ], + "records": [ + { + "run_id": "8ba500d0-14cb-46a7-a347-c9ba9be3a3a8", + "task_id": "weather-london", + "model": "oracle-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "1f240f6b-6c51-4970-94c8-86a99dd6a36e", + "task_id": "weather-london", + "model": "oracle-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "891080de-9c5f-45d3-897f-94db7557d1c1", + "task_id": "weather-london", + "model": "oracle-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "f32129b5-3004-4733-90c2-6137c3a74d0f", + "task_id": "stock-aapl", + "model": "oracle-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "28f35db3-bab5-4b00-9aa3-e29371acf680", + "task_id": "stock-aapl", + "model": "oracle-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "770c7467-c2b6-4090-8826-0182ccfae9da", + "task_id": "stock-aapl", + "model": "oracle-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "c4624c8d-fb7b-4d9e-9248-209a6679336b", + "task_id": "exchange-usd-eur", + "model": "oracle-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "4d565c9f-3a0c-4dc2-ad7c-4848b1e3f561", + "task_id": "exchange-usd-eur", + "model": "oracle-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "c8ab76aa-1e1b-4c55-8f77-f583d283d344", + "task_id": "exchange-usd-eur", + "model": "oracle-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "3a5d9a59-34e8-4209-b700-633819ddcedf", + "task_id": "news-ai", + "model": "oracle-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "1190b998-aceb-484b-8e96-8f711b737082", + "task_id": "news-ai", + "model": "oracle-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "9e1336ba-27ab-4d05-b4e2-72f24e2d9fd2", + "task_id": "news-ai", + "model": "oracle-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "7ddaaf21-9eb5-4cbf-9e9e-ad0cf2b4da30", + "task_id": "papers-climate", + "model": "oracle-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "aa7da099-31bf-4f14-94b5-04b148b81bde", + "task_id": "papers-climate", + "model": "oracle-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "b42f18f4-c0fe-443f-9d07-cb4870e88af6", + "task_id": "papers-climate", + "model": "oracle-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "ab798371-3cf3-4945-87c6-4240b8b9e36f", + "task_id": "timezone-tokyo", + "model": "oracle-v1", + "trial": 1, + "completed": false, + "executed": false, + "selection_grounded": false, + "inspection_grounded": false, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": null, + "result_valid": null, + "workflow_success": false, + "error_stage": "select", + "error_reason": "missing_tool_id", + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "3879f2c1-7e03-4819-b2b4-4dc353c9b6b6", + "task_id": "timezone-tokyo", + "model": "oracle-v1", + "trial": 2, + "completed": false, + "executed": false, + "selection_grounded": false, + "inspection_grounded": false, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": null, + "result_valid": null, + "workflow_success": false, + "error_stage": "select", + "error_reason": "missing_tool_id", + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "cb7160d0-c6ea-4e8b-a60e-bffc3e6747df", + "task_id": "timezone-tokyo", + "model": "oracle-v1", + "trial": 3, + "completed": false, + "executed": false, + "selection_grounded": false, + "inspection_grounded": false, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": null, + "result_valid": null, + "workflow_success": false, + "error_stage": "select", + "error_reason": "missing_tool_id", + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "c004662d-e17c-441a-8439-d18a4a1988c5", + "task_id": "geocode-address", + "model": "oracle-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "aa9437b4-4a68-4467-9195-e421a33d242d", + "task_id": "geocode-address", + "model": "oracle-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "c3632840-0ea8-46f3-9c0b-d721c14211ed", + "task_id": "geocode-address", + "model": "oracle-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "a0d8e047-f8fa-4579-bec2-de6526299e1a", + "task_id": "ip-geolocation", + "model": "oracle-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "bc8d8e9b-d160-465f-bda4-3b185ef33115", + "task_id": "ip-geolocation", + "model": "oracle-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "cd7516ab-c10d-4af9-8063-273a79f68af3", + "task_id": "ip-geolocation", + "model": "oracle-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "be7b9cb2-88b4-4148-b1aa-4a17f035c27a", + "task_id": "crypto-price", + "model": "oracle-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "fa26dcf3-d647-434e-8fc4-cfb726806704", + "task_id": "crypto-price", + "model": "oracle-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "5a70425d-4ab1-48a2-a484-b91463c77e34", + "task_id": "crypto-price", + "model": "oracle-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "cc44232f-8d5d-40e4-87d0-914004721dfc", + "task_id": "web-search", + "model": "oracle-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "0ecbcede-88b3-403b-934f-8e1ce9909259", + "task_id": "web-search", + "model": "oracle-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "7b1f0b31-d0a7-4fb7-b807-d8633bd42387", + "task_id": "web-search", + "model": "oracle-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "e85eed29-7b7c-4c24-875b-7c4ec6f2afc6", + "task_id": "air-quality", + "model": "oracle-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "203b9a62-2140-4edb-b40a-61c95d9c3146", + "task_id": "air-quality", + "model": "oracle-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "95854287-bfed-486a-9aff-20d839999229", + "task_id": "air-quality", + "model": "oracle-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "ff1c9565-ffe8-4a70-adc2-b5215c3410ff", + "task_id": "market-news", + "model": "oracle-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "56a668ab-8a69-4277-8729-4a1717af668c", + "task_id": "market-news", + "model": "oracle-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "2fd403d8-a11c-41e1-9390-d076ca5dfdb0", + "task_id": "market-news", + "model": "oracle-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "8b3f5092-fa39-4426-a908-35e2af2c9312", + "task_id": "domain-intel", + "model": "oracle-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "d454cf08-3fbd-4a50-bb4a-9af533a5675d", + "task_id": "domain-intel", + "model": "oracle-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "d24fe5b1-5ffb-427b-a45a-37baffde5d36", + "task_id": "domain-intel", + "model": "oracle-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "66cecd06-6fce-4084-94a3-4997791e2303", + "task_id": "image-search", + "model": "oracle-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "ffdc3990-5991-4e84-bfb9-4997434ec7e8", + "task_id": "image-search", + "model": "oracle-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "8fc69d41-be93-424e-ae30-754fb957f63b", + "task_id": "image-search", + "model": "oracle-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "5cd557cc-6800-41f3-b397-f194865cc601", + "task_id": "market-holidays", + "model": "oracle-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "5c5c80cb-d46e-42c9-bbb7-c173a487851d", + "task_id": "market-holidays", + "model": "oracle-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "e0657299-6f28-4851-8554-1b4d50126bfb", + "task_id": "market-holidays", + "model": "oracle-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "53649f10-7f15-439b-b58e-c8e01a811995", + "task_id": "earthquake-catalog", + "model": "oracle-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "cdefdbdd-5005-472b-b3e0-d33e3ecc51c5", + "task_id": "earthquake-catalog", + "model": "oracle-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "929db9db-2ef4-466d-a80b-2e14af1f4c7c", + "task_id": "earthquake-catalog", + "model": "oracle-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "711a0f3a-7f40-4b3d-ab18-0fd661803757", + "task_id": "company-profile", + "model": "oracle-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "342d9249-b2e3-43e1-9db2-9d6584889277", + "task_id": "company-profile", + "model": "oracle-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "063b8a50-9c2c-41dd-89df-4a639c354c74", + "task_id": "company-profile", + "model": "oracle-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "34788c2f-c12d-46b8-9a88-d4285ab363de", + "task_id": "sports-standings", + "model": "oracle-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "1ae754a2-5fe1-4baa-85b8-e5c6e8664576", + "task_id": "sports-standings", + "model": "oracle-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + }, + { + "run_id": "ddfc24d1-5ac5-4861-b959-7013b1160520", + "task_id": "sports-standings", + "model": "oracle-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_valid": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "oracle", + "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", + "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", + "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", + "api_base_url": "https://qveris.ai/api/v1", + "discovery_limit": 10, + "execute": true + } + } + ] +} diff --git a/benchmarks/discover-call/results/README.md b/benchmarks/discover-call/results/README.md index e2228c7..a8c79b2 100644 --- a/benchmarks/discover-call/results/README.md +++ b/benchmarks/discover-call/results/README.md @@ -1,6 +1,83 @@ # Published benchmark results -## 2026-07-23 — `gpt-5.6-sol` +## 2026-07-23 — v3 comparison lanes + +The v3 baseline runs the deterministic Oracle and pinned-model lanes over the +same 18 tasks, three trials per task, Top 10 discovery limit, and real QVeris +calls. No failed trial was removed or selectively rerun. + +| Metric | Oracle | `gpt-5.6-sol` pinned model | +| --- | ---: | ---: | +| Runs | 54 | 54 | +| Completed and executed | 51 / 54 | 52 / 54 | +| Selection grounded | 94.44% | 100% | +| Inspection grounded | 94.44% | 100% | +| Required-parameter accuracy | 100% | 100% | +| Constraint accuracy | 94.44% | 83.33% | +| Call success among attempted calls | 100% (51 / 51) | 88.46% (46 / 52) | +| Non-empty result among attempted calls | 100% (51 / 51) | 88.46% (46 / 52) | +| Strict workflow success | 94.44% (51 / 54) | 72.22% (39 / 54) | +| Workflow success, 95% Wilson interval | 84.89%–98.09% | 59.11%–82.38% | + +The strict routing gap is **22.22 percentage points**: Oracle 94.44% minus +pinned model 72.22%. + +Shared reproducibility metadata: + +- toolkit revision: `5006c2a0789e2352944ffd87b1a154b96ec0566f` +- task set: `tasks/v3.jsonl` +- task-set SHA-256: + `d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866` +- API base URL: `https://qveris.ai/api/v1` +- discovery limit: `10` + +Oracle metadata: + +- model: `oracle-v1` +- lane: `oracle` +- adapter revision: + `5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1` + +Pinned-model metadata: + +- model: `gpt-5.6-sol` +- lane: `pinned-model` +- reasoning effort: `medium` +- Codex CLI: `0.144.1` +- adapter revision: + `5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium` + +Artifacts: + +- [`2026-07-23-oracle-v1-v3.runs.jsonl`](2026-07-23-oracle-v1-v3.runs.jsonl) +- [`2026-07-23-oracle-v1-v3.summary.json`](2026-07-23-oracle-v1-v3.summary.json) +- [`2026-07-23-gpt-5.6-sol-v3.runs.jsonl`](2026-07-23-gpt-5.6-sol-v3.runs.jsonl) +- [`2026-07-23-gpt-5.6-sol-v3.summary.json`](2026-07-23-gpt-5.6-sol-v3.summary.json) + +### v3 interpretation + +The Oracle's three strict failures are all `timezone-tokyo`: the fixed +discovery query returned no Top 10 capability that accepts a city or +coordinates. All 51 Oracle-selected calls succeeded and returned non-empty +results. + +The pinned model also missed Tokyo semantically in all three trials: it selected +a successful timezone-list capability with no Tokyo input. Six well-formed +calls returned `success: false`: three selected +`weather_api.ip_lookup.retrieve.v1.e095d904`, and three selected +`tradefeeds.compinfo.retrieve.v1.c2314765`. Two domain-intelligence +parameterization attempts were rejected because the isolated model session +tried to use a disabled external tool; the remaining trial called the selected +tool with `{}` and therefore missed the domain constraint. + +Three cryptocurrency calls succeeded and returned non-empty Bitcoin results via +CoinMarketCap's provider-specific numeric `id=1`, but the deterministic task +constraint expects a parameter value containing `BTC`; these remain strict +constraint misses rather than being silently reclassified after the run. This +known identifier-mapping limitation should be addressed in a future immutable +task-set revision. + +## 2026-07-23 — `gpt-5.6-sol` historical v2 baseline This is the first controlled result for the versioned `tasks/v2.jsonl` contract benchmark. It contains 18 tasks, three trials per task, and real QVeris calls. diff --git a/docs/en-US/discover-call-benchmark.md b/docs/en-US/discover-call-benchmark.md index 1625d47..e197640 100644 --- a/docs/en-US/discover-call-benchmark.md +++ b/docs/en-US/discover-call-benchmark.md @@ -56,29 +56,30 @@ provider error bodies must never be published. ## Published results -The first controlled result was run on 2026-07-23 with `gpt-5.6-sol`, medium -reasoning effort, Codex CLI 0.144.1, `tasks/v2.jsonl`, and three trials per task. -All 54 trials used real execution and all failures remain in the denominator. - -| Metric | Result | -| --- | ---: | -| Completed and executed | 50 / 54 | -| Selection grounded | 100% | -| Inspection grounded | 100% | -| Required-parameter accuracy | 100% | -| Constraint accuracy | 75.93% | -| Call success among attempted calls | 88.00% (44 / 50) | -| Strict workflow success | 64.81% (35 / 54) | -| Workflow success, 95% Wilson interval | 51.48%–76.18% | - -Call success and workflow success have different denominators: strict workflow -success includes all 54 trials and also requires every preceding scoring -component to pass. Four trials failed during model parameterization and six -well-formed calls returned `success: false`. Additional constraint misses -include a valid combined currency pair that the current v2 alias scorer does -not split, URL-encoded news queries, and timezone-list selections that could not -accept the requested city. +The v3 comparison was run on 2026-07-23 with three trials per task and real +execution. The deterministic Oracle measures the current fixed-query platform +ceiling; the pinned model uses `gpt-5.6-sol`, medium reasoning effort, and Codex +CLI 0.144.1. + +| Metric | Oracle | Pinned model | +| --- | ---: | ---: | +| Completed and executed | 51 / 54 | 52 / 54 | +| Constraint accuracy | 94.44% | 83.33% | +| Call and non-empty-result success | 100% (51 / 51) | 88.46% (46 / 52) | +| Strict workflow success | 94.44% (51 / 54) | 72.22% (39 / 54) | +| Workflow success, 95% Wilson interval | 84.89%–98.09% | 59.11%–82.38% | + +The strict routing gap is 22.22 percentage points. The Oracle's only failures +were the three Tokyo-timezone trials, where the fixed query returned no Top 10 +capability accepting a city or coordinates. The pinned model had the same three +semantic misses, six selected-tool execution failures, two safely classified +disabled-tool attempts, and additional strict constraint misses. Three +successful Bitcoin calls used CoinMarketCap's numeric `id=1`; because the v3 +constraint expects a value containing `BTC`, they remain known identifier-mapping +false negatives rather than being reclassified after the run. See the [result notes, immutable revisions, raw JSONL, and generated summary](../../benchmarks/discover-call/results/README.md). The synthetic scorer -fixture remains test-only and is not a product-performance claim. +fixture remains test-only and is not a product-performance claim. The original +v2 result remains preserved there as a historical baseline under its original +scoring contract. diff --git a/docs/zh-CN/discover-call-benchmark.md b/docs/zh-CN/discover-call-benchmark.md index 04585f6..1302840 100644 --- a/docs/zh-CN/discover-call-benchmark.md +++ b/docs/zh-CN/discover-call-benchmark.md @@ -41,25 +41,23 @@ Oracle 与模型的严格工作流成功率之差就是 routing gap。还需要 ## 已发布结果 -首次受控结果于 2026-07-23 运行，使用 `gpt-5.6-sol`、medium reasoning effort、 -Codex CLI 0.144.1、`tasks/v2.jsonl`，每个任务执行三个 trial。54 个 trial 均启用了真实执行， -所有失败均保留在分母中。 - -| 指标 | 结果 | -| --- | ---: | -| 完成参数化并执行 | 50 / 54 | -| 选择来自 discover | 100% | -| inspect 确认同一能力 | 100% | -| 必填参数完整率 | 100% | -| 任务约束准确率 | 75.93% | -| 已尝试 call 的成功率 | 88.00%（44 / 50） | -| 严格工作流成功率 | 64.81%（35 / 54） | -| 工作流成功率 95% Wilson 区间 | 51.48%–76.18% | - -call 成功率和工作流成功率的分母不同：严格工作流成功率包含全部 54 个 trial，并要求前置评分项 -全部通过。4 个 trial 在模型参数化阶段失败，另有 6 个参数格式有效的 call 返回 -`success: false`。其他约束未命中包括：当前 v2 别名评分器没有把有效的组合货币对拆成基础/报价 -货币、新闻查询值经过 URL 编码，以及选择了无法接收目标城市的时区列表能力。 +v3 对比基线于 2026-07-23 运行，每个任务执行三个 trial，并启用真实执行。确定性的 Oracle +用于衡量固定查询下的当前平台上限；pinned model 使用 `gpt-5.6-sol`、medium reasoning effort +和 Codex CLI 0.144.1。 + +| 指标 | Oracle | Pinned model | +| --- | ---: | ---: | +| 完成参数化并执行 | 51 / 54 | 52 / 54 | +| 任务约束准确率 | 94.44% | 83.33% | +| call 成功且结果非空 | 100%（51 / 51） | 88.46%（46 / 52） | +| 严格工作流成功率 | 94.44%（51 / 54） | 72.22%（39 / 54） | +| 工作流成功率 95% Wilson 区间 | 84.89%–98.09% | 59.11%–82.38% | + +严格 routing gap 为 22.22 个百分点。Oracle 的三个失败全部来自东京时区任务：固定查询的 Top 10 +中没有能够接收城市或坐标的能力。Pinned model 同样出现三个时区语义未命中，另有六次所选工具 +执行失败、两次经过安全分类的禁用工具尝试，以及其他严格约束未命中。三次 Bitcoin 调用通过 +CoinMarketCap 的数字标识 `id=1` 成功返回非空结果，但 v3 约束要求参数值包含 `BTC`，因此仍作为 +已知的标识映射假阴性保留，未在运行后静默改分。 完整的[结果说明、不可变 revision、原始 JSONL 与生成汇总](../../benchmarks/discover-call/results/README.md) -均已公开。scorer fixture 仍只用于测试，不代表产品性能。 +均已公开。原始 v2 结果按其原评分契约保留为历史基线。scorer fixture 仍只用于测试，不代表产品性能。 From deeda797e50a0dae649bf91d7503d09ec497b324 Mon Sep 17 00:00:00 2001 From: Linfang Wang Date: Thu, 23 Jul 2026 19:13:08 +0800 Subject: [PATCH 08/19] fix(benchmark): harden public quality baseline --- .../discover-call/PUBLICATION_POLICY.md | 44 + benchmarks/discover-call/README.md | 102 +- benchmarks/discover-call/adapters/oracle.mjs | 57 +- .../discover-call/adapters/reference.mjs | 58 ++ benchmarks/discover-call/package.json | 1 + .../discover-call/publication-policy.json | 47 + .../2026-07-23-gpt-5.6-sol-v2.runs.jsonl | 108 +-- .../2026-07-23-gpt-5.6-sol-v2.summary.json | 823 ++++++++++++++-- .../2026-07-23-gpt-5.6-sol-v3.runs.jsonl | 108 +-- .../2026-07-23-gpt-5.6-sol-v3.summary.json | 876 ++++++++++++++---- .../2026-07-23-oracle-v1-v3.runs.jsonl | 108 +-- .../2026-07-23-oracle-v1-v3.summary.json | 876 ++++++++++++++---- benchmarks/discover-call/results/README.md | 59 +- benchmarks/discover-call/src/api.mjs | 19 + benchmarks/discover-call/src/harness.mjs | 40 +- benchmarks/discover-call/src/publication.mjs | 180 ++++ benchmarks/discover-call/src/publish.mjs | 62 ++ benchmarks/discover-call/src/run.mjs | 57 +- benchmarks/discover-call/src/scoring.mjs | 84 +- benchmarks/discover-call/src/validate.mjs | 22 +- benchmarks/discover-call/tasks/v4.jsonl | 18 + .../discover-call/test/adapter.test.mjs | 8 +- benchmarks/discover-call/test/api.test.mjs | 16 + .../discover-call/test/harness.test.mjs | 21 +- .../discover-call/test/publication.test.mjs | 82 ++ .../discover-call/test/scoring.test.mjs | 42 +- 26 files changed, 3191 insertions(+), 727 deletions(-) create mode 100644 benchmarks/discover-call/PUBLICATION_POLICY.md create mode 100644 benchmarks/discover-call/adapters/reference.mjs create mode 100644 benchmarks/discover-call/publication-policy.json create mode 100644 benchmarks/discover-call/src/publication.mjs create mode 100644 benchmarks/discover-call/src/publish.mjs create mode 100644 benchmarks/discover-call/tasks/v4.jsonl create mode 100644 benchmarks/discover-call/test/publication.test.mjs diff --git a/benchmarks/discover-call/PUBLICATION_POLICY.md b/benchmarks/discover-call/PUBLICATION_POLICY.md new file mode 100644 index 0000000..b1ee7c8 --- /dev/null +++ b/benchmarks/discover-call/PUBLICATION_POLICY.md @@ -0,0 +1,44 @@ +# Benchmark catalog and artifact publication policy + +The public repository documents what QVeris can do without publishing account, +connection, execution, or bulk catalog-operational data. + +## Visibility classes + +- `public`: approved provider/tool metadata may be searched and referenced in + public documentation and benchmark examples. +- `unlisted`: metadata is available only through an explicit link, invitation, + or allowlist and must not appear in public search or benchmark artifacts. +- `private`: metadata is visible only to its tenant or owner. +- `quarantined` / `deprecated`: metadata is excluded from public recommendation + and new benchmark baselines. + +Until the production discovery contract exposes an explicit visibility field, +catalog entries default to private for bulk artifact publication. + +## Public metadata + +For an explicitly approved public tool, public surfaces may include its stable +tool ID, provider name, capability description, categories, version, input and +output schema, authentication type, required scopes, documentation, deprecation +status, and safe examples. + +Public reliability data must use minimum sample thresholds and rounded bands. +Exact ranking features, weights, call counts, provider costs, tenant eligibility, +and account-specific availability remain internal. + +## Never public + +Public artifacts must not contain API keys, OAuth tokens, connection/account +identifiers, `search_id`, `execution_id`, credit balances, raw provider errors, +raw tool results, private prompts, tenant-private tools, or the unfiltered +ordered discovery catalog. + +The publication command enforces `publication-policy.json`. It replaces the +ordered discovery list with its count and SHA-256 digest, preserves grounded +selection as an attestation, and refuses to publish a selected tool unless its +ID is explicitly approved. + +Provider authorization and namespace ownership must be confirmed before adding a +tool ID to the public allowlist. Removing an approval requires regenerating all +affected public artifacts. diff --git a/benchmarks/discover-call/README.md b/benchmarks/discover-call/README.md index 63970f1..1166372 100644 --- a/benchmarks/discover-call/README.md +++ b/benchmarks/discover-call/README.md @@ -34,7 +34,8 @@ The scorer reports these metrics separately: - **required-parameter accuracy**: fraction of required schema parameters supplied; - **constraint accuracy**: fraction of task facts represented through accepted parameter aliases; - **call success**: successful real executions among attempted calls; -- **result validity**: successful calls returned a non-empty result; +- **result non-empty**: successful calls returned a structurally non-empty + result; this does not assert semantic correctness; - **workflow success**: all preceding checks equal 100%, the real call succeeds, and its result is non-empty. @@ -43,9 +44,10 @@ failure, matching the public clients' transient-failure behavior. The summary reports failures by stage; all exhausted API failures remain in the strict workflow-success denominator. -The summary includes a 95% Wilson interval for workflow success. Dry runs never -count as workflow success, so a published success rate cannot be produced -without exercising the full workflow. +The summary includes a deterministic 95% task-cluster bootstrap interval for +workflow success. Tasks, rather than individual trials, are resampled so three +trials of one task are not treated as three independent task draws. Dry runs +never count as workflow success. ## Task sets @@ -60,11 +62,14 @@ Task sets are versioned and immutable once referenced by a published result: multiple parameterizable capabilities within the default discovery limit, and constraint aliases were taken from the actual parameter names of those capabilities. -- `tasks/v3.jsonl` — the 18-task comparison set for lane-based baselines. It - preserves the v2 prompts and discovery queries, adds explicit Oracle - candidates, recognizes combined parameters such as `symbol=USD/EUR`, and - applies URL decoding only where a task opts into it. An empty Oracle candidate - list records a verified discovery-coverage gap; it is not silently excluded. +- `tasks/v3.jsonl` — a diagnostic comparison set. It added deterministic + reference candidates and task-specific normalization, but its crypto + constraint did not recognize CoinMarketCap's provider-specific Bitcoin + identifier `id=1`. +- `tasks/v4.jsonl` — the current immutable quality-baseline contract. It copies + v3 and explicitly accepts `id=1` only for the crypto task's `id` alias. + This task-versioned mapping fixes the known v3 scoring false negative without + retroactively changing v3. New tasks land as a new `tasks/vN.jsonl` revision, never by editing an existing file, and must still score deterministically at the contract level. @@ -73,15 +78,18 @@ different task-set hashes. ## Run -Set `QVERIS_API_KEY`, then provide an adapter executable and an immutable model -identifier. The adapter command is spawned directly without shell parsing. +Set `QVERIS_API_KEY`, then provide an adapter executable and a model identifier. +Use `--lane pinned-model` only when the provider exposes a verifiable immutable +model revision and record it with `--model-revision`. Otherwise use +`configured-model` or `current-model` and record `unreported`. +The adapter command is spawned directly without shell parsing. The harness removes `QVERIS_API_KEY` from the adapter environment; adapters must use separately named credentials for their model provider. ```bash node src/run.mjs \ --model provider/model-version \ - --lane pinned-model \ + --lane configured-model \ --adapter node \ --adapter-arg /absolute/path/to/model-adapter.mjs \ --adapter-revision adapter-git-sha-or-config-hash \ @@ -97,47 +105,60 @@ not publish dry-run records as benchmark results. Use the lanes together; none is a substitute for the others: -- `oracle` measures the current platform ceiling for the fixed discovery query. - The deterministic adapter selects only a configured candidate that appears in - the current Top 10 and supplies fixed valid parameters. A task with no suitable - returned candidate remains a strict failure. +- `reference` is a curated reference route for the fixed discovery query. The + deterministic adapter selects only a configured candidate that appears in the + observed Top 10 and supplies fixed parameters. It represents only those + curated candidates; a task with no suitable returned candidate remains a + strict failure. +- `configured-model` measures a named model plus recorded CLI, reasoning, adapter, + and task-set configuration when an immutable provider snapshot is unavailable. - `pinned-model` measures longitudinal changes with one immutable model, - reasoning configuration, CLI, adapter revision, and task set. + provider revision, reasoning configuration, CLI, adapter revision, and task + set. The runner rejects this lane without `--model-revision`. - `current-model` measures the currently recommended model under the same task contract. - `model` is retained for backward-compatible records that predate explicit lanes. -The primary routing gap is `oracle workflow success - model workflow success`. +The strict benchmark gap is +`curated reference route workflow success - model workflow success`. Compare component metrics and failure reasons alongside it so discovery -coverage, model routing, parameter construction, execution, and result validity -are not collapsed into one diagnosis. +coverage, model routing, parameter construction, execution, and result non-emptiness +are not collapsed into one diagnosis. Here, result non-emptiness is structural, +not a semantic-quality judgment. -Run the deterministic Oracle lane with the same task file supplied both to the +Run the curated reference route with the same task file supplied both to the harness and adapter: ```bash node src/run.mjs \ - --model oracle-v1 \ - --lane oracle \ + --model reference-v1 \ + --model-revision deterministic-reference-v1 \ + --lane reference \ --adapter node \ - --adapter-arg "$PWD/adapters/oracle.mjs" \ - --adapter-arg "$PWD/tasks/v3.jsonl" \ - --adapter-revision toolkit-git-sha/oracle-v1 \ - --tasks tasks/v3.jsonl \ + --adapter-arg "$PWD/adapters/reference.mjs" \ + --adapter-arg "$PWD/tasks/v4.jsonl" \ + --adapter-revision toolkit-git-sha/reference-v1 \ + --tasks tasks/v4.jsonl \ --trials 3 \ --execute \ - --output runs/oracle-v1.jsonl + --output runs/reference-v1.raw.jsonl ``` -Score the records: +Generate public artifacts from raw records: ```bash -node src/score.mjs \ - --runs runs/model-version.jsonl \ - --output runs/model-version.summary.json +npm run publish -- \ + --tasks tasks/v4.jsonl \ + --runs runs/model-version.raw.jsonl \ + --output-runs results/model-version-v4.runs.jsonl \ + --output-summary results/model-version-v4.summary.json ``` +Keep raw records outside the public repository. Publication removes operational +identifiers and the ordered discovery catalog, adds count/digest attestations, +and rejects selected tools that are not explicitly approved. + Validate the checked-in task set, fixtures, and scorer: ```bash @@ -234,19 +255,28 @@ The adapter uses the CLI's existing authentication and never receives An official result must include: -- the raw JSONL records and generated summary; +- sanitized public JSONL records and generated summary; raw records remain + private; - the toolkit commit SHA and the exact, unchanged task-set file used (e.g. `tasks/v2.jsonl`); -- an immutable model version, adapter source revision, trial count, and date; -- the recorded API base URL, discovery limit, and task-set SHA-256; +- the model identifier and provider revision (or explicit `unreported`), adapter + source revision, runtime, trial count, and date; +- the recorded API base URL, observed API revision, catalog revision when + reported, catalog-observation SHA-256, discovery limit, and task-set SHA-256; - at least three trials per task; - `--execute` records for every reported workflow-success denominator; - failures retained in the denominator; no selective reruns or task removal; -- no API keys, access tokens, private prompts, or raw provider error bodies. +- no API keys, access tokens, private prompts, raw provider error bodies, + execution/search/connection identifiers, or unfiltered catalog results. The scorer rejects missing tasks, unequal trial counts, duplicate trials, and non-consecutive trial numbering for every model. It also rejects duplicate run IDs and aggregation across different adapter, toolkit, task-set, endpoint, discovery-limit, or execution settings. +The catalog and artifact visibility boundary is defined in +[`PUBLICATION_POLICY.md`](PUBLICATION_POLICY.md). Until discovery reports +per-tool visibility, bulk catalog content defaults to private; only approved +selected tool IDs may appear in public benchmark records. + The checked-in fixture validates the scorer only. It is synthetic and must not be presented as product performance. diff --git a/benchmarks/discover-call/adapters/oracle.mjs b/benchmarks/discover-call/adapters/oracle.mjs index d4a1e82..76abf8e 100644 --- a/benchmarks/discover-call/adapters/oracle.mjs +++ b/benchmarks/discover-call/adapters/oracle.mjs @@ -1,57 +1,4 @@ #!/usr/bin/env node -import { readFile } from 'node:fs/promises'; - -try { - const tasksPath = parseTasksPath(process.argv.slice(2)); - const tasks = await readTasks(tasksPath); - const taskById = new Map(tasks.map((task) => [task.id, task])); - - let input = ''; - for await (const chunk of process.stdin) input += chunk; - const payload = JSON.parse(input); - const task = taskById.get(payload.input?.task_id); - if (!task?.oracle || !Array.isArray(task.oracle.candidates)) throw oracleFailure('missing_oracle'); - - if (payload.stage === 'select') { - const discovered = new Set( - Array.isArray(payload.input?.discovery?.results) - ? payload.input.discovery.results.map((tool) => tool?.tool_id).filter(Boolean) - : [], - ); - const candidate = task.oracle.candidates.find((item) => discovered.has(item.tool_id)); - process.stdout.write(JSON.stringify({ tool_id: candidate?.tool_id ?? null })); - } else if (payload.stage === 'parameterize') { - const selectedToolId = payload.input?.selected_tool?.tool_id; - const candidate = task.oracle.candidates.find((item) => item.tool_id === selectedToolId); - if (!candidate) throw oracleFailure('missing_oracle_candidate'); - process.stdout.write(JSON.stringify({ parameters: candidate.parameters })); - } else { - throw oracleFailure('unsupported_stage'); - } -} catch (error) { - process.stderr.write(`QVERIS_BENCHMARK_ADAPTER_ERROR=${error?.oracleCode || 'missing_oracle'}\n`); - process.exitCode = 2; -} - -async function readTasks(path) { - const text = await readFile(path, 'utf8'); - return text - .split(/\r?\n/) - .map((line) => line.trim()) - .filter((line) => line && !line.startsWith('#')) - .map((line) => JSON.parse(line)); -} - -function parseTasksPath(argv) { - if (argv[0] && !argv[0].startsWith('--')) return argv[0]; - const index = argv.indexOf('--tasks'); - if (index === -1 || !argv[index + 1]) throw oracleFailure('missing_oracle'); - return argv[index + 1]; -} - -function oracleFailure(code) { - const error = new Error(code); - error.oracleCode = code; - return error; -} +// Backward-compatible entry point for historical v3 commands. +await import('./reference.mjs'); diff --git a/benchmarks/discover-call/adapters/reference.mjs b/benchmarks/discover-call/adapters/reference.mjs new file mode 100644 index 0000000..3ac5e4b --- /dev/null +++ b/benchmarks/discover-call/adapters/reference.mjs @@ -0,0 +1,58 @@ +#!/usr/bin/env node + +import { readFile } from 'node:fs/promises'; + +try { + const tasksPath = parseTasksPath(process.argv.slice(2)); + const tasks = await readTasks(tasksPath); + const taskById = new Map(tasks.map((task) => [task.id, task])); + + let input = ''; + for await (const chunk of process.stdin) input += chunk; + const payload = JSON.parse(input); + const task = taskById.get(payload.input?.task_id); + const reference = task?.reference ?? task?.oracle; + if (!reference || !Array.isArray(reference.candidates)) throw referenceFailure('missing_reference'); + + if (payload.stage === 'select') { + const discovered = new Set( + Array.isArray(payload.input?.discovery?.results) + ? payload.input.discovery.results.map((tool) => tool?.tool_id).filter(Boolean) + : [], + ); + const candidate = reference.candidates.find((item) => discovered.has(item.tool_id)); + process.stdout.write(JSON.stringify({ tool_id: candidate?.tool_id ?? null })); + } else if (payload.stage === 'parameterize') { + const selectedToolId = payload.input?.selected_tool?.tool_id; + const candidate = reference.candidates.find((item) => item.tool_id === selectedToolId); + if (!candidate) throw referenceFailure('missing_reference_candidate'); + process.stdout.write(JSON.stringify({ parameters: candidate.parameters })); + } else { + throw referenceFailure('unsupported_stage'); + } +} catch (error) { + process.stderr.write(`QVERIS_BENCHMARK_ADAPTER_ERROR=${error?.referenceCode || 'missing_reference'}\n`); + process.exitCode = 2; +} + +async function readTasks(path) { + const text = await readFile(path, 'utf8'); + return text + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith('#')) + .map((line) => JSON.parse(line)); +} + +function parseTasksPath(argv) { + if (argv[0] && !argv[0].startsWith('--')) return argv[0]; + const index = argv.indexOf('--tasks'); + if (index === -1 || !argv[index + 1]) throw referenceFailure('missing_reference'); + return argv[index + 1]; +} + +function referenceFailure(code) { + const error = new Error(code); + error.referenceCode = code; + return error; +} diff --git a/benchmarks/discover-call/package.json b/benchmarks/discover-call/package.json index 4390e73..7b2af1c 100644 --- a/benchmarks/discover-call/package.json +++ b/benchmarks/discover-call/package.json @@ -9,6 +9,7 @@ "scripts": { "test": "node --test", "validate": "node src/validate.mjs", + "publish": "node src/publish.mjs", "score:fixture": "node src/score.mjs --runs fixtures/sample-runs.jsonl" } } diff --git a/benchmarks/discover-call/publication-policy.json b/benchmarks/discover-call/publication-policy.json new file mode 100644 index 0000000..e88b774 --- /dev/null +++ b/benchmarks/discover-call/publication-policy.json @@ -0,0 +1,47 @@ +{ + "schema_version": 1, + "policy_id": "public-artifact-v1", + "catalog_visibility_default": "private", + "publish_parameters": true, + "legacy_lane_rewrites": { + "oracle": "reference", + "pinned-model": "configured-model" + }, + "approved_selected_tool_ids": [ + "amap_webservice.geocode.geo.retrieve.v3", + "api-football.standings.list.v3.76f7703e", + "api_sports.standings.list.v1.39568ba4", + "api_sports.standings.list.v1.98a153bd", + "api_sports.timezone.retrieve.v1.e993615d", + "binance.ticker.price.list.v3.8675eca0", + "brave_search.images.search.list.v1", + "cloudsway.smart_search_overseas", + "coinmarketcap.tools.price-conversion.execute.v1.2b99a4df", + "dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8", + "financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2", + "finnhub.market.news.list.v1", + "finnhub.stock.marketholiday.retrieve.v1", + "mcp_gildata.uscompanybasicinfo.v1", + "newsapi.everything.retrieve.v2.e77ab1df", + "openweathermap.air_pollution.current.v2_5", + "qveris_academic.paper_aggregation_v1", + "tradefeeds.compinfo.retrieve.v1.c2314765", + "twelvedata.currencyconversion.retrieve.v1.60fe48d4", + "twelvedata.quote.retrieve.v1.affbefe3", + "usgs_earthquake_catalog.event.events.list.v1", + "virustotal.domains.report.retrieve.v3.c5459740", + "virustotal.domain_reports.retrieve.v3.c5459740", + "virustotal.ip.info.retrieve.v3.ab34b0d1", + "virustotal.ip.report.retrieve.v3.ab34b0d1", + "visualcrossing.timeline.retrieve.v1", + "weather_api.ip_lookup.retrieve.v1.e095d904" + ], + "forbidden_fields": [ + "execution_id", + "search_id", + "connection_id", + "remaining_credits", + "result_tool_ids", + "returned_tool_ids" + ] +} diff --git a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.runs.jsonl b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.runs.jsonl index e36983d..a3a1fe3 100644 --- a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.runs.jsonl +++ b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.runs.jsonl @@ -1,54 +1,54 @@ -{"schema_version":1,"benchmark_version":"v1","run_id":"ac88304b-0662-4d2b-bb4b-778a65d4d84d","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"weather-london","trial":1,"started_at":"2026-07-23T06:26:39.032Z","status":"completed","discovery":{"result_tool_ids":["weather_gov.gridpoints.forecast.retrieve.v3.29bbec80","weather_gov.gridpoints.forecast.retrieve.v1.40a9a3ce","weather_gov.gridpoints.forecast.hourly.retrieve.v3.808ea240","amap_webservice.weather.weatherinfo.retrieve.v3","weather_gov.icons.retrieve.v3.ad0b4d80","weather_gov.gridpoints.retrieve.v3.a0c89365","weather_gov.products.retrieve.v1.18657145","openweathermap.forecast.retrieve.v2.5","weather_gov.gridpoints.retrieve.v3.2cf46664","visualcrossing.timeline.retrieve.v1"]},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"returned_tool_ids":["visualcrossing.timeline.retrieve.v1"],"required_parameters":["location"]},"parameters":{"location":"London"},"call":{"attempted":true,"success":true,"execution_id":"2ecc61c2-a8bd-4c5f-9b76-efdb23b44641"},"finished_at":"2026-07-23T06:27:00.228Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"5b911181-4d6c-472a-a2a0-4cb12e433f06","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"weather-london","trial":2,"started_at":"2026-07-23T06:27:00.228Z","status":"completed","discovery":{"result_tool_ids":["weather_gov.gridpoints.forecast.retrieve.v3.29bbec80","weather_gov.gridpoints.forecast.retrieve.v1.40a9a3ce","weather_gov.gridpoints.forecast.hourly.retrieve.v3.808ea240","amap_webservice.weather.weatherinfo.retrieve.v3","weather_gov.icons.retrieve.v3.ad0b4d80","weather_gov.gridpoints.retrieve.v3.a0c89365","weather_gov.products.retrieve.v1.18657145","openweathermap.forecast.retrieve.v2.5","weather_gov.gridpoints.retrieve.v3.2cf46664","visualcrossing.timeline.retrieve.v1"]},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"returned_tool_ids":["visualcrossing.timeline.retrieve.v1"],"required_parameters":["location"]},"parameters":{"location":"London"},"call":{"attempted":true,"success":true,"execution_id":"435720e5-bf11-47df-881b-c5056e26b6ad"},"finished_at":"2026-07-23T06:27:23.671Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"a8268395-01a9-413b-b2af-ba30d2e71f9a","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"weather-london","trial":3,"started_at":"2026-07-23T06:27:23.671Z","status":"completed","discovery":{"result_tool_ids":["weather_gov.gridpoints.forecast.retrieve.v3.29bbec80","weather_gov.gridpoints.forecast.retrieve.v1.40a9a3ce","weather_gov.gridpoints.forecast.hourly.retrieve.v3.808ea240","amap_webservice.weather.weatherinfo.retrieve.v3","weather_gov.icons.retrieve.v3.ad0b4d80","weather_gov.gridpoints.retrieve.v3.a0c89365","weather_gov.products.retrieve.v1.18657145","openweathermap.forecast.retrieve.v2.5","weather_gov.gridpoints.retrieve.v3.2cf46664","visualcrossing.timeline.retrieve.v1"]},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"returned_tool_ids":["visualcrossing.timeline.retrieve.v1"],"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"execution_id":"01fdd003-d088-4dfc-8232-444e096d9008"},"finished_at":"2026-07-23T06:27:50.399Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"4e925856-364c-4ca7-b769-c8d4d43e85c2","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-23T06:27:50.399Z","status":"completed","discovery":{"result_tool_ids":["eodhd.live_v2.us_quote_delayed.retrieve.v1.f0e13d45","twelvedata.quote.retrieve.v1.affbefe3","massive_stocks.options.reference.conditions.list.v3.41a82e07","alphavantage.realtime_bulk_quotes.retrieve.v1","fiu_mcp_server.postv1stockquoteextend.create.v2.a30e86e7","hangseng_polysource.a_shares_live_quote.query.v2.10fe0581","mcp_gildata.usstockdailyquotes.v1","massive_stocks.stocks.marketstatus.retrieve.v1.3045ae6e","fiu_mcp_server.postv3stockquoteextend.create.v2.b35a7f02","financialmodelingprep.cusip.search.v1.ad19189f"]},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"returned_tool_ids":["twelvedata.quote.retrieve.v1.affbefe3"],"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"0229f37b-abf6-411e-927c-91c372c00d76"},"finished_at":"2026-07-23T06:28:19.862Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"9c2b558d-b803-4299-9b8d-7778e56d0fcb","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-23T06:28:19.862Z","status":"completed","discovery":{"result_tool_ids":["eodhd.live_v2.us_quote_delayed.retrieve.v1.f0e13d45","twelvedata.quote.retrieve.v1.affbefe3","massive_stocks.options.reference.conditions.list.v3.41a82e07","alphavantage.realtime_bulk_quotes.retrieve.v1","fiu_mcp_server.postv1stockquoteextend.create.v2.a30e86e7","hangseng_polysource.a_shares_live_quote.query.v2.10fe0581","mcp_gildata.usstockdailyquotes.v1","massive_stocks.stocks.marketstatus.retrieve.v1.3045ae6e","fiu_mcp_server.postv3stockquoteextend.create.v2.b35a7f02","financialmodelingprep.cusip.search.v1.ad19189f"]},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"returned_tool_ids":["twelvedata.quote.retrieve.v1.affbefe3"],"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"0c743985-72c3-4029-8bfe-adc9d90d4cd1"},"finished_at":"2026-07-23T06:28:47.286Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"6762a6af-2002-4349-9239-89c6c62a597b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-23T06:28:47.286Z","status":"completed","discovery":{"result_tool_ids":["eodhd.live_v2.us_quote_delayed.retrieve.v1.f0e13d45","twelvedata.quote.retrieve.v1.affbefe3","massive_stocks.options.reference.conditions.list.v3.41a82e07","alphavantage.realtime_bulk_quotes.retrieve.v1","fiu_mcp_server.postv1stockquoteextend.create.v2.a30e86e7","hangseng_polysource.a_shares_live_quote.query.v2.10fe0581","mcp_gildata.usstockdailyquotes.v1","massive_stocks.stocks.marketstatus.retrieve.v1.3045ae6e","fiu_mcp_server.postv3stockquoteextend.create.v2.b35a7f02","financialmodelingprep.cusip.search.v1.ad19189f"]},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"returned_tool_ids":["twelvedata.quote.retrieve.v1.affbefe3"],"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"d04c5819-306b-4a68-8464-68807a24fa3f"},"finished_at":"2026-07-23T06:29:13.981Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"aa24674f-2a54-49ce-9652-95a7afc5a139","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-23T06:29:13.981Z","status":"completed","discovery":{"result_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4","coinmarketcap.tools.priceconversion.retrieve.v2.24e991d9","twelvedata.exchangerate.retrieve.v1.9eeb3b0d","bybitp2p.pay.fx.convert.v5.9221b0c4","coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","coinmarketcap.tools.priceconversion.retrieve.v2.52e4cb2d","coingecko_pro.exchangerates.retrieve.v3.0f6133ea","bybitp2p.payment.fx.convert.v5.7883b755","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d","okx_api_v5.market.exchange-rate.retrieve.v5.0d3fd1c2"]},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"returned_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4"],"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":100,"format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"0da4d9b1-d29f-47bd-b44f-d0c1b56c1588"},"finished_at":"2026-07-23T06:29:40.942Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"d038e88d-be8e-4d2e-810c-ec4f155b6871","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-23T06:29:40.943Z","status":"completed","discovery":{"result_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4","coinmarketcap.tools.priceconversion.retrieve.v2.24e991d9","twelvedata.exchangerate.retrieve.v1.9eeb3b0d","bybitp2p.pay.fx.convert.v5.9221b0c4","coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","coinmarketcap.tools.priceconversion.retrieve.v2.52e4cb2d","coingecko_pro.exchangerates.retrieve.v3.0f6133ea","bybitp2p.payment.fx.convert.v5.7883b755","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d","okx_api_v5.market.exchange-rate.retrieve.v5.0d3fd1c2"]},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"returned_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4"],"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":100,"format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"60b1956f-2bec-40f6-a5f8-5b8d4d263a9f"},"finished_at":"2026-07-23T06:30:10.765Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"c54fe3bb-0b0a-4758-beec-b7a5ce2bb540","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-23T06:30:10.765Z","status":"completed","discovery":{"result_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4","coinmarketcap.tools.priceconversion.retrieve.v2.24e991d9","twelvedata.exchangerate.retrieve.v1.9eeb3b0d","bybitp2p.pay.fx.convert.v5.9221b0c4","coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","coinmarketcap.tools.priceconversion.retrieve.v2.52e4cb2d","coingecko_pro.exchangerates.retrieve.v3.0f6133ea","bybitp2p.payment.fx.convert.v5.7883b755","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d","okx_api_v5.market.exchange-rate.retrieve.v5.0d3fd1c2"]},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"returned_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4"],"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"62073171-a903-4900-b9b5-084eca6eeb40"},"finished_at":"2026-07-23T06:30:38.654Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"6bede096-05fb-47c7-a2eb-5a2f12dad17d","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"news-ai","trial":1,"started_at":"2026-07-23T06:30:38.654Z","status":"completed","discovery":{"result_tool_ids":["x_developer.2.news.search.retrieve.v1.19b3e678","newsdata.news.search.v1.b65ccc56","newsapi.everything.retrieve.v2.e77ab1df","qveris_finance.finance_news_aggregation_v1","x_developer.2.tweets.search.recent.retrieve.v2.f424c8f9","newsapi.top_headlines.list.v2.38face94","linkup.search.v1","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic","jina_search.rerank.create.v2026.23dc486d","brave_search.news.search.list.v1.6ab52eaa"]},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"returned_tool_ids":["newsapi.everything.retrieve.v2.e77ab1df"],"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20%22generative%20AI%22","from":"2026-07-16","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"execution_id":"49aa189a-d829-4250-90af-8eb1d560f478"},"finished_at":"2026-07-23T06:31:10.595Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"44918a28-652b-430d-ac7e-53daadb0e69b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"news-ai","trial":2,"started_at":"2026-07-23T06:31:10.595Z","status":"completed","discovery":{"result_tool_ids":["x_developer.2.news.search.retrieve.v1.19b3e678","newsdata.news.search.v1.b65ccc56","newsapi.everything.retrieve.v2.e77ab1df","qveris_finance.finance_news_aggregation_v1","x_developer.2.tweets.search.recent.retrieve.v2.f424c8f9","newsapi.top_headlines.list.v2.38face94","linkup.search.v1","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic","jina_search.rerank.create.v2026.23dc486d","brave_search.news.search.list.v1.6ab52eaa"]},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"returned_tool_ids":["newsapi.everything.retrieve.v2.e77ab1df"],"required_parameters":["q"]},"parameters":{"q":"%28%22artificial%20intelligence%22%20OR%20%22generative%20AI%22%20OR%20OpenAI%20OR%20Anthropic%29","from":"2026-07-16T00:00:00","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"execution_id":"6acf3bf4-4c36-4b8b-82a9-3eddbb04ed86"},"finished_at":"2026-07-23T06:31:44.443Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"a306ce69-4682-4778-a8e5-47b0c598e895","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"news-ai","trial":3,"started_at":"2026-07-23T06:31:44.443Z","status":"completed","discovery":{"result_tool_ids":["x_developer.2.news.search.retrieve.v1.19b3e678","newsdata.news.search.v1.b65ccc56","x_developer.2.tweets.search.recent.retrieve.v2.f424c8f9","qveris_finance.finance_news_aggregation_v1","linkup.search.v1","newsapi.everything.retrieve.v2.e77ab1df","jina_search.rerank.create.v2026.23dc486d","newsapi.top_headlines.list.v2.38face94","brave_search.news.search.list.v1.6ab52eaa","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic"]},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"returned_tool_ids":["newsapi.everything.retrieve.v2.e77ab1df"],"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20%22generative%20AI%22","from":"2026-07-16","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"execution_id":"3e7f4c5d-b285-4304-be83-2973fb06448d"},"finished_at":"2026-07-23T06:32:18.995Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"9377cc54-be7e-42b4-bf14-f4d14d013c54","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T06:32:18.995Z","status":"completed","discovery":{"result_tool_ids":["semantic_scholar.paper.search.bulk.execute.v1.239dfd66","qveris_academic.paper_aggregation_v1","semantic_scholar.paper.search.bulk.retrieve.v1.096de627","semantic_scholar.paper.search.match.retrieve.v1.b9f166f3","semantic_scholar.paper.search.match.retrieve.v1.d3fd06f6","semantic_scholar.paper.autocomplete.retrieve.v1.e16c9798","semantic_scholar.paper.autocomplete.list.v1.99d870f6","semantic_scholar.author.search.query.v1.1c428051","xiaosu.scholar.search.v1.f79d2374","semantic_scholar.author.search.list.v1.ea3671fb"]},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"returned_tool_ids":["qveris_academic.paper_aggregation_v1"],"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"execution_id":"f4d795bf-c2bc-4801-b538-167c2e8cfe23"},"finished_at":"2026-07-23T06:32:52.395Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"c7815fa0-d6c8-4346-8463-1ae287470639","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T06:32:52.395Z","status":"completed","discovery":{"result_tool_ids":["semantic_scholar.paper.search.bulk.execute.v1.239dfd66","qveris_academic.paper_aggregation_v1","semantic_scholar.paper.search.bulk.retrieve.v1.096de627","semantic_scholar.paper.search.match.retrieve.v1.b9f166f3","semantic_scholar.paper.search.match.retrieve.v1.d3fd06f6","semantic_scholar.paper.autocomplete.retrieve.v1.e16c9798","semantic_scholar.paper.autocomplete.list.v1.99d870f6","semantic_scholar.author.search.query.v1.1c428051","xiaosu.scholar.search.v1.f79d2374","semantic_scholar.author.search.list.v1.ea3671fb"]},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"returned_tool_ids":["qveris_academic.paper_aggregation_v1"],"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"execution_id":"b390cf81-6110-49a5-8730-9a9e4ec1730f"},"finished_at":"2026-07-23T06:33:23.255Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"844e8bbf-ae65-4047-9349-cbd1c1bf7b50","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T06:33:23.255Z","status":"completed","discovery":{"result_tool_ids":["semantic_scholar.paper.search.bulk.execute.v1.239dfd66","qveris_academic.paper_aggregation_v1","semantic_scholar.paper.search.bulk.retrieve.v1.096de627","semantic_scholar.paper.search.match.retrieve.v1.b9f166f3","semantic_scholar.paper.search.match.retrieve.v1.d3fd06f6","semantic_scholar.paper.autocomplete.retrieve.v1.e16c9798","semantic_scholar.paper.autocomplete.list.v1.99d870f6","semantic_scholar.author.search.query.v1.1c428051","xiaosu.scholar.search.v1.f79d2374","semantic_scholar.author.search.list.v1.ea3671fb"]},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"returned_tool_ids":["qveris_academic.paper_aggregation_v1"],"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"execution_id":"eb3ac762-0475-4c55-827c-6a78a9b4551d"},"finished_at":"2026-07-23T06:33:51.402Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"cc32c600-4ef5-4fd8-b37c-6ae16c084f7e","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T06:33:51.402Z","status":"completed","discovery":{"result_tool_ids":["api_sports.timezone.retrieve.v1.e993615d","api_sports.timezone.retrieve.v1.6792e6aa","api_sports.timezone.list.v1.504ebeef","api_sports.timezone.retrieve.v1.9b882d11","api_sports.timezone.list.v1.5a9c7ba0","api_sports.timezone.retrieve.v1.ae6ae8d7","api_sports.timezone.list.v1.bd878c58","api_sports.timezone.list.v1.58366f46","api_sports.timezone.retrieve.v1.64cfc345","api_sports.timezone.retrieve.v1.895e2257"]},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"returned_tool_ids":["api_sports.timezone.retrieve.v1.e993615d"],"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"execution_id":"16a904e5-16af-473d-8cf5-69a3f3ad4924"},"finished_at":"2026-07-23T06:34:27.740Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"de6093e2-ce59-4692-ae3c-da0801ac1675","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T06:34:27.740Z","status":"completed","discovery":{"result_tool_ids":["api_sports.timezone.retrieve.v1.e993615d","api_sports.timezone.retrieve.v1.6792e6aa","api_sports.timezone.list.v1.504ebeef","api_sports.timezone.retrieve.v1.9b882d11","api_sports.timezone.list.v1.5a9c7ba0","api_sports.timezone.retrieve.v1.ae6ae8d7","api_sports.timezone.list.v1.bd878c58","api_sports.timezone.list.v1.58366f46","api_sports.timezone.retrieve.v1.64cfc345","api_sports.timezone.retrieve.v1.895e2257"]},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"returned_tool_ids":["api_sports.timezone.retrieve.v1.e993615d"],"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"execution_id":"a05ed585-79cd-4953-afc4-f9b28295ae27"},"finished_at":"2026-07-23T06:34:55.284Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"a66e8303-b561-4d85-a2fa-baf46e0a953b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T06:34:55.284Z","status":"completed","discovery":{"result_tool_ids":["api_sports.timezone.retrieve.v1.e993615d","api_sports.timezone.retrieve.v1.6792e6aa","api_sports.timezone.list.v1.504ebeef","api_sports.timezone.retrieve.v1.9b882d11","api_sports.timezone.list.v1.5a9c7ba0","api_sports.timezone.retrieve.v1.ae6ae8d7","api_sports.timezone.list.v1.bd878c58","api_sports.timezone.list.v1.58366f46","api_sports.timezone.retrieve.v1.64cfc345","api_sports.timezone.retrieve.v1.895e2257"]},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"returned_tool_ids":["api_sports.timezone.retrieve.v1.e993615d"],"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"execution_id":"3810f78b-e90d-4129-a174-eebc48bdb9dd"},"finished_at":"2026-07-23T06:35:32.765Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"28718552-be81-486c-b313-75a5550ff3cc","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T06:35:32.765Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.geocoder.reverse.v2","baidu_map.geocoder.query.v2","baidu_map.reverse_geocoding.query.v2","baidu_map.geocoding.reverse_geocode.v1","baidu_map.geocoder.wgeoreverse.create.v1","amap_webservice.geocode.geo.retrieve.v3","baidu_map.reversegeocoding.create.v1","baidu_map.geocoder.wgeobatch.create.v1","arcgis_ocean.geocoding.reverse_geocode.v1.4d40d93e","baidu_map.geoconv.convert.v1"]},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"returned_tool_ids":["amap_webservice.geocode.geo.retrieve.v3"],"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京","output":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"668c4105-b57e-46af-89be-b199a1a01735"},"finished_at":"2026-07-23T06:36:03.749Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"26e071b2-d139-4ee3-879c-5900578b1233","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T06:36:03.749Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.geocoder.reverse.v2","baidu_map.geocoder.query.v2","baidu_map.reverse_geocoding.query.v2","baidu_map.geocoding.reverse_geocode.v1","baidu_map.geocoder.wgeoreverse.create.v1","amap_webservice.geocode.geo.retrieve.v3","baidu_map.reversegeocoding.create.v1","baidu_map.geocoder.wgeobatch.create.v1","arcgis_ocean.geocoding.reverse_geocode.v1.4d40d93e","baidu_map.geoconv.convert.v1"]},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"returned_tool_ids":["amap_webservice.geocode.geo.retrieve.v3"],"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"84744b45-4016-4932-b441-c3f8a2b4b236"},"finished_at":"2026-07-23T06:36:38.806Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"e2f6f4c2-a8e7-4967-98b1-1a8c29aa3410","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T06:36:38.806Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.geocoder.reverse.v2","baidu_map.geocoder.query.v2","baidu_map.reverse_geocoding.query.v2","baidu_map.geocoding.reverse_geocode.v1","baidu_map.geocoder.wgeoreverse.create.v1","amap_webservice.geocode.geo.retrieve.v3","baidu_map.reversegeocoding.create.v1","baidu_map.geocoder.wgeobatch.create.v1","arcgis_ocean.geocoding.reverse_geocode.v1.4d40d93e","baidu_map.geoconv.convert.v1"]},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"returned_tool_ids":["amap_webservice.geocode.geo.retrieve.v3"],"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"b0668851-5bb2-4489-8abd-c23dd883fdf8"},"finished_at":"2026-07-23T06:37:14.052Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"fde5c3d4-8ff3-4813-a8c5-5e6a61ed3c2e","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-23T06:37:14.052Z","status":"completed","discovery":{"result_tool_ids":["decodo.isp_pay_per_ip.ip.retrieve.v1.bc2c6371","baidu_map.location.ip.retrieve.v1","baidu_map.location.ip.execute.v1","amap_webservice.ip.retrieve.v3","virustotal.ip.info.retrieve.v3.ab34b0d1","virustotal.ip.report.retrieve.v3.ab34b0d1","baidu_map.ip_location.retrieve.v1","weather_api.ip_lookup.retrieve.v1.e095d904","virustotal.ip_addresses.retrieve.v3.ab34b0d1","virustotal.ip-address.report.retrieve.v2.79606e33"]},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"returned_tool_ids":["weather_api.ip_lookup.retrieve.v1.e095d904"],"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"execution_id":"3da6c262-0b07-4ca0-90db-b627a142ad22"},"finished_at":"2026-07-23T06:37:44.040Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"cb085acd-e437-40df-9e07-e91343e2af9d","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-23T06:37:44.040Z","status":"completed","discovery":{"result_tool_ids":["decodo.isp_pay_per_ip.ip.retrieve.v1.bc2c6371","baidu_map.location.ip.retrieve.v1","baidu_map.location.ip.execute.v1","amap_webservice.ip.retrieve.v3","virustotal.ip.info.retrieve.v3.ab34b0d1","virustotal.ip.report.retrieve.v3.ab34b0d1","baidu_map.ip_location.retrieve.v1","weather_api.ip_lookup.retrieve.v1.e095d904","virustotal.ip_addresses.retrieve.v3.ab34b0d1","virustotal.ip-address.report.retrieve.v2.79606e33"]},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"returned_tool_ids":["weather_api.ip_lookup.retrieve.v1.e095d904"],"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"execution_id":"31469806-e596-4d4e-a570-362026d6c4f1"},"finished_at":"2026-07-23T06:38:14.212Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"143359c4-589b-437f-ae30-9a99b9f1636b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-23T06:38:14.212Z","status":"completed","discovery":{"result_tool_ids":["decodo.isp_pay_per_ip.ip.retrieve.v1.bc2c6371","baidu_map.location.ip.retrieve.v1","baidu_map.location.ip.execute.v1","amap_webservice.ip.retrieve.v3","virustotal.ip.info.retrieve.v3.ab34b0d1","virustotal.ip.report.retrieve.v3.ab34b0d1","baidu_map.ip_location.retrieve.v1","weather_api.ip_lookup.retrieve.v1.e095d904","virustotal.ip_addresses.retrieve.v3.ab34b0d1","virustotal.ip-address.report.retrieve.v2.79606e33"]},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"returned_tool_ids":["weather_api.ip_lookup.retrieve.v1.e095d904"],"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"execution_id":"9fd3e989-f14d-4d66-96db-e84835184d9e"},"finished_at":"2026-07-23T06:38:55.364Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"58e4b1fd-8de8-4f85-8db1-478b57309b00","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"crypto-price","trial":1,"started_at":"2026-07-23T06:38:55.364Z","status":"completed","discovery":{"result_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","binance.avg_price.retrieve.v1.aef0de4f","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.251242d7","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.9dc24be4","binance.ticker.price.list.v3.8675eca0","binance.avgPrice.retrieve.v3.7b0852b4","deribit.public.get_index_price.retrieve.v2.d47dcdcb","coinmarketcap.cryptocurrency.listings.latest.retrieve.v1.8dddc12f","etherscan.api.stats.ethprice.retrieve.v2.8eeca59e","binance.ticker.query.v1.0437113d"]},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"returned_tool_ids":["binance.ticker.price.list.v3.8675eca0"],"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"execution_id":"999554c2-cfb9-4494-802a-d945cf5d93d3"},"finished_at":"2026-07-23T06:39:36.286Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"f498a668-e254-4a1f-a109-0b11043a9a77","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"crypto-price","trial":2,"started_at":"2026-07-23T06:39:36.286Z","status":"completed","discovery":{"result_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","binance.avg_price.retrieve.v1.aef0de4f","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.251242d7","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.9dc24be4","binance.ticker.price.list.v3.8675eca0","binance.avgPrice.retrieve.v3.7b0852b4","deribit.public.get_index_price.retrieve.v2.d47dcdcb","coinmarketcap.cryptocurrency.listings.latest.retrieve.v1.8dddc12f","etherscan.api.stats.ethprice.retrieve.v2.8eeca59e","binance.ticker.query.v1.0437113d"]},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"returned_tool_ids":["binance.ticker.price.list.v3.8675eca0"],"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"execution_id":"e0046687-c092-40e5-9cbd-7e79da6e216a"},"finished_at":"2026-07-23T06:40:06.523Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"3b48193b-6026-491e-8472-51034ae93749","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"crypto-price","trial":3,"started_at":"2026-07-23T06:40:06.523Z","status":"completed","discovery":{"result_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","binance.avg_price.retrieve.v1.aef0de4f","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.251242d7","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.9dc24be4","binance.ticker.price.list.v3.8675eca0","binance.avgPrice.retrieve.v3.7b0852b4","deribit.public.get_index_price.retrieve.v2.d47dcdcb","coinmarketcap.cryptocurrency.listings.latest.retrieve.v1.8dddc12f","etherscan.api.stats.ethprice.retrieve.v2.8eeca59e","binance.ticker.query.v1.0437113d"]},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"returned_tool_ids":["binance.ticker.price.list.v3.8675eca0"],"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"execution_id":"883d9f06-9b8c-4d64-8a25-4b1c8020c4b3"},"finished_at":"2026-07-23T06:40:52.281Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"3d70bc1e-3063-41e6-8b57-40c6e6b963e2","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"web-search","trial":1,"started_at":"2026-07-23T06:40:52.281Z","status":"completed","discovery":{"result_tool_ids":["linkup.search.v1","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic","cloudsway.smart_search_overseas","firecrawl.search.create.v2.a99b6f74","brave_search.images.search.list.v1","x_developer.2.news.search.retrieve.v1.19b3e678","serpapi.locations.locations.list.v1","x_developer.2.spaces.search.retrieve.v1.43a1f31c","tavily.map.execute.v1","brave_search.web.search.list.v1.f02ed09d"]},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"returned_tool_ids":["cloudsway.smart_search_overseas"],"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"execution_id":"37c52450-eeda-417c-9849-d67132e689b0"},"finished_at":"2026-07-23T06:41:28.119Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"39c78597-f3db-4d54-b56e-9ed6e6b3d412","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"web-search","trial":2,"started_at":"2026-07-23T06:41:28.119Z","status":"completed","discovery":{"result_tool_ids":["linkup.search.v1","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic","cloudsway.smart_search_overseas","firecrawl.search.create.v2.a99b6f74","brave_search.images.search.list.v1","x_developer.2.news.search.retrieve.v1.19b3e678","serpapi.locations.locations.list.v1","x_developer.2.spaces.search.retrieve.v1.43a1f31c","tavily.map.execute.v1","brave_search.web.search.list.v1.f02ed09d"]},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"returned_tool_ids":["cloudsway.smart_search_overseas"],"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"execution_id":"8e870c4f-ff66-4af3-be0a-a3e57848cc37"},"finished_at":"2026-07-23T06:42:00.561Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"cf50c76f-316f-4793-877b-2f51474c6981","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"web-search","trial":3,"started_at":"2026-07-23T06:42:00.561Z","status":"completed","discovery":{"result_tool_ids":["linkup.search.v1","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic","cloudsway.smart_search_overseas","brave_search.images.search.list.v1","firecrawl.search.create.v2.a99b6f74","x_developer.2.spaces.search.retrieve.v1.43a1f31c","serpapi.locations.locations.list.v1","x_developer.2.news.search.retrieve.v1.19b3e678","tavily.map.execute.v1","brave_search.web.search.list.v1.f02ed09d"]},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"returned_tool_ids":["cloudsway.smart_search_overseas"],"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"execution_id":"17f61a10-d829-4e93-b2ee-08cd5cd5fe38"},"finished_at":"2026-07-23T06:42:34.259Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"5bf899ee-7846-44cf-8e8a-ebeb8bfbff65","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"air-quality","trial":1,"started_at":"2026-07-23T06:42:34.259Z","status":"completed","discovery":{"result_tool_ids":["openweathermap.air_pollution.current.v2_5","openweathermap.air_pollution.retrieve.v2.5","openweathermap.air_pollution.history.v2_5","openweathermap.air_pollution.history.retrieve.v2.5","openweathermap.air_pollution.forecast.retrieve.v2.5","epa_envirofacts.qaannualperformanceevaluations.bystate.retrieve.v1.61a2aa7d","epa_envirofacts.transactionsqaannualperformanceevaluations.bystate.retrieve.v1.4143b8bf","epa_envirofacts.transactionssample.bysite.retrieve.v1.93889e72","epa_envirofacts.annualdata.bybox.retrieve.v1.4b74a0d6","epa_envirofacts.dailydata.bycounty.retrieve.v1.f1ee8d1b"]},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"returned_tool_ids":["openweathermap.air_pollution.current.v2_5"],"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"execution_id":"b8834f76-74db-4431-9f9a-3e0e73012872"},"finished_at":"2026-07-23T06:43:06.453Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"673ccda1-c286-49cd-9299-75ce17706e92","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"air-quality","trial":2,"started_at":"2026-07-23T06:43:06.453Z","status":"completed","discovery":{"result_tool_ids":["openweathermap.air_pollution.current.v2_5","openweathermap.air_pollution.retrieve.v2.5","openweathermap.air_pollution.history.v2_5","openweathermap.air_pollution.history.retrieve.v2.5","openweathermap.air_pollution.forecast.retrieve.v2.5","epa_envirofacts.qaannualperformanceevaluations.bystate.retrieve.v1.61a2aa7d","epa_envirofacts.transactionsqaannualperformanceevaluations.bystate.retrieve.v1.4143b8bf","epa_envirofacts.transactionssample.bysite.retrieve.v1.93889e72","epa_envirofacts.annualdata.bybox.retrieve.v1.4b74a0d6","epa_envirofacts.dailydata.bycounty.retrieve.v1.f1ee8d1b"]},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"returned_tool_ids":["openweathermap.air_pollution.current.v2_5"],"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"execution_id":"d478235d-935b-4887-8d4c-286f524c9f3d"},"finished_at":"2026-07-23T06:43:43.650Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"859aee1a-1670-4140-b9ac-91435124e6dc","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"air-quality","trial":3,"started_at":"2026-07-23T06:43:43.650Z","status":"completed","discovery":{"result_tool_ids":["openweathermap.air_pollution.current.v2_5","openweathermap.air_pollution.retrieve.v2.5","openweathermap.air_pollution.history.v2_5","openweathermap.air_pollution.history.retrieve.v2.5","openweathermap.air_pollution.forecast.retrieve.v2.5","epa_envirofacts.qaannualperformanceevaluations.bystate.retrieve.v1.61a2aa7d","epa_envirofacts.transactionsqaannualperformanceevaluations.bystate.retrieve.v1.4143b8bf","epa_envirofacts.transactionssample.bysite.retrieve.v1.93889e72","epa_envirofacts.annualdata.bybox.retrieve.v1.4b74a0d6","epa_envirofacts.dailydata.bycounty.retrieve.v1.f1ee8d1b"]},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"returned_tool_ids":["openweathermap.air_pollution.current.v2_5"],"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"execution_id":"a0193cf7-479f-4113-8dd3-ec693b39bfbc"},"finished_at":"2026-07-23T06:44:12.213Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"70a47304-ae9e-4cb5-8331-5df6a8e83a43","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-news","trial":1,"started_at":"2026-07-23T06:44:12.213Z","status":"completed","discovery":{"result_tool_ids":["finnhub.market.news.list.v1","finnhub.news.retrieve.v1.51b2567e","finnhub.news.retrieve.v1.2e72ea5d","finnhub.news.execute.v1","finnhub.news.retrieve.v1","alphavantage.news_sentiment.query.v1.467a92c0","finnhub.companynews.retrieve.v1.4d11f47b","finnhub.companynews.retrieve.v1","finnhub.companynews.retrieve.v1.e428b704","finnhub.companynews.execute.v1"]},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"returned_tool_ids":["finnhub.market.news.list.v1"],"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"execution_id":"d9e295bf-8357-492a-92b0-6d6ccc1ad06d"},"finished_at":"2026-07-23T06:44:39.351Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"73355630-bc92-4e0a-a1c8-4f25f895a949","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-news","trial":2,"started_at":"2026-07-23T06:44:39.351Z","status":"completed","discovery":{"result_tool_ids":["finnhub.market.news.list.v1","finnhub.news.retrieve.v1.51b2567e","finnhub.news.retrieve.v1.2e72ea5d","finnhub.news.execute.v1","finnhub.news.retrieve.v1","alphavantage.news_sentiment.query.v1.467a92c0","finnhub.companynews.retrieve.v1.4d11f47b","finnhub.companynews.retrieve.v1","finnhub.companynews.retrieve.v1.e428b704","finnhub.companynews.execute.v1"]},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"returned_tool_ids":["finnhub.market.news.list.v1"],"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"execution_id":"19ddaebd-5dd6-4792-93a2-16617d3db005"},"finished_at":"2026-07-23T06:45:14.542Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"f05cea11-e23c-463a-98f1-e5c0b4f51e88","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-news","trial":3,"started_at":"2026-07-23T06:45:14.542Z","status":"completed","discovery":{"result_tool_ids":["finnhub.market.news.list.v1","finnhub.news.retrieve.v1.51b2567e","finnhub.news.retrieve.v1.2e72ea5d","finnhub.news.execute.v1","finnhub.news.retrieve.v1","alphavantage.news_sentiment.query.v1.467a92c0","finnhub.companynews.retrieve.v1.4d11f47b","finnhub.companynews.retrieve.v1","finnhub.companynews.retrieve.v1.e428b704","finnhub.companynews.execute.v1"]},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"returned_tool_ids":["finnhub.market.news.list.v1"],"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"execution_id":"11798d87-25ba-45c9-80fd-4a20871b981d"},"finished_at":"2026-07-23T06:45:40.767Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"ae3e60aa-8ee9-4650-b3e2-6d906a42da8b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T06:45:40.767Z","status":"failed","discovery":{"result_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8","dataforseo.dataforseo_labs.google.domain_whois_overview.live.execute.v3","dataforseo.databases.domains.query.v3.c1720ec0","virustotal.domains.report.retrieve.v3.c5459740","bainiu_api.common.company_domain.query.v1.e978ca32","virustotal.domain_reports.retrieve.v3.c5459740","dataforseo.domain_analytics.technologies.domains_by_technology.live.create.v3.bfc3dc2d","virustotal.domains.retrieve.v3.c5459740","virustotal.domains.analyse.v3.c755c69e","dataforseo.domain_analytics.technologies.domains_by_html_terms.live.create.v3.d3760ae1"]},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"returned_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"],"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","message":"Adapter invocation failed"},"finished_at":"2026-07-23T06:46:30.257Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"cc1eebe6-e68b-45ba-8f32-827f98baa240","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T06:46:30.257Z","status":"failed","discovery":{"result_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8","dataforseo.dataforseo_labs.google.domain_whois_overview.live.execute.v3","dataforseo.databases.domains.query.v3.c1720ec0","virustotal.domains.report.retrieve.v3.c5459740","bainiu_api.common.company_domain.query.v1.e978ca32","virustotal.domain_reports.retrieve.v3.c5459740","dataforseo.domain_analytics.technologies.domains_by_technology.live.create.v3.bfc3dc2d","virustotal.domains.retrieve.v3.c5459740","virustotal.domains.analyse.v3.c755c69e","dataforseo.domain_analytics.technologies.domains_by_html_terms.live.create.v3.d3760ae1"]},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"returned_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"],"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","message":"Adapter invocation failed"},"finished_at":"2026-07-23T06:47:52.318Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"19382342-d937-46dc-ba4a-c71139081673","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T06:47:52.318Z","status":"failed","discovery":{"result_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8","dataforseo.dataforseo_labs.google.domain_whois_overview.live.execute.v3","dataforseo.databases.domains.query.v3.c1720ec0","virustotal.domains.report.retrieve.v3.c5459740","bainiu_api.common.company_domain.query.v1.e978ca32","virustotal.domain_reports.retrieve.v3.c5459740","dataforseo.domain_analytics.technologies.domains_by_technology.live.create.v3.bfc3dc2d","virustotal.domains.retrieve.v3.c5459740","virustotal.domains.analyse.v3.c755c69e","dataforseo.domain_analytics.technologies.domains_by_html_terms.live.create.v3.d3760ae1"]},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"returned_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"],"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","message":"Adapter invocation failed"},"finished_at":"2026-07-23T06:48:55.681Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"48e281c3-3a7a-4327-9911-427072bc6fcd","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"image-search","trial":1,"started_at":"2026-07-23T06:48:55.681Z","status":"completed","discovery":{"result_tool_ids":["brave_search.images.search.list.v1","shutterstock.images.search.list.v2.69059569","shutterstock.images.search.list.v2.c2b44682","dataforseo.serp.google.search_by_image.task.create.v3","jina_search.rerank.create.v2026.23dc486d","jina_search.embeddings.create.v2026.5c7cb661","linkup.search.v1","shutterstock.images.search.list.v2.15f15845","shutterstock.images.search.suggestions.list.v2.f6850c10","hailuoai.image_generation.create.v1"]},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"returned_tool_ids":["brave_search.images.search.list.v1"],"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"execution_id":"1262ba5e-f345-4ef0-a799-515d53cb2082"},"finished_at":"2026-07-23T06:49:28.588Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"d23e1fc9-03a5-4c76-8761-22c9de0e6231","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"image-search","trial":2,"started_at":"2026-07-23T06:49:28.588Z","status":"completed","discovery":{"result_tool_ids":["brave_search.images.search.list.v1","shutterstock.images.search.list.v2.69059569","shutterstock.images.search.list.v2.c2b44682","dataforseo.serp.google.search_by_image.task.create.v3","jina_search.rerank.create.v2026.23dc486d","jina_search.embeddings.create.v2026.5c7cb661","linkup.search.v1","shutterstock.images.search.list.v2.15f15845","shutterstock.images.search.suggestions.list.v2.f6850c10","hailuoai.image_generation.create.v1"]},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"returned_tool_ids":["brave_search.images.search.list.v1"],"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":10,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"execution_id":"87f346a7-38d1-4666-ae15-3e1843c5e0a5"},"finished_at":"2026-07-23T06:49:55.653Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"262b0d23-181c-4b1c-8241-8cfd530dbb96","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"image-search","trial":3,"started_at":"2026-07-23T06:49:55.653Z","status":"completed","discovery":{"result_tool_ids":["brave_search.images.search.list.v1","shutterstock.images.search.list.v2.69059569","shutterstock.images.search.list.v2.c2b44682","dataforseo.serp.google.search_by_image.task.create.v3","jina_search.rerank.create.v2026.23dc486d","jina_search.embeddings.create.v2026.5c7cb661","linkup.search.v1","shutterstock.images.search.list.v2.15f15845","shutterstock.images.search.suggestions.list.v2.f6850c10","hailuoai.image_generation.create.v1"]},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"returned_tool_ids":["brave_search.images.search.list.v1"],"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"execution_id":"4f2204fb-38c5-45c9-a415-52d2b6f3294a"},"finished_at":"2026-07-23T06:50:22.831Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"3d110e2a-4ee1-47f7-a018-f5ae9d166e46","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-holidays","trial":1,"started_at":"2026-07-23T06:50:22.831Z","status":"completed","discovery":{"result_tool_ids":["eodhd.exchange_details.retrieve.v1.7da32390","finnhub.stock.marketholiday.retrieve.v1","finnhub.stock.marketholiday.execute.v1","finnhub.stock.marketholiday.retrieve.v1.5c5bfe9a","finnhub.stock.marketholiday.retrieve.v1.75d6c1f9","massive_stocks.stocks.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.retrieve.v1","massive_stocks.options.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.list.v1","theta_data.calendar.yearholidays.retrieve.v3.125586f0"]},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"returned_tool_ids":["finnhub.stock.marketholiday.retrieve.v1"],"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"execution_id":"67e0f2b5-0ae5-4de8-8d5d-285971094a0d"},"finished_at":"2026-07-23T06:50:57.424Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"b619dc10-8cc0-4413-9219-1aea84479b72","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-holidays","trial":2,"started_at":"2026-07-23T06:50:57.424Z","status":"completed","discovery":{"result_tool_ids":["eodhd.exchange_details.retrieve.v1.7da32390","finnhub.stock.marketholiday.retrieve.v1","finnhub.stock.marketholiday.execute.v1","finnhub.stock.marketholiday.retrieve.v1.5c5bfe9a","finnhub.stock.marketholiday.retrieve.v1.75d6c1f9","massive_stocks.stocks.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.retrieve.v1","massive_stocks.options.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.list.v1","theta_data.calendar.yearholidays.retrieve.v3.125586f0"]},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"returned_tool_ids":["finnhub.stock.marketholiday.retrieve.v1"],"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"execution_id":"596fd2ea-7672-49ce-9c15-b6ca3f725589"},"finished_at":"2026-07-23T06:51:30.279Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"0ab6d79a-6510-4b11-966c-7063b6646f73","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-holidays","trial":3,"started_at":"2026-07-23T06:51:30.279Z","status":"completed","discovery":{"result_tool_ids":["eodhd.exchange_details.retrieve.v1.7da32390","finnhub.stock.marketholiday.retrieve.v1","finnhub.stock.marketholiday.execute.v1","finnhub.stock.marketholiday.retrieve.v1.5c5bfe9a","finnhub.stock.marketholiday.retrieve.v1.75d6c1f9","massive_stocks.stocks.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.retrieve.v1","massive_stocks.options.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.list.v1","theta_data.calendar.yearholidays.retrieve.v3.125586f0"]},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"returned_tool_ids":["finnhub.stock.marketholiday.retrieve.v1"],"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"execution_id":"49e1d181-d923-4d8b-9e0c-26ff778119a3"},"finished_at":"2026-07-23T06:51:58.663Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"245b91ee-988d-474f-8fd1-82da7c62183a","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-23T06:51:58.663Z","status":"completed","discovery":{"result_tool_ids":["usgs_earthquake_catalog.event.events.list.v1","usgs_earthquake_catalog.fdsnws.event.query.retrieve.v1.025549c9","usgs_earthquake_catalog.earthquakes.query.v1","usgs_earthquake_catalog.fdsnws.event.query.v1.025549c9","usgs_earthquake_catalog.events.query.v1.025549c9","usgs_earthquake.catalogs.list.v1","usgs_earthquake.contributors.list.v1","usgs-earthquake-event-api.event.query.list.v1","usgs_earthquake.parameters.list.v1","usgs_earthquake.version.retrieve.v1"]},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"returned_tool_ids":["usgs_earthquake_catalog.event.events.list.v1"],"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"execution_id":"0a2f5cc0-b3e1-4538-8cad-9f2b807e253d"},"finished_at":"2026-07-23T06:52:30.880Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"dc38691f-d8f6-40f8-9e77-9288c33a7e68","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-23T06:52:30.881Z","status":"completed","discovery":{"result_tool_ids":["usgs_earthquake_catalog.event.events.list.v1","usgs_earthquake_catalog.fdsnws.event.query.retrieve.v1.025549c9","usgs_earthquake_catalog.earthquakes.query.v1","usgs_earthquake_catalog.fdsnws.event.query.v1.025549c9","usgs_earthquake_catalog.events.query.v1.025549c9","usgs_earthquake.catalogs.list.v1","usgs_earthquake.contributors.list.v1","usgs-earthquake-event-api.event.query.list.v1","usgs_earthquake.parameters.list.v1","usgs_earthquake.version.retrieve.v1"]},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"returned_tool_ids":["usgs_earthquake_catalog.event.events.list.v1"],"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"execution_id":"1c998fc3-bda1-437c-9f78-90b00026a63d"},"finished_at":"2026-07-23T06:53:04.476Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"c1624700-5aba-4cfc-8a68-acabf3de024a","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T06:53:04.476Z","status":"failed","discovery":{"result_tool_ids":["usgs_earthquake_catalog.event.events.list.v1","usgs_earthquake_catalog.fdsnws.event.query.retrieve.v1.025549c9","usgs_earthquake_catalog.earthquakes.query.v1","usgs_earthquake_catalog.fdsnws.event.query.v1.025549c9","usgs_earthquake_catalog.events.query.v1.025549c9","usgs_earthquake.catalogs.list.v1","usgs_earthquake.contributors.list.v1","usgs-earthquake-event-api.event.query.list.v1","usgs_earthquake.parameters.list.v1","usgs_earthquake.version.retrieve.v1"]},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"returned_tool_ids":["usgs_earthquake_catalog.event.events.list.v1"],"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","message":"Adapter invocation failed"},"finished_at":"2026-07-23T06:55:18.009Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"0e485876-22f8-4265-91e0-51990c3644e9","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"company-profile","trial":1,"started_at":"2026-07-23T06:55:18.009Z","status":"completed","discovery":{"result_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2","bainiu_api.company.detail.execute.v1.f6b528e7","bainiu_api.openapi.company.alter.execute.v1.8fa090d1","tradefeeds.compinfo.retrieve.v1.c2314765","financialmodelingprep.stable.companynotes.retrieve.v1.930aa468","financialmodelingprep.search.cusip.retrieve.v1.51d0c621","bainiu_api.openapi.common.company_info_deep_search.execute.v1.8e5f009d","financialmodelingprep.stable.secfilingscompanysearch.symbol.retrieve.v1.5cf7397d","finnhub.stock.profile2.retrieve.v1.59df26f6","bainiu_api.company_detail.create.v1.8fa090d1"]},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"returned_tool_ids":["tradefeeds.compinfo.retrieve.v1.c2314765"],"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"execution_id":"59b6de87-9bf7-4cb9-beef-7dfc854746e1"},"finished_at":"2026-07-23T06:55:45.466Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"ea693fd1-b596-40b1-bd16-eec213eecb12","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"company-profile","trial":2,"started_at":"2026-07-23T06:55:45.466Z","status":"completed","discovery":{"result_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2","bainiu_api.company.detail.execute.v1.f6b528e7","bainiu_api.openapi.company.alter.execute.v1.8fa090d1","tradefeeds.compinfo.retrieve.v1.c2314765","financialmodelingprep.stable.companynotes.retrieve.v1.930aa468","financialmodelingprep.search.cusip.retrieve.v1.51d0c621","bainiu_api.openapi.common.company_info_deep_search.execute.v1.8e5f009d","financialmodelingprep.stable.secfilingscompanysearch.symbol.retrieve.v1.5cf7397d","finnhub.stock.profile2.retrieve.v1.59df26f6","bainiu_api.company_detail.create.v1.8fa090d1"]},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"returned_tool_ids":["tradefeeds.compinfo.retrieve.v1.c2314765"],"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"execution_id":"a278f80c-4d27-428b-a610-39eb6993d3bb"},"finished_at":"2026-07-23T06:56:19.074Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"abf65b68-8d35-453f-89b6-d30447a432e7","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"company-profile","trial":3,"started_at":"2026-07-23T06:56:19.074Z","status":"completed","discovery":{"result_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2","bainiu_api.company.detail.execute.v1.f6b528e7","bainiu_api.openapi.company.alter.execute.v1.8fa090d1","tradefeeds.compinfo.retrieve.v1.c2314765","financialmodelingprep.stable.companynotes.retrieve.v1.930aa468","financialmodelingprep.search.cusip.retrieve.v1.51d0c621","bainiu_api.openapi.common.company_info_deep_search.execute.v1.8e5f009d","financialmodelingprep.stable.secfilingscompanysearch.symbol.retrieve.v1.5cf7397d","finnhub.stock.profile2.retrieve.v1.59df26f6","bainiu_api.company_detail.create.v1.8fa090d1"]},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"returned_tool_ids":["tradefeeds.compinfo.retrieve.v1.c2314765"],"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"execution_id":"831c21f1-85e0-450c-b5f1-22ea733384bc"},"finished_at":"2026-07-23T06:56:53.486Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"a6ccdda6-dc1f-4765-873e-bd8c22197a6f","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"sports-standings","trial":1,"started_at":"2026-07-23T06:56:53.486Z","status":"completed","discovery":{"result_tool_ids":["api_sports.standings.list.v1.98a153bd","api_sports.standings.list.v1.39568ba4","api_sports.standings.groups.list.v1.a077a680","api_sports.standings.groups.list.v1.0d8a40cf","api_sports.standings.groups.list.v1.57393fb3","api_sports.standings.retrieve.v1.ff019893","api_sports.standings.stages.list.v1.838f769a","api_sports.standings.stages.list.v1.5ba13a89","api-football.standings.list.v3.76f7703e","api_sports.standings.groups.list.v1.bed67ce0"]},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"returned_tool_ids":["api-football.standings.list.v3.76f7703e"],"required_parameters":["season"]},"parameters":{"league":39,"season":2025},"call":{"attempted":true,"success":true,"execution_id":"470514d9-7c25-4168-be3e-bfebf7fdf390"},"finished_at":"2026-07-23T06:57:26.157Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"a025004c-c94c-444b-a2c9-bb176ed51ea3","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"sports-standings","trial":2,"started_at":"2026-07-23T06:57:26.157Z","status":"completed","discovery":{"result_tool_ids":["api_sports.standings.list.v1.98a153bd","api_sports.standings.list.v1.39568ba4","api_sports.standings.groups.list.v1.a077a680","api_sports.standings.groups.list.v1.0d8a40cf","api_sports.standings.groups.list.v1.57393fb3","api_sports.standings.retrieve.v1.ff019893","api_sports.standings.stages.list.v1.838f769a","api_sports.standings.stages.list.v1.5ba13a89","api-football.standings.list.v3.76f7703e","api_sports.standings.groups.list.v1.bed67ce0"]},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"returned_tool_ids":["api-football.standings.list.v3.76f7703e"],"required_parameters":["season"]},"parameters":{"league":39,"season":2025},"call":{"attempted":true,"success":true,"execution_id":"a0359f15-d70c-4906-97b9-2faeb215a13c"},"finished_at":"2026-07-23T06:57:50.791Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"f15c0986-0c0c-4eab-8dac-99cc4f6f7dea","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"sports-standings","trial":3,"started_at":"2026-07-23T06:57:50.791Z","status":"completed","discovery":{"result_tool_ids":["api_sports.standings.list.v1.98a153bd","api_sports.standings.list.v1.39568ba4","api_sports.standings.groups.list.v1.a077a680","api_sports.standings.groups.list.v1.0d8a40cf","api_sports.standings.groups.list.v1.57393fb3","api_sports.standings.retrieve.v1.ff019893","api_sports.standings.stages.list.v1.838f769a","api_sports.standings.stages.list.v1.5ba13a89","api-football.standings.list.v3.76f7703e","api_sports.standings.groups.list.v1.bed67ce0"]},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"returned_tool_ids":["api-football.standings.list.v3.76f7703e"],"required_parameters":["season"]},"parameters":{"league":39,"season":2025},"call":{"attempted":true,"success":true,"execution_id":"056d8f80-b061-4b18-940e-ba3a460c330d"},"finished_at":"2026-07-23T06:58:17.129Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ac88304b-0662-4d2b-bb4b-778a65d4d84d","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":1,"started_at":"2026-07-23T06:26:39.032Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8ff1444540e79a77ed35ac1b5cb0b6de9b5cbfad78aca4b2d6f896484d9fd1d4","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:27:00.228Z","source_record_sha256":"e5922113293b9cbaef44499c19030629c3cd20fb1d48276eb4ef820887c78c9d"} +{"schema_version":1,"benchmark_version":"v1","run_id":"5b911181-4d6c-472a-a2a0-4cb12e433f06","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":2,"started_at":"2026-07-23T06:27:00.228Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8ff1444540e79a77ed35ac1b5cb0b6de9b5cbfad78aca4b2d6f896484d9fd1d4","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:27:23.671Z","source_record_sha256":"daa24fddaeade477213fd0cfdafdaf2a07333f517ebf55f0d5d65dcb0bbb9f6b"} +{"schema_version":1,"benchmark_version":"v1","run_id":"a8268395-01a9-413b-b2af-ba30d2e71f9a","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":3,"started_at":"2026-07-23T06:27:23.671Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8ff1444540e79a77ed35ac1b5cb0b6de9b5cbfad78aca4b2d6f896484d9fd1d4","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:27:50.399Z","source_record_sha256":"75bc10aabffa921d7baefe47658cf7b371196e6ee76a29059d64acad8687a267"} +{"schema_version":1,"benchmark_version":"v1","run_id":"4e925856-364c-4ca7-b769-c8d4d43e85c2","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-23T06:27:50.399Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8c4e8e2ab56f10ce503086b98981d6be57f293b4043dec00a86237144a107a5b","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:28:19.862Z","source_record_sha256":"06c1b8e490bc19f2da0294fb66c4db895a9fa3337d8ed4c8ed35438b17bb33b6"} +{"schema_version":1,"benchmark_version":"v1","run_id":"9c2b558d-b803-4299-9b8d-7778e56d0fcb","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-23T06:28:19.862Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8c4e8e2ab56f10ce503086b98981d6be57f293b4043dec00a86237144a107a5b","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:28:47.286Z","source_record_sha256":"0e1e42c0295011ad188ea782a3eda5887fa763faab0dbe0f5c3f399da3a5e1f8"} +{"schema_version":1,"benchmark_version":"v1","run_id":"6762a6af-2002-4349-9239-89c6c62a597b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-23T06:28:47.286Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8c4e8e2ab56f10ce503086b98981d6be57f293b4043dec00a86237144a107a5b","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:29:13.981Z","source_record_sha256":"fdea4303bae16790c241f8d7743d7ced5d74783b6681a8c8792d6ca6f9dd0c4c"} +{"schema_version":1,"benchmark_version":"v1","run_id":"aa24674f-2a54-49ce-9652-95a7afc5a139","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-23T06:29:13.981Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e78e9a2e15b7817ec95c9a9f2dd8863f05b1049624a65dd87d0387e113e06d29","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":100,"format":"JSON"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:29:40.942Z","source_record_sha256":"f0206728537187674d6417f5baa0210e91dc590443b30c2f658957890ba782f1"} +{"schema_version":1,"benchmark_version":"v1","run_id":"d038e88d-be8e-4d2e-810c-ec4f155b6871","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-23T06:29:40.943Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e78e9a2e15b7817ec95c9a9f2dd8863f05b1049624a65dd87d0387e113e06d29","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":100,"format":"JSON"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:30:10.765Z","source_record_sha256":"ffac927ecd59c4b15f6433f9e4a697caea50585c7a1f31b3fbff9f5b726e0da3"} +{"schema_version":1,"benchmark_version":"v1","run_id":"c54fe3bb-0b0a-4758-beec-b7a5ce2bb540","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-23T06:30:10.765Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e78e9a2e15b7817ec95c9a9f2dd8863f05b1049624a65dd87d0387e113e06d29","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:30:38.654Z","source_record_sha256":"5e3fff916b17629db0000dccb4d7df9783a86ba705981beeac9a3f061819e2df"} +{"schema_version":1,"benchmark_version":"v1","run_id":"6bede096-05fb-47c7-a2eb-5a2f12dad17d","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":1,"started_at":"2026-07-23T06:30:38.654Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"048ef068d6abc7e544421421e151018f4495f28c24194b2c149e5b5fdc19a19f","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20%22generative%20AI%22","from":"2026-07-16","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:31:10.595Z","source_record_sha256":"0e8d2e4dec3b62efe1c8dc241d837790f077bf855a59e34e146ba36fb9d87157"} +{"schema_version":1,"benchmark_version":"v1","run_id":"44918a28-652b-430d-ac7e-53daadb0e69b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":2,"started_at":"2026-07-23T06:31:10.595Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"048ef068d6abc7e544421421e151018f4495f28c24194b2c149e5b5fdc19a19f","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"%28%22artificial%20intelligence%22%20OR%20%22generative%20AI%22%20OR%20OpenAI%20OR%20Anthropic%29","from":"2026-07-16T00:00:00","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:31:44.443Z","source_record_sha256":"7d26eae6c76e79caa8b781a941faa89d732adc2180de4aca7b4a3ee2e1040558"} +{"schema_version":1,"benchmark_version":"v1","run_id":"a306ce69-4682-4778-a8e5-47b0c598e895","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":3,"started_at":"2026-07-23T06:31:44.443Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"86c0120a22c9e323afefb1ad57da1301f45e352b84c74122ca3bdcbbbd59ffb7","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20%22generative%20AI%22","from":"2026-07-16","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:32:18.995Z","source_record_sha256":"42521c23757219360713aa5afcf829bf3b9f312d45a0d82f0534446f5c0fe40a"} +{"schema_version":1,"benchmark_version":"v1","run_id":"9377cc54-be7e-42b4-bf14-f4d14d013c54","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T06:32:18.995Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"301a7dca171878182649e610f180ae3c6f7db7f92485847b354714b924f969e4","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:32:52.395Z","source_record_sha256":"44b9472e368322522d38d3d194dc4a259d7d86e92fd6b3ebc65af3335e3d8fbf"} +{"schema_version":1,"benchmark_version":"v1","run_id":"c7815fa0-d6c8-4346-8463-1ae287470639","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T06:32:52.395Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"301a7dca171878182649e610f180ae3c6f7db7f92485847b354714b924f969e4","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:33:23.255Z","source_record_sha256":"1706ab297ad8e5469871e4c5dabc845a2a6bc64322945ea48954e30d43e3f099"} +{"schema_version":1,"benchmark_version":"v1","run_id":"844e8bbf-ae65-4047-9349-cbd1c1bf7b50","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T06:33:23.255Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"301a7dca171878182649e610f180ae3c6f7db7f92485847b354714b924f969e4","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:33:51.402Z","source_record_sha256":"7d86a9a686125488ce2a9c9827e7d8615a48aa4589c6f196894dec97f271cde3"} +{"schema_version":1,"benchmark_version":"v1","run_id":"cc32c600-4ef5-4fd8-b37c-6ae16c084f7e","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T06:33:51.402Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"850daeb30805dbaca59081825c8550458c4087938c88b78fb51cbc9e06aa928f","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:34:27.740Z","source_record_sha256":"e9825ffa157918d1b5bc4d9a98b4e2287745715b9d48bfb78f75999a5729c28d"} +{"schema_version":1,"benchmark_version":"v1","run_id":"de6093e2-ce59-4692-ae3c-da0801ac1675","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T06:34:27.740Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"850daeb30805dbaca59081825c8550458c4087938c88b78fb51cbc9e06aa928f","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:34:55.284Z","source_record_sha256":"14985dc8546323241dd9218ff7d0d7d60342c29e2f67882fc90ca3c9fdb9d946"} +{"schema_version":1,"benchmark_version":"v1","run_id":"a66e8303-b561-4d85-a2fa-baf46e0a953b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T06:34:55.284Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"850daeb30805dbaca59081825c8550458c4087938c88b78fb51cbc9e06aa928f","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:35:32.765Z","source_record_sha256":"187dfceaab10938e6587cda861b728789d536bf3b28d28b0cd4ba929ae1283d4"} +{"schema_version":1,"benchmark_version":"v1","run_id":"28718552-be81-486c-b313-75a5550ff3cc","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T06:35:32.765Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5783235b11728761fe41459111710a4cb2120d0d39e45f52a0cf8924ce54f453","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京","output":"JSON"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:36:03.749Z","source_record_sha256":"c6b25e36cd3b6ba2ed3b94a5041676171634d2ea7861d065351eab66243722ae"} +{"schema_version":1,"benchmark_version":"v1","run_id":"26e071b2-d139-4ee3-879c-5900578b1233","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T06:36:03.749Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5783235b11728761fe41459111710a4cb2120d0d39e45f52a0cf8924ce54f453","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:36:38.806Z","source_record_sha256":"bc95771cdcc075b15bbab52498067471e1ec7bf88d9d018f6906c25212f49e33"} +{"schema_version":1,"benchmark_version":"v1","run_id":"e2f6f4c2-a8e7-4967-98b1-1a8c29aa3410","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T06:36:38.806Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5783235b11728761fe41459111710a4cb2120d0d39e45f52a0cf8924ce54f453","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:37:14.052Z","source_record_sha256":"17081aa679bee8e6eea626e26319507fcf79576ed5dea927e450a7f1ec7a3d7a"} +{"schema_version":1,"benchmark_version":"v1","run_id":"fde5c3d4-8ff3-4813-a8c5-5e6a61ed3c2e","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-23T06:37:14.052Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"554e0d179f358300caff88da68d3aa9a452bb6b0b2fd0c9b3d7c4e72105773bf","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false},"finished_at":"2026-07-23T06:37:44.040Z","source_record_sha256":"f91cf8ce9139290d5c6da675f04813d6e6ee21b29963bfe35b8997feb17074b6"} +{"schema_version":1,"benchmark_version":"v1","run_id":"cb085acd-e437-40df-9e07-e91343e2af9d","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-23T06:37:44.040Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"554e0d179f358300caff88da68d3aa9a452bb6b0b2fd0c9b3d7c4e72105773bf","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false},"finished_at":"2026-07-23T06:38:14.212Z","source_record_sha256":"bf745350830585628ac0756e443a10066e9bb18c4ef7b5b5a4a19b4595e9e1bf"} +{"schema_version":1,"benchmark_version":"v1","run_id":"143359c4-589b-437f-ae30-9a99b9f1636b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-23T06:38:14.212Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"554e0d179f358300caff88da68d3aa9a452bb6b0b2fd0c9b3d7c4e72105773bf","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false},"finished_at":"2026-07-23T06:38:55.364Z","source_record_sha256":"f4d2ba49e80c77e7e6ea19c287b2c01827f9ae5a37900e71462cdfabf3d2ca97"} +{"schema_version":1,"benchmark_version":"v1","run_id":"58e4b1fd-8de8-4f85-8db1-478b57309b00","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":1,"started_at":"2026-07-23T06:38:55.364Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d6fafa5dcc5b80de51ca0ad0cd3e975827e3281c0aa5dbb07e57ede5fb7c75d6","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:39:36.286Z","source_record_sha256":"b3b701d6ebe68b0c1cc8c50a5cb2d0dd1f99fd925aab1c5195e0757bd216732f"} +{"schema_version":1,"benchmark_version":"v1","run_id":"f498a668-e254-4a1f-a109-0b11043a9a77","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":2,"started_at":"2026-07-23T06:39:36.286Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d6fafa5dcc5b80de51ca0ad0cd3e975827e3281c0aa5dbb07e57ede5fb7c75d6","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:40:06.523Z","source_record_sha256":"0b053eb5eb7799ea2e73b49aae1f3c395b57cb29bb4357530c73f123d47db293"} +{"schema_version":1,"benchmark_version":"v1","run_id":"3b48193b-6026-491e-8472-51034ae93749","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":3,"started_at":"2026-07-23T06:40:06.523Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d6fafa5dcc5b80de51ca0ad0cd3e975827e3281c0aa5dbb07e57ede5fb7c75d6","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:40:52.281Z","source_record_sha256":"ed953a426b8091ac61b9d3f826d950ca92c83ece434f088c6a5144b0378dd77d"} +{"schema_version":1,"benchmark_version":"v1","run_id":"3d70bc1e-3063-41e6-8b57-40c6e6b963e2","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":1,"started_at":"2026-07-23T06:40:52.281Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7785fb29a5fc11f07890b56ab20c7fc1235e8dca0651ca769eaf65c71fb42f5f","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:41:28.119Z","source_record_sha256":"a95a691983bf0e0c7d529f7562a557be3507430d675a4bd3d839df3216f6a5e9"} +{"schema_version":1,"benchmark_version":"v1","run_id":"39c78597-f3db-4d54-b56e-9ed6e6b3d412","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":2,"started_at":"2026-07-23T06:41:28.119Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7785fb29a5fc11f07890b56ab20c7fc1235e8dca0651ca769eaf65c71fb42f5f","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:42:00.561Z","source_record_sha256":"a0b6604983a5cb7cb672c3b78d0ab62371334c2073304ac0557bc2198840ad9f"} +{"schema_version":1,"benchmark_version":"v1","run_id":"cf50c76f-316f-4793-877b-2f51474c6981","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":3,"started_at":"2026-07-23T06:42:00.561Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1cedb5127086b90b58363d555e4053bea288d9ebbab0631b1e0947958d156844","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:42:34.259Z","source_record_sha256":"b785d59be2fe7f270be2cf59048ad17214b13a2d6387d0d28a2176dfda9a7d91"} +{"schema_version":1,"benchmark_version":"v1","run_id":"5bf899ee-7846-44cf-8e8a-ebeb8bfbff65","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":1,"started_at":"2026-07-23T06:42:34.259Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:43:06.453Z","source_record_sha256":"4f635a3e21f7ec6a43b30500f049ff3208cce239ac1be3ace62b2c03ff541897"} +{"schema_version":1,"benchmark_version":"v1","run_id":"673ccda1-c286-49cd-9299-75ce17706e92","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":2,"started_at":"2026-07-23T06:43:06.453Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:43:43.650Z","source_record_sha256":"d6f433e283139085ada4f2415ec556e220ec66bd7f11fc504f2c51f0584e89c5"} +{"schema_version":1,"benchmark_version":"v1","run_id":"859aee1a-1670-4140-b9ac-91435124e6dc","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":3,"started_at":"2026-07-23T06:43:43.650Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:44:12.213Z","source_record_sha256":"db5ddc2bef71959a0abfe67ed9111c89e5d7a91db808692068481910ae1c49a2"} +{"schema_version":1,"benchmark_version":"v1","run_id":"70a47304-ae9e-4cb5-8331-5df6a8e83a43","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":1,"started_at":"2026-07-23T06:44:12.213Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e5da17c29aa94866e3abc93b7ae7169f66e151ae69893d952e89fcee78460d21","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:44:39.351Z","source_record_sha256":"3f8d52a2dd4bbf58e9f9be36f1e434b694eb757da69d8159f0243888a598eb4b"} +{"schema_version":1,"benchmark_version":"v1","run_id":"73355630-bc92-4e0a-a1c8-4f25f895a949","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":2,"started_at":"2026-07-23T06:44:39.351Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e5da17c29aa94866e3abc93b7ae7169f66e151ae69893d952e89fcee78460d21","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:45:14.542Z","source_record_sha256":"3a19bbc3b8533c11da052cc57dac3f7c2d2f5f04a555da0668605d0369dd1b6e"} +{"schema_version":1,"benchmark_version":"v1","run_id":"f05cea11-e23c-463a-98f1-e5c0b4f51e88","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":3,"started_at":"2026-07-23T06:45:14.542Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e5da17c29aa94866e3abc93b7ae7169f66e151ae69893d952e89fcee78460d21","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:45:40.767Z","source_record_sha256":"b1f8a12fd2910883acab983ca94f2d489b62427c0daa9759d3ed850c590cde0d"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ae3e60aa-8ee9-4650-b3e2-6d906a42da8b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T06:45:40.767Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","message":"Adapter invocation failed"},"finished_at":"2026-07-23T06:46:30.257Z","source_record_sha256":"a662ab99b03038d4da9d8e89149c4d3180ec80a1ddba685c896089c14c660114"} +{"schema_version":1,"benchmark_version":"v1","run_id":"cc1eebe6-e68b-45ba-8f32-827f98baa240","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T06:46:30.257Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","message":"Adapter invocation failed"},"finished_at":"2026-07-23T06:47:52.318Z","source_record_sha256":"e8e64e193750c200d9940be5a99c552cad0ecb01e495121124aa07804da7b2d0"} +{"schema_version":1,"benchmark_version":"v1","run_id":"19382342-d937-46dc-ba4a-c71139081673","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T06:47:52.318Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","message":"Adapter invocation failed"},"finished_at":"2026-07-23T06:48:55.681Z","source_record_sha256":"487eeb948773c4df6e60d2333f6e54cfa05a7e143dbd87ea81953b8cfcf02f6f"} +{"schema_version":1,"benchmark_version":"v1","run_id":"48e281c3-3a7a-4327-9911-427072bc6fcd","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":1,"started_at":"2026-07-23T06:48:55.681Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"47a10fc51eaffa7d634cfeb673e3977424960edb526cf36296eab022d673e074","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:49:28.588Z","source_record_sha256":"8429545504e4b2eef7082c3c1fce7c561bd1be1de082be34fd96e2c5149690c5"} +{"schema_version":1,"benchmark_version":"v1","run_id":"d23e1fc9-03a5-4c76-8761-22c9de0e6231","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":2,"started_at":"2026-07-23T06:49:28.588Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"47a10fc51eaffa7d634cfeb673e3977424960edb526cf36296eab022d673e074","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":10,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:49:55.653Z","source_record_sha256":"7164bd4f310c9dda3a27e9ff046018d44184c359ec7c7f0db3880e16476eee65"} +{"schema_version":1,"benchmark_version":"v1","run_id":"262b0d23-181c-4b1c-8241-8cfd530dbb96","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":3,"started_at":"2026-07-23T06:49:55.653Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"47a10fc51eaffa7d634cfeb673e3977424960edb526cf36296eab022d673e074","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:50:22.831Z","source_record_sha256":"8620cabf967667758d4370970f8e6d82026af991bf28e2b2dfbf7fc602d9c6b1"} +{"schema_version":1,"benchmark_version":"v1","run_id":"3d110e2a-4ee1-47f7-a018-f5ae9d166e46","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":1,"started_at":"2026-07-23T06:50:22.831Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5d74987403b7f6009ae5a6da3cdcfac27dc35aa8f8dc5c333e65725c97f0cd0a","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:50:57.424Z","source_record_sha256":"1862e1000eaaa7832e064d1857baa8f7ecbe97a1739265e5b4666a0a4622ec05"} +{"schema_version":1,"benchmark_version":"v1","run_id":"b619dc10-8cc0-4413-9219-1aea84479b72","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":2,"started_at":"2026-07-23T06:50:57.424Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5d74987403b7f6009ae5a6da3cdcfac27dc35aa8f8dc5c333e65725c97f0cd0a","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:51:30.279Z","source_record_sha256":"c85b8d8d773950ae3a9274f3b0bb65b300c1639dc132b34c2a053531392c8a6c"} +{"schema_version":1,"benchmark_version":"v1","run_id":"0ab6d79a-6510-4b11-966c-7063b6646f73","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":3,"started_at":"2026-07-23T06:51:30.279Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5d74987403b7f6009ae5a6da3cdcfac27dc35aa8f8dc5c333e65725c97f0cd0a","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:51:58.663Z","source_record_sha256":"5c935b04a6e98677b01766c543ce5eea027cdf4b2f27d6f12677d78d1bbfa0a7"} +{"schema_version":1,"benchmark_version":"v1","run_id":"245b91ee-988d-474f-8fd1-82da7c62183a","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-23T06:51:58.663Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:52:30.880Z","source_record_sha256":"d86e68f4637cad728737fb4b0e4283ba4766a64304df4ed5b0843bb5ffc8185a"} +{"schema_version":1,"benchmark_version":"v1","run_id":"dc38691f-d8f6-40f8-9e77-9288c33a7e68","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-23T06:52:30.881Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:53:04.476Z","source_record_sha256":"c97ae4740814eb914f3859c0d16a16dec1298fa3ff5caf4ae386299734840da7"} +{"schema_version":1,"benchmark_version":"v1","run_id":"c1624700-5aba-4cfc-8a68-acabf3de024a","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T06:53:04.476Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","message":"Adapter invocation failed"},"finished_at":"2026-07-23T06:55:18.009Z","source_record_sha256":"d567ee1f534ced827b85bb21fc9c7c8b41325ea27130743f8e95d0396532f4bd"} +{"schema_version":1,"benchmark_version":"v1","run_id":"0e485876-22f8-4265-91e0-51990c3644e9","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":1,"started_at":"2026-07-23T06:55:18.009Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fa45aa3f97145ca8594d0a5ff1f0fba0f1abb533dfb4cfabbf84a07e4a76fba8","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false},"finished_at":"2026-07-23T06:55:45.466Z","source_record_sha256":"e8d8d305c2052be04c186b5fb864515ed6c6066c552f64a7df60e6b7aec061a6"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ea693fd1-b596-40b1-bd16-eec213eecb12","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":2,"started_at":"2026-07-23T06:55:45.466Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fa45aa3f97145ca8594d0a5ff1f0fba0f1abb533dfb4cfabbf84a07e4a76fba8","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false},"finished_at":"2026-07-23T06:56:19.074Z","source_record_sha256":"0ebe2c1c3adbf481010ee766b90c35e838b2c50bfd738c866806459e727f03b5"} +{"schema_version":1,"benchmark_version":"v1","run_id":"abf65b68-8d35-453f-89b6-d30447a432e7","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":3,"started_at":"2026-07-23T06:56:19.074Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fa45aa3f97145ca8594d0a5ff1f0fba0f1abb533dfb4cfabbf84a07e4a76fba8","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false},"finished_at":"2026-07-23T06:56:53.486Z","source_record_sha256":"83d6dc9000481394c87614ff3a0dff0fcf553b0b50f7e2f7b556a5f986c5db06"} +{"schema_version":1,"benchmark_version":"v1","run_id":"a6ccdda6-dc1f-4765-873e-bd8c22197a6f","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":1,"started_at":"2026-07-23T06:56:53.486Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fcf3a79fe980fdf6f8b86e5034754e48d6405528a3a58b79d4ce0c7533cc5a2d","selection_grounded":true},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"selection_grounded":true,"required_parameters":["season"]},"parameters":{"league":39,"season":2025},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:57:26.157Z","source_record_sha256":"a592538886a42e90179278a71fc1bbc54d59f461ad659fcf68c646f88bd8af74"} +{"schema_version":1,"benchmark_version":"v1","run_id":"a025004c-c94c-444b-a2c9-bb176ed51ea3","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":2,"started_at":"2026-07-23T06:57:26.157Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fcf3a79fe980fdf6f8b86e5034754e48d6405528a3a58b79d4ce0c7533cc5a2d","selection_grounded":true},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"selection_grounded":true,"required_parameters":["season"]},"parameters":{"league":39,"season":2025},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:57:50.791Z","source_record_sha256":"b633b44046b54479076c360986169b474cd47cfd3baffcab34502a7a1a143287"} +{"schema_version":1,"benchmark_version":"v1","run_id":"f15c0986-0c0c-4eab-8dac-99cc4f6f7dea","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":3,"started_at":"2026-07-23T06:57:50.791Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fcf3a79fe980fdf6f8b86e5034754e48d6405528a3a58b79d4ce0c7533cc5a2d","selection_grounded":true},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"selection_grounded":true,"required_parameters":["season"]},"parameters":{"league":39,"season":2025},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:58:17.129Z","source_record_sha256":"ac3e0c86a18c7046786aeddaaee62ddca5f592e7d47b05c1076884e962aeb5de"} diff --git a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.summary.json b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.summary.json index 016316d..ade5af6 100644 --- a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.summary.json +++ b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.summary.json @@ -1,10 +1,11 @@ { "schema_version": 1, "methodology": "discover-call-v1", - "generated_at": "2026-07-23T06:58:29.076Z", + "generated_at": "2026-07-23T11:10:34.155Z", "models": [ { "model": "gpt-5.6-sol", + "lane": "model", "tasks": 18, "trials_per_task": 3, "runs": 54, @@ -15,14 +16,16 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0.7593, "call_success_rate": 0.88, + "result_nonempty_rate": 0.88, "workflow_success_rate": 0.6481, - "workflow_success_wilson_95": [ - 0.5148, - 0.7618 + "workflow_success_task_cluster_bootstrap_95": [ + 0.4259, + 0.8519 ], "failures_by_stage": { "adapter": 4 - } + }, + "failures_by_reason": {} } ], "records": [ @@ -38,15 +41,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -61,15 +77,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -84,15 +113,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -107,15 +149,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -130,15 +185,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -153,15 +221,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -176,15 +257,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, + "result_nonempty": true, "workflow_success": false, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -199,15 +293,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, + "result_nonempty": true, "workflow_success": false, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -222,15 +329,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, + "result_nonempty": true, "workflow_success": false, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -245,15 +365,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, + "result_nonempty": true, "workflow_success": false, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -268,15 +401,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, + "result_nonempty": true, "workflow_success": false, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -291,15 +437,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, + "result_nonempty": true, "workflow_success": false, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -314,15 +473,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -337,15 +509,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -360,15 +545,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -383,15 +581,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, + "result_nonempty": true, "workflow_success": false, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -406,15 +617,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, + "result_nonempty": true, "workflow_success": false, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -429,15 +653,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, + "result_nonempty": true, "workflow_success": false, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -452,15 +689,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -475,15 +725,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -498,15 +761,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -521,15 +797,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": false, + "result_nonempty": false, "workflow_success": false, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -544,15 +833,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": false, + "result_nonempty": false, "workflow_success": false, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -567,15 +869,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": false, + "result_nonempty": false, "workflow_success": false, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -590,15 +905,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -613,15 +941,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -636,15 +977,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -659,15 +1013,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -682,15 +1049,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -705,15 +1085,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -728,15 +1121,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -751,15 +1157,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -774,15 +1193,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -797,15 +1229,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -820,15 +1265,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -843,15 +1301,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -866,15 +1337,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": null, + "result_nonempty": null, "workflow_success": false, "error_stage": "adapter", + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -889,15 +1373,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": null, + "result_nonempty": null, "workflow_success": false, "error_stage": "adapter", + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -912,15 +1409,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": null, + "result_nonempty": null, "workflow_success": false, "error_stage": "adapter", + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -935,15 +1445,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -958,15 +1481,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -981,15 +1517,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1004,15 +1553,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1027,15 +1589,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1050,15 +1625,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1073,15 +1661,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1096,15 +1697,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1119,15 +1733,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": null, + "result_nonempty": null, "workflow_success": false, "error_stage": "adapter", + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1142,15 +1769,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": false, + "result_nonempty": false, "workflow_success": false, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1165,15 +1805,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": false, + "result_nonempty": false, "workflow_success": false, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1188,15 +1841,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": false, + "result_nonempty": false, "workflow_success": false, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1211,15 +1877,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1234,15 +1913,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1257,15 +1949,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, + "error_reason": null, "metadata": { "adapter_revision": "72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium", "toolkit_revision": "fd08165e979a8ffaa030c56a4bd4853d805bf095", "task_set_sha256": "67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "lane": "model", + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } } ] diff --git a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.runs.jsonl b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.runs.jsonl index 6a66e29..7c27d0b 100644 --- a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.runs.jsonl +++ b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.runs.jsonl @@ -1,54 +1,54 @@ -{"schema_version":1,"benchmark_version":"v1","run_id":"23187609-52d7-44b6-b60e-91f68d32777c","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"weather-london","trial":1,"started_at":"2026-07-23T09:40:25.828Z","status":"completed","discovery":{"result_tool_ids":["weather_gov.gridpoints.forecast.retrieve.v1.40a9a3ce","weather_gov.gridpoints.forecast.retrieve.v3.29bbec80","amap_webservice.weather.weatherinfo.retrieve.v3","weather_gov.gridpoints.forecast.hourly.retrieve.v3.808ea240","weather_gov.icons.retrieve.v3.ad0b4d80","weather_gov.products.retrieve.v3.43ee57b8","visualcrossing.timeline.retrieve.v1","weather_gov.gridpoints.retrieve.v3.a0c89365","weather_gov.gridpoints.retrieve.v3.2cf46664","weather_gov.products.retrieve.v3.511d7df1"]},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"returned_tool_ids":["visualcrossing.timeline.retrieve.v1"],"required_parameters":["location"]},"parameters":{"location":"London"},"call":{"attempted":true,"success":true,"execution_id":"2cb5aa8e-91fa-49b8-a383-880e16acda3e","result_valid":true},"finished_at":"2026-07-23T09:40:51.455Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"d2bfdd97-1ac6-4388-aab6-dc04bd389b4d","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"weather-london","trial":2,"started_at":"2026-07-23T09:40:51.455Z","status":"completed","discovery":{"result_tool_ids":["weather_gov.gridpoints.forecast.retrieve.v1.40a9a3ce","weather_gov.gridpoints.forecast.retrieve.v3.29bbec80","amap_webservice.weather.weatherinfo.retrieve.v3","weather_gov.gridpoints.forecast.hourly.retrieve.v3.808ea240","weather_gov.icons.retrieve.v3.ad0b4d80","weather_gov.products.retrieve.v3.43ee57b8","visualcrossing.timeline.retrieve.v1","weather_gov.gridpoints.retrieve.v3.a0c89365","weather_gov.gridpoints.retrieve.v3.2cf46664","weather_gov.products.retrieve.v3.511d7df1"]},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"returned_tool_ids":["visualcrossing.timeline.retrieve.v1"],"required_parameters":["location"]},"parameters":{"location":"London"},"call":{"attempted":true,"success":true,"execution_id":"cc681d1e-1538-4244-b3ac-ed123a0e37c4","result_valid":true},"finished_at":"2026-07-23T09:41:19.367Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"9dc77ec7-6fea-42e6-8786-f8970a33da5d","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"weather-london","trial":3,"started_at":"2026-07-23T09:41:19.367Z","status":"completed","discovery":{"result_tool_ids":["weather_gov.gridpoints.forecast.retrieve.v1.40a9a3ce","weather_gov.gridpoints.forecast.retrieve.v3.29bbec80","amap_webservice.weather.weatherinfo.retrieve.v3","weather_gov.gridpoints.forecast.hourly.retrieve.v3.808ea240","weather_gov.icons.retrieve.v3.ad0b4d80","weather_gov.products.retrieve.v3.43ee57b8","visualcrossing.timeline.retrieve.v1","weather_gov.gridpoints.retrieve.v3.a0c89365","weather_gov.gridpoints.retrieve.v3.2cf46664","weather_gov.products.retrieve.v3.511d7df1"]},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"returned_tool_ids":["visualcrossing.timeline.retrieve.v1"],"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"execution_id":"5c4cd3ab-c6c5-4cbe-b494-c60d8e290c44","result_valid":true},"finished_at":"2026-07-23T09:41:45.578Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"98d4488a-8f0e-48fe-8743-ed08df8940f8","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-23T09:41:45.578Z","status":"completed","discovery":{"result_tool_ids":["eodhd.live_v2.us_quote_delayed.retrieve.v1.f0e13d45","twelvedata.quote.retrieve.v1.affbefe3","massive_stocks.options.reference.conditions.list.v3.41a82e07","alphavantage.realtime_bulk_quotes.retrieve.v1","fiu_mcp_server.postv1stockquoteextend.create.v2.a30e86e7","hangseng_polysource.a_shares_live_quote.query.v2.10fe0581","mcp_gildata.usstockdailyquotes.v1","financialmodelingprep.cusip.search.v1.ad19189f","fiu_mcp_server.postv3stockquoteextend.create.v2.b35a7f02","massive_stocks.stocks.marketstatus.retrieve.v1.3045ae6e"]},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"returned_tool_ids":["twelvedata.quote.retrieve.v1.affbefe3"],"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"20ac0b50-e5e7-45fe-94da-dae66e4faa18","result_valid":true},"finished_at":"2026-07-23T09:42:10.683Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"450cf5b6-8404-4be2-b73e-c28a6ae3227d","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-23T09:42:10.683Z","status":"completed","discovery":{"result_tool_ids":["eodhd.live_v2.us_quote_delayed.retrieve.v1.f0e13d45","twelvedata.quote.retrieve.v1.affbefe3","massive_stocks.options.reference.conditions.list.v3.41a82e07","alphavantage.realtime_bulk_quotes.retrieve.v1","fiu_mcp_server.postv1stockquoteextend.create.v2.a30e86e7","hangseng_polysource.a_shares_live_quote.query.v2.10fe0581","mcp_gildata.usstockdailyquotes.v1","financialmodelingprep.cusip.search.v1.ad19189f","fiu_mcp_server.postv3stockquoteextend.create.v2.b35a7f02","massive_stocks.stocks.marketstatus.retrieve.v1.3045ae6e"]},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"returned_tool_ids":["twelvedata.quote.retrieve.v1.affbefe3"],"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"a8db7a5c-7f32-4569-ab8d-2f7e913ab169","result_valid":true},"finished_at":"2026-07-23T09:42:35.790Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"456b1477-ec0a-4772-887c-8971ac437a95","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-23T09:42:35.790Z","status":"completed","discovery":{"result_tool_ids":["eodhd.live_v2.us_quote_delayed.retrieve.v1.f0e13d45","twelvedata.quote.retrieve.v1.affbefe3","massive_stocks.options.reference.conditions.list.v3.41a82e07","alphavantage.realtime_bulk_quotes.retrieve.v1","fiu_mcp_server.postv1stockquoteextend.create.v2.a30e86e7","hangseng_polysource.a_shares_live_quote.query.v2.10fe0581","mcp_gildata.usstockdailyquotes.v1","financialmodelingprep.cusip.search.v1.ad19189f","fiu_mcp_server.postv3stockquoteextend.create.v2.b35a7f02","massive_stocks.stocks.marketstatus.retrieve.v1.3045ae6e"]},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"returned_tool_ids":["twelvedata.quote.retrieve.v1.affbefe3"],"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"6bc11716-c8cd-4a51-b753-21e5c544fd76","result_valid":true},"finished_at":"2026-07-23T09:43:04.020Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"850f6ae9-de46-4e92-a722-81edd34dd191","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-23T09:43:04.020Z","status":"completed","discovery":{"result_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4","twelvedata.exchangerate.retrieve.v1.9eeb3b0d","coinmarketcap.tools.priceconversion.retrieve.v2.24e991d9","bybitp2p.payment.fx.convert.v5.7883b755","coingecko_pro.exchangerates.retrieve.v3.0f6133ea","bybitp2p.pay.fx.convert.v5.9221b0c4","coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","coinmarketcap.tools.priceconversion.retrieve.v2.52e4cb2d","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d","nbp_pl.exchangerates.rates.last.list.v1"]},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"returned_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4"],"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":100,"format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"47709f24-24dc-4a0e-a4d1-693fdd193e2f","result_valid":true},"finished_at":"2026-07-23T09:43:40.323Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"378c4fcc-1e86-4149-8b38-82d8230e5d11","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-23T09:43:40.323Z","status":"completed","discovery":{"result_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4","twelvedata.exchangerate.retrieve.v1.9eeb3b0d","coinmarketcap.tools.priceconversion.retrieve.v2.24e991d9","bybitp2p.payment.fx.convert.v5.7883b755","coingecko_pro.exchangerates.retrieve.v3.0f6133ea","bybitp2p.pay.fx.convert.v5.9221b0c4","coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","coinmarketcap.tools.priceconversion.retrieve.v2.52e4cb2d","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d","nbp_pl.exchangerates.rates.last.list.v1"]},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"returned_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4"],"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"8564bc40-9c95-4f99-9ef3-1388a755e461","result_valid":true},"finished_at":"2026-07-23T09:44:16.949Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"beb58cdc-281f-42cf-a388-9d9f30fa74be","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-23T09:44:16.949Z","status":"completed","discovery":{"result_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4","twelvedata.exchangerate.retrieve.v1.9eeb3b0d","coinmarketcap.tools.priceconversion.retrieve.v2.24e991d9","bybitp2p.payment.fx.convert.v5.7883b755","coingecko_pro.exchangerates.retrieve.v3.0f6133ea","bybitp2p.pay.fx.convert.v5.9221b0c4","coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","coinmarketcap.tools.priceconversion.retrieve.v2.52e4cb2d","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d","nbp_pl.exchangerates.rates.last.list.v1"]},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"returned_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4"],"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"ef1b196f-77fb-4f9b-b49f-504c5ff96748","result_valid":true},"finished_at":"2026-07-23T09:44:53.121Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"b602eea9-1fa5-4b43-adf2-59adff282a87","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"news-ai","trial":1,"started_at":"2026-07-23T09:44:53.121Z","status":"completed","discovery":{"result_tool_ids":["x_developer.2.news.search.retrieve.v1.19b3e678","newsdata.news.search.v1.b65ccc56","x_developer.2.tweets.search.recent.retrieve.v2.f424c8f9","qveris_finance.finance_news_aggregation_v1","jina_search.rerank.create.v2026.23dc486d","linkup.search.v1","newsapi.top_headlines.list.v2.38face94","newsapi.everything.retrieve.v2.e77ab1df","brave_search.news.search.list.v1.6ab52eaa","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic"]},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"returned_tool_ids":["newsapi.everything.retrieve.v2.e77ab1df"],"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20AI","from":"2026-07-16","to":"2026-07-23T23:59:59+08:00","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"execution_id":"f9669577-aab3-42e1-acbe-a311c92d0040","result_valid":true},"finished_at":"2026-07-23T09:45:25.145Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"bfc0ab6d-9a15-4768-a672-e466f0d23414","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"news-ai","trial":2,"started_at":"2026-07-23T09:45:25.145Z","status":"completed","discovery":{"result_tool_ids":["x_developer.2.news.search.retrieve.v1.19b3e678","newsdata.news.search.v1.b65ccc56","x_developer.2.tweets.search.recent.retrieve.v2.f424c8f9","qveris_finance.finance_news_aggregation_v1","jina_search.rerank.create.v2026.23dc486d","linkup.search.v1","newsapi.top_headlines.list.v2.38face94","newsapi.everything.retrieve.v2.e77ab1df","brave_search.news.search.list.v1.6ab52eaa","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic"]},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"returned_tool_ids":["newsapi.everything.retrieve.v2.e77ab1df"],"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20AI","from":"2026-07-16","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"execution_id":"60c01932-2aeb-4bf1-89d0-afb8ada449a8","result_valid":true},"finished_at":"2026-07-23T09:46:05.132Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"c2bab891-e16c-49ce-af35-9ba5220ef929","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"news-ai","trial":3,"started_at":"2026-07-23T09:46:05.132Z","status":"completed","discovery":{"result_tool_ids":["x_developer.2.news.search.retrieve.v1.19b3e678","newsdata.news.search.v1.b65ccc56","x_developer.2.tweets.search.recent.retrieve.v2.f424c8f9","qveris_finance.finance_news_aggregation_v1","jina_search.rerank.create.v2026.23dc486d","linkup.search.v1","newsapi.top_headlines.list.v2.38face94","newsapi.everything.retrieve.v2.e77ab1df","brave_search.news.search.list.v1.6ab52eaa","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic"]},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"returned_tool_ids":["newsapi.everything.retrieve.v2.e77ab1df"],"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20%22generative%20AI%22","from":"2026-07-20","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"execution_id":"ea44ee78-ac4d-42c2-957a-a9cfe5f5d904","result_valid":true},"finished_at":"2026-07-23T09:46:35.462Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"7c3b1d86-cdf3-42aa-94d0-bfa2024284ec","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T09:46:35.462Z","status":"completed","discovery":{"result_tool_ids":["semantic_scholar.paper.search.bulk.execute.v1.239dfd66","qveris_academic.paper_aggregation_v1","semantic_scholar.paper.search.bulk.retrieve.v1.096de627","semantic_scholar.paper.search.match.retrieve.v1.d3fd06f6","semantic_scholar.paper.search.match.retrieve.v1.b9f166f3","semantic_scholar.paper.autocomplete.retrieve.v1.e16c9798","semantic_scholar.paper.autocomplete.list.v1.99d870f6","xiaosu.scholar.search.v1.f79d2374","semantic_scholar.author.search.query.v1.1c428051","semantic_scholar.author.search.list.v1.ea3671fb"]},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"returned_tool_ids":["qveris_academic.paper_aggregation_v1"],"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"execution_id":"c78b5f36-0aa8-4732-945f-0c7ae03e33d0","result_valid":true},"finished_at":"2026-07-23T09:47:02.518Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"0dfd6d1c-ca90-4916-8b8f-4ae4d3e7f273","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T09:47:02.518Z","status":"completed","discovery":{"result_tool_ids":["semantic_scholar.paper.search.bulk.execute.v1.239dfd66","qveris_academic.paper_aggregation_v1","semantic_scholar.paper.search.bulk.retrieve.v1.096de627","semantic_scholar.paper.search.match.retrieve.v1.d3fd06f6","semantic_scholar.paper.search.match.retrieve.v1.b9f166f3","semantic_scholar.paper.autocomplete.retrieve.v1.e16c9798","semantic_scholar.paper.autocomplete.list.v1.99d870f6","xiaosu.scholar.search.v1.f79d2374","semantic_scholar.author.search.query.v1.1c428051","semantic_scholar.author.search.list.v1.ea3671fb"]},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"returned_tool_ids":["qveris_academic.paper_aggregation_v1"],"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"execution_id":"c93cbab8-c8de-4eb6-8352-6ad43723e538","result_valid":true},"finished_at":"2026-07-23T09:47:32.328Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"ce6fbc71-c894-461a-adb1-4a607e7d4bdc","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T09:47:32.328Z","status":"completed","discovery":{"result_tool_ids":["semantic_scholar.paper.search.bulk.execute.v1.239dfd66","qveris_academic.paper_aggregation_v1","semantic_scholar.paper.search.bulk.retrieve.v1.096de627","semantic_scholar.paper.search.match.retrieve.v1.d3fd06f6","semantic_scholar.paper.search.match.retrieve.v1.b9f166f3","semantic_scholar.paper.autocomplete.retrieve.v1.e16c9798","semantic_scholar.paper.autocomplete.list.v1.99d870f6","xiaosu.scholar.search.v1.f79d2374","semantic_scholar.author.search.query.v1.1c428051","semantic_scholar.author.search.list.v1.ea3671fb"]},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"returned_tool_ids":["qveris_academic.paper_aggregation_v1"],"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"execution_id":"785d5fb3-a7ae-44ad-902b-a09ad5db2659","result_valid":true},"finished_at":"2026-07-23T09:47:58.422Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"b43508b4-6a54-487e-b3fd-dd511201af19","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T09:47:58.422Z","status":"completed","discovery":{"result_tool_ids":["api_sports.timezone.retrieve.v1.e993615d","api_sports.timezone.retrieve.v1.6792e6aa","api_sports.timezone.retrieve.v1.9b882d11","api_sports.timezone.retrieve.v1.00ab54ff","api_sports.timezone.retrieve.v1.ae6ae8d7","api_sports.timezone.retrieve.v1.64cfc345","api_sports.timezone.list.v1.bd878c58","api_sports.timezone.list.v1.fafec612","api_sports.timezone.list.v1.58366f46","api_sports.timezone.retrieve.v1.895e2257"]},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"returned_tool_ids":["api_sports.timezone.retrieve.v1.e993615d"],"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"execution_id":"499009d1-ebcc-470c-af61-581ca41f69ff","result_valid":true},"finished_at":"2026-07-23T09:48:24.635Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"704ee017-af2e-469b-a055-de7b367c3ffb","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T09:48:24.635Z","status":"completed","discovery":{"result_tool_ids":["api_sports.timezone.retrieve.v1.e993615d","api_sports.timezone.retrieve.v1.6792e6aa","api_sports.timezone.retrieve.v1.9b882d11","api_sports.timezone.retrieve.v1.00ab54ff","api_sports.timezone.retrieve.v1.ae6ae8d7","api_sports.timezone.retrieve.v1.64cfc345","api_sports.timezone.list.v1.bd878c58","api_sports.timezone.list.v1.fafec612","api_sports.timezone.list.v1.58366f46","api_sports.timezone.retrieve.v1.895e2257"]},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"returned_tool_ids":["api_sports.timezone.retrieve.v1.e993615d"],"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"execution_id":"527fbd2d-6eac-4400-8274-f30654f8f0f2","result_valid":true},"finished_at":"2026-07-23T09:48:55.300Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"495bda63-87f8-4ada-bcd2-a2a4076f6431","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T09:48:55.300Z","status":"completed","discovery":{"result_tool_ids":["api_sports.timezone.retrieve.v1.e993615d","api_sports.timezone.retrieve.v1.6792e6aa","api_sports.timezone.retrieve.v1.9b882d11","api_sports.timezone.retrieve.v1.00ab54ff","api_sports.timezone.retrieve.v1.ae6ae8d7","api_sports.timezone.retrieve.v1.64cfc345","api_sports.timezone.list.v1.bd878c58","api_sports.timezone.list.v1.fafec612","api_sports.timezone.list.v1.58366f46","api_sports.timezone.retrieve.v1.895e2257"]},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"returned_tool_ids":["api_sports.timezone.retrieve.v1.e993615d"],"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"execution_id":"18d67dfc-1fcd-417a-9892-44647cd21c74","result_valid":true},"finished_at":"2026-07-23T09:49:26.689Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"10a88d8d-1053-4d50-bab5-4b7114dd0ee7","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T09:49:26.689Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.geocoder.reverse.v2","baidu_map.geocoder.query.v2","baidu_map.geocoding.reverse_geocode.v1","baidu_map.reversegeocoding.create.v1","baidu_map.reverse_geocoding.query.v2","amap_webservice.geocode.geo.retrieve.v3","baidu_map.geocoder.wgeoreverse.create.v1","baidu_map.geocoder.wgeobatch.create.v1","arcgis_ocean.geocoding.reverse_geocode.v1.4d40d93e","baidu_map.geoconv.convert.v1"]},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"returned_tool_ids":["amap_webservice.geocode.geo.retrieve.v3"],"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京","output":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"ad2703a5-83ed-49b6-a202-1d4023478acd","result_valid":true},"finished_at":"2026-07-23T09:49:55.414Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"252d6b99-8605-4a66-aaec-d2b63a4fe36f","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T09:49:55.414Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.geocoder.reverse.v2","baidu_map.geocoder.query.v2","baidu_map.geocoding.reverse_geocode.v1","baidu_map.reversegeocoding.create.v1","baidu_map.reverse_geocoding.query.v2","amap_webservice.geocode.geo.retrieve.v3","baidu_map.geocoder.wgeoreverse.create.v1","baidu_map.geocoder.wgeobatch.create.v1","arcgis_ocean.geocoding.reverse_geocode.v1.4d40d93e","baidu_map.geoconv.convert.v1"]},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"returned_tool_ids":["amap_webservice.geocode.geo.retrieve.v3"],"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京","output":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"ce188a4a-8f86-4f1b-8152-d0a5a1e621b1","result_valid":true},"finished_at":"2026-07-23T09:50:31.198Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"3b2b6bf9-ad89-4298-92a7-3622fb115a13","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T09:50:31.198Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.geocoder.reverse.v2","baidu_map.geocoder.query.v2","baidu_map.geocoding.reverse_geocode.v1","baidu_map.reversegeocoding.create.v1","baidu_map.reverse_geocoding.query.v2","amap_webservice.geocode.geo.retrieve.v3","baidu_map.geocoder.wgeoreverse.create.v1","baidu_map.geocoder.wgeobatch.create.v1","arcgis_ocean.geocoding.reverse_geocode.v1.4d40d93e","baidu_map.geoconv.convert.v1"]},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"returned_tool_ids":["amap_webservice.geocode.geo.retrieve.v3"],"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京","output":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"27e3f46b-504e-4d9e-8ba5-a6413bb3abf4","result_valid":true},"finished_at":"2026-07-23T09:51:04.973Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"aaac2eb2-646e-42ff-ae32-1326ee9682b4","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-23T09:51:04.973Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.location.ip.retrieve.v1","decodo.isp_pay_per_ip.ip.retrieve.v1.bc2c6371","baidu_map.location.ip.execute.v1","amap_webservice.ip.retrieve.v3","virustotal.ip.info.retrieve.v3.ab34b0d1","weather_api.ip_lookup.retrieve.v1.e095d904","virustotal.ip.report.retrieve.v3.ab34b0d1","baidu_map.ip_location.retrieve.v1","weather_api.search.list.v1.73de2282","virustotal.ip-address.report.retrieve.v2.79606e33"]},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"returned_tool_ids":["weather_api.ip_lookup.retrieve.v1.e095d904"],"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"execution_id":"0dfbbf4f-3249-4340-9c3b-fdf6a09fb1f4","result_valid":false},"finished_at":"2026-07-23T09:51:37.056Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"3e947ba6-ca5a-47dc-abe3-404e9464239b","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-23T09:51:37.056Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.location.ip.retrieve.v1","decodo.isp_pay_per_ip.ip.retrieve.v1.bc2c6371","baidu_map.location.ip.execute.v1","amap_webservice.ip.retrieve.v3","virustotal.ip.info.retrieve.v3.ab34b0d1","weather_api.ip_lookup.retrieve.v1.e095d904","baidu_map.ip_location.retrieve.v1","virustotal.ip-address.report.retrieve.v2.79606e33","virustotal.ip_addresses.retrieve.v3.ab34b0d1","weather_api.search.list.v1.73de2282"]},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"returned_tool_ids":["weather_api.ip_lookup.retrieve.v1.e095d904"],"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"execution_id":"29664b6b-1928-4a74-9e22-7d3213840dd1","result_valid":false},"finished_at":"2026-07-23T09:52:06.521Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"69e303d3-d9ea-4f86-9fa3-0b4bd9c3087c","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-23T09:52:06.521Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.location.ip.retrieve.v1","decodo.isp_pay_per_ip.ip.retrieve.v1.bc2c6371","baidu_map.location.ip.execute.v1","amap_webservice.ip.retrieve.v3","virustotal.ip.info.retrieve.v3.ab34b0d1","weather_api.ip_lookup.retrieve.v1.e095d904","baidu_map.ip_location.retrieve.v1","virustotal.ip-address.report.retrieve.v2.79606e33","virustotal.ip_addresses.retrieve.v3.ab34b0d1","weather_api.search.list.v1.73de2282"]},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"returned_tool_ids":["weather_api.ip_lookup.retrieve.v1.e095d904"],"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"execution_id":"885aef06-36b7-4c20-af1d-57100415e97e","result_valid":false},"finished_at":"2026-07-23T09:52:38.206Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"16873a4e-2cd3-47d9-845e-0067aeb15a42","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"crypto-price","trial":1,"started_at":"2026-07-23T09:52:38.206Z","status":"completed","discovery":{"result_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","binance.ticker.price.list.v3.8675eca0","binance.avg_price.retrieve.v1.aef0de4f","coinmarketcap.cryptocurrency.listings.latest.retrieve.v1.8dddc12f","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.9dc24be4","etherscan.api.stats.ethprice.retrieve.v2.8eeca59e","binance.avgPrice.retrieve.v3.7b0852b4","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.251242d7","deribit.public.get_index_price.retrieve.v2.d47dcdcb","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d"]},"selection":{"tool_id":"coinmarketcap.tools.price-conversion.execute.v1.2b99a4df"},"inspection":{"returned_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df"],"required_parameters":["amount"]},"parameters":{"amount":1,"id":"1","convert":"USD"},"call":{"attempted":true,"success":true,"execution_id":"7f24abeb-87ee-4715-b069-c50153111792","result_valid":true},"finished_at":"2026-07-23T09:53:13.658Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"190b85e5-c226-468b-91d5-99a08a96768d","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"crypto-price","trial":2,"started_at":"2026-07-23T09:53:13.658Z","status":"completed","discovery":{"result_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","binance.ticker.price.list.v3.8675eca0","binance.avg_price.retrieve.v1.aef0de4f","coinmarketcap.cryptocurrency.listings.latest.retrieve.v1.8dddc12f","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.9dc24be4","etherscan.api.stats.ethprice.retrieve.v2.8eeca59e","binance.avgPrice.retrieve.v3.7b0852b4","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.251242d7","deribit.public.get_index_price.retrieve.v2.d47dcdcb","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d"]},"selection":{"tool_id":"coinmarketcap.tools.price-conversion.execute.v1.2b99a4df"},"inspection":{"returned_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df"],"required_parameters":["amount"]},"parameters":{"amount":1,"id":"1","convert":"USD"},"call":{"attempted":true,"success":true,"execution_id":"98f54c1a-21f0-490e-97d2-2e96781fae9a","result_valid":true},"finished_at":"2026-07-23T09:53:52.934Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"533afeba-6917-4bdb-83f5-6314d89c1207","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"crypto-price","trial":3,"started_at":"2026-07-23T09:53:52.934Z","status":"completed","discovery":{"result_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","binance.ticker.price.list.v3.8675eca0","binance.avg_price.retrieve.v1.aef0de4f","coinmarketcap.cryptocurrency.listings.latest.retrieve.v1.8dddc12f","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.9dc24be4","etherscan.api.stats.ethprice.retrieve.v2.8eeca59e","binance.avgPrice.retrieve.v3.7b0852b4","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.251242d7","deribit.public.get_index_price.retrieve.v2.d47dcdcb","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d"]},"selection":{"tool_id":"coinmarketcap.tools.price-conversion.execute.v1.2b99a4df"},"inspection":{"returned_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df"],"required_parameters":["amount"]},"parameters":{"amount":1,"id":"1","convert":"USD"},"call":{"attempted":true,"success":true,"execution_id":"04fb61ae-dccb-4ddb-a0ff-c8d0056f702a","result_valid":true},"finished_at":"2026-07-23T09:54:29.273Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"4e7bf26c-1907-441c-8103-49071ebcef84","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"web-search","trial":1,"started_at":"2026-07-23T09:54:29.273Z","status":"completed","discovery":{"result_tool_ids":["linkup.search.v1","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic","firecrawl.search.create.v2.a99b6f74","brave_search.images.search.list.v1","cloudsway.smart_search_overseas","serpapi.locations.locations.list.v1","x_developer.2.news.search.retrieve.v1.19b3e678","x_developer.2.spaces.search.retrieve.v1.43a1f31c","tavily.map.execute.v1","brave_search.web.search.list.v1.f02ed09d"]},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"returned_tool_ids":["cloudsway.smart_search_overseas"],"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"execution_id":"83b31afa-4453-4d1f-8c68-cc9caffa3277","result_valid":true},"finished_at":"2026-07-23T09:54:58.710Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"7c57619b-bf15-4504-8587-08a778c4c502","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"web-search","trial":2,"started_at":"2026-07-23T09:54:58.710Z","status":"completed","discovery":{"result_tool_ids":["linkup.search.v1","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic","firecrawl.search.create.v2.a99b6f74","brave_search.images.search.list.v1","cloudsway.smart_search_overseas","serpapi.locations.locations.list.v1","x_developer.2.news.search.retrieve.v1.19b3e678","x_developer.2.spaces.search.retrieve.v1.43a1f31c","tavily.map.execute.v1","brave_search.web.search.list.v1.f02ed09d"]},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"returned_tool_ids":["cloudsway.smart_search_overseas"],"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"execution_id":"1ecaa507-b75e-45ba-84da-475c507d2e2d","result_valid":true},"finished_at":"2026-07-23T09:55:30.042Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"b0c765a8-abef-4d88-b1d1-583d7a7083f5","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"web-search","trial":3,"started_at":"2026-07-23T09:55:30.042Z","status":"completed","discovery":{"result_tool_ids":["linkup.search.v1","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic","firecrawl.search.create.v2.a99b6f74","brave_search.images.search.list.v1","cloudsway.smart_search_overseas","serpapi.locations.locations.list.v1","x_developer.2.news.search.retrieve.v1.19b3e678","x_developer.2.spaces.search.retrieve.v1.43a1f31c","tavily.map.execute.v1","brave_search.web.search.list.v1.f02ed09d"]},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"returned_tool_ids":["cloudsway.smart_search_overseas"],"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"execution_id":"edc9ab7b-457d-4ed5-96a2-9179cf35a6a5","result_valid":true},"finished_at":"2026-07-23T09:56:01.974Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"3573e6ae-8c2b-4c45-9b9f-a7dbdaf459d8","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"air-quality","trial":1,"started_at":"2026-07-23T09:56:01.974Z","status":"completed","discovery":{"result_tool_ids":["openweathermap.air_pollution.current.v2_5","openweathermap.air_pollution.retrieve.v2.5","openweathermap.air_pollution.history.v2_5","openweathermap.air_pollution.forecast.retrieve.v2.5","openweathermap.air_pollution.history.retrieve.v2.5","epa_envirofacts.qaannualperformanceevaluations.bystate.retrieve.v1.61a2aa7d","epa_envirofacts.transactionsqaannualperformanceevaluations.bystate.retrieve.v1.4143b8bf","epa_envirofacts.transactionssample.bysite.retrieve.v1.93889e72","epa_envirofacts.annualdata.bybox.retrieve.v1.4b74a0d6","epa_envirofacts.dailydata.bycounty.retrieve.v1.f1ee8d1b"]},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"returned_tool_ids":["openweathermap.air_pollution.current.v2_5"],"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"execution_id":"2f2b2ad6-dd18-4f13-b8b4-b6729820ac07","result_valid":true},"finished_at":"2026-07-23T09:56:42.264Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"7835af93-e41c-428a-995f-4447571133fe","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"air-quality","trial":2,"started_at":"2026-07-23T09:56:42.264Z","status":"completed","discovery":{"result_tool_ids":["openweathermap.air_pollution.current.v2_5","openweathermap.air_pollution.retrieve.v2.5","openweathermap.air_pollution.history.v2_5","openweathermap.air_pollution.forecast.retrieve.v2.5","openweathermap.air_pollution.history.retrieve.v2.5","epa_envirofacts.qaannualperformanceevaluations.bystate.retrieve.v1.61a2aa7d","epa_envirofacts.transactionsqaannualperformanceevaluations.bystate.retrieve.v1.4143b8bf","epa_envirofacts.transactionssample.bysite.retrieve.v1.93889e72","epa_envirofacts.annualdata.bybox.retrieve.v1.4b74a0d6","epa_envirofacts.dailydata.bycounty.retrieve.v1.f1ee8d1b"]},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"returned_tool_ids":["openweathermap.air_pollution.current.v2_5"],"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"execution_id":"8cd90627-f7bc-4483-9bf9-729efe21d1a9","result_valid":true},"finished_at":"2026-07-23T09:57:10.056Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"73d62bf3-09c9-4c6f-af68-cbe62631e4c3","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"air-quality","trial":3,"started_at":"2026-07-23T09:57:10.056Z","status":"completed","discovery":{"result_tool_ids":["openweathermap.air_pollution.current.v2_5","openweathermap.air_pollution.retrieve.v2.5","openweathermap.air_pollution.history.v2_5","openweathermap.air_pollution.forecast.retrieve.v2.5","openweathermap.air_pollution.history.retrieve.v2.5","epa_envirofacts.qaannualperformanceevaluations.bystate.retrieve.v1.61a2aa7d","epa_envirofacts.transactionsqaannualperformanceevaluations.bystate.retrieve.v1.4143b8bf","epa_envirofacts.transactionssample.bysite.retrieve.v1.93889e72","epa_envirofacts.annualdata.bybox.retrieve.v1.4b74a0d6","epa_envirofacts.dailydata.bycounty.retrieve.v1.f1ee8d1b"]},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"returned_tool_ids":["openweathermap.air_pollution.current.v2_5"],"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"execution_id":"1532ee19-92f8-4ba5-995b-1a29d2abb90f","result_valid":true},"finished_at":"2026-07-23T09:57:36.396Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"0e125c12-7a7b-4c87-9e64-54a8be79f774","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-news","trial":1,"started_at":"2026-07-23T09:57:36.396Z","status":"completed","discovery":{"result_tool_ids":["finnhub.market.news.list.v1","finnhub.news.retrieve.v1.51b2567e","finnhub.news.retrieve.v1.2e72ea5d","finnhub.news.retrieve.v1","finnhub.news.execute.v1","finnhub.companynews.retrieve.v1","alphavantage.news_sentiment.query.v1.467a92c0","finnhub.companynews.retrieve.v1.4d11f47b","finnhub.companynews.retrieve.v1.e428b704","finnhub.companynews.execute.v1"]},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"returned_tool_ids":["finnhub.market.news.list.v1"],"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"execution_id":"c1e2d58e-6c82-446f-bfad-7421f700489e","result_valid":true},"finished_at":"2026-07-23T09:58:04.308Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"7a2ca54b-b346-4343-b6fb-54a264c5b778","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-news","trial":2,"started_at":"2026-07-23T09:58:04.308Z","status":"completed","discovery":{"result_tool_ids":["finnhub.market.news.list.v1","finnhub.news.retrieve.v1.51b2567e","finnhub.news.retrieve.v1.2e72ea5d","finnhub.news.retrieve.v1","finnhub.news.execute.v1","finnhub.companynews.retrieve.v1","alphavantage.news_sentiment.query.v1.467a92c0","finnhub.companynews.retrieve.v1.4d11f47b","finnhub.companynews.retrieve.v1.e428b704","finnhub.companynews.execute.v1"]},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"returned_tool_ids":["finnhub.market.news.list.v1"],"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"execution_id":"f8c8716d-b880-4871-9df6-fe9040f8ddd1","result_valid":true},"finished_at":"2026-07-23T09:58:29.894Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"e98dc7fd-0ed4-4487-a753-c6c197254603","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-news","trial":3,"started_at":"2026-07-23T09:58:29.894Z","status":"completed","discovery":{"result_tool_ids":["finnhub.market.news.list.v1","finnhub.news.retrieve.v1.51b2567e","finnhub.news.retrieve.v1.2e72ea5d","finnhub.news.retrieve.v1","finnhub.news.execute.v1","finnhub.companynews.retrieve.v1","alphavantage.news_sentiment.query.v1.467a92c0","finnhub.companynews.retrieve.v1.4d11f47b","finnhub.companynews.retrieve.v1.e428b704","finnhub.companynews.execute.v1"]},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"returned_tool_ids":["finnhub.market.news.list.v1"],"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"execution_id":"95bd80bb-dbd4-4137-873a-2d3307eabf77","result_valid":true},"finished_at":"2026-07-23T09:58:58.074Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"a22d10aa-0f23-4f76-b2a2-692576b4b2a2","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T09:58:58.074Z","status":"failed","discovery":{"result_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8","dataforseo.dataforseo_labs.google.domain_whois_overview.live.execute.v3","dataforseo.databases.domains.query.v3.c1720ec0","virustotal.domains.report.retrieve.v3.c5459740","bainiu_api.common.company_domain.query.v1.e978ca32","virustotal.domain_reports.retrieve.v3.c5459740","dataforseo.domain_analytics.technologies.domains_by_technology.live.create.v3.bfc3dc2d","virustotal.domains.retrieve.v3.c5459740","virustotal.domains.analyse.v3.c755c69e","dataforseo.domain_analytics.technologies.domains_by_html_terms.live.create.v3.d3760ae1"]},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"returned_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"],"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected","message":"Adapter invocation failed"},"finished_at":"2026-07-23T09:59:40.997Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"97361151-f794-478a-8b90-e8e544c484f7","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T09:59:40.997Z","status":"completed","discovery":{"result_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8","dataforseo.dataforseo_labs.google.domain_whois_overview.live.execute.v3","dataforseo.databases.domains.query.v3.c1720ec0","virustotal.domains.report.retrieve.v3.c5459740","bainiu_api.common.company_domain.query.v1.e978ca32","virustotal.domain_reports.retrieve.v3.c5459740","dataforseo.domain_analytics.technologies.domains_by_technology.live.create.v3.bfc3dc2d","virustotal.domains.retrieve.v3.c5459740","virustotal.domains.analyse.v3.c755c69e","dataforseo.domain_analytics.technologies.domains_by_html_terms.live.create.v3.d3760ae1"]},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"returned_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"],"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"execution_id":"d3182215-e0d8-4571-b0c8-e7192d34d59a","result_valid":true},"finished_at":"2026-07-23T10:00:15.027Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"e877975e-2fff-4ec9-a487-0bddba738e4f","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T10:00:15.027Z","status":"failed","discovery":{"result_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8","dataforseo.dataforseo_labs.google.domain_whois_overview.live.execute.v3","dataforseo.databases.domains.query.v3.c1720ec0","virustotal.domains.report.retrieve.v3.c5459740","bainiu_api.common.company_domain.query.v1.e978ca32","virustotal.domain_reports.retrieve.v3.c5459740","dataforseo.domain_analytics.technologies.domains_by_technology.live.create.v3.bfc3dc2d","virustotal.domains.retrieve.v3.c5459740","virustotal.domains.analyse.v3.c755c69e","dataforseo.domain_analytics.technologies.domains_by_html_terms.live.create.v3.d3760ae1"]},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"returned_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"],"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected","message":"Adapter invocation failed"},"finished_at":"2026-07-23T10:00:55.710Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"d9fe796d-cd3d-4d0c-8bb3-0cd1fc7e128d","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"image-search","trial":1,"started_at":"2026-07-23T10:00:55.710Z","status":"completed","discovery":{"result_tool_ids":["brave_search.images.search.list.v1","shutterstock.images.search.list.v2.69059569","dataforseo.serp.google.search_by_image.task.create.v3","shutterstock.images.search.list.v2.c2b44682","jina_search.embeddings.create.v2026.5c7cb661","jina_search.rerank.create.v2026.23dc486d","linkup.search.v1","shutterstock.images.search.list.v2.15f15845","shutterstock.images.search.suggestions.list.v2.f6850c10","shutterstock.images.search.suggestions.list.v2"]},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"returned_tool_ids":["brave_search.images.search.list.v1"],"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"execution_id":"fa985665-5752-4080-9878-40d7480d003e","result_valid":true},"finished_at":"2026-07-23T10:01:21.601Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"e3d0d1fd-e32c-4701-b87a-7d2907aae773","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"image-search","trial":2,"started_at":"2026-07-23T10:01:21.601Z","status":"completed","discovery":{"result_tool_ids":["brave_search.images.search.list.v1","shutterstock.images.search.list.v2.69059569","dataforseo.serp.google.search_by_image.task.create.v3","shutterstock.images.search.list.v2.c2b44682","jina_search.embeddings.create.v2026.5c7cb661","jina_search.rerank.create.v2026.23dc486d","linkup.search.v1","shutterstock.images.search.list.v2.15f15845","shutterstock.images.search.suggestions.list.v2.f6850c10","shutterstock.images.search.suggestions.list.v2"]},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"returned_tool_ids":["brave_search.images.search.list.v1"],"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"execution_id":"89c4a370-2374-4ecc-8132-095b9ba65e48","result_valid":true},"finished_at":"2026-07-23T10:01:46.789Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"109d0ded-433f-4ef6-a7ed-f8cf19b897be","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"image-search","trial":3,"started_at":"2026-07-23T10:01:46.789Z","status":"completed","discovery":{"result_tool_ids":["brave_search.images.search.list.v1","shutterstock.images.search.list.v2.69059569","dataforseo.serp.google.search_by_image.task.create.v3","shutterstock.images.search.list.v2.c2b44682","jina_search.rerank.create.v2026.23dc486d","jina_search.embeddings.create.v2026.5c7cb661","linkup.search.v1","shutterstock.images.search.list.v2.15f15845","shutterstock.images.search.suggestions.list.v2.f6850c10","shutterstock.images.search.suggestions.list.v2"]},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"returned_tool_ids":["brave_search.images.search.list.v1"],"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"execution_id":"1a25b138-27e5-45df-a6a9-683259e805c3","result_valid":true},"finished_at":"2026-07-23T10:02:17.125Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"e55b1799-5e11-481c-a18f-e4be01b4e744","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-holidays","trial":1,"started_at":"2026-07-23T10:02:17.125Z","status":"completed","discovery":{"result_tool_ids":["finnhub.stock.marketholiday.retrieve.v1.75d6c1f9","finnhub.stock.marketholiday.execute.v1","eodhd.exchange_details.retrieve.v1.7da32390","finnhub.stock.marketholiday.retrieve.v1","finnhub.stock.marketholiday.retrieve.v1.5c5bfe9a","massive_stocks.stocks.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.retrieve.v1","massive_stocks.options.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.list.v1","massive_stocks.options.marketstatus.retrieve.v1.3045ae6e"]},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"returned_tool_ids":["finnhub.stock.marketholiday.retrieve.v1"],"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"execution_id":"c1b4c819-ac15-449f-8408-3944918ff86b","result_valid":true},"finished_at":"2026-07-23T10:02:44.222Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"a88a3226-9842-4032-ac67-fcae35d59a03","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-holidays","trial":2,"started_at":"2026-07-23T10:02:44.222Z","status":"completed","discovery":{"result_tool_ids":["finnhub.stock.marketholiday.retrieve.v1.75d6c1f9","finnhub.stock.marketholiday.execute.v1","eodhd.exchange_details.retrieve.v1.7da32390","finnhub.stock.marketholiday.retrieve.v1","finnhub.stock.marketholiday.retrieve.v1.5c5bfe9a","massive_stocks.stocks.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.retrieve.v1","massive_stocks.options.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.list.v1","massive_stocks.options.marketstatus.retrieve.v1.3045ae6e"]},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"returned_tool_ids":["finnhub.stock.marketholiday.retrieve.v1"],"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"execution_id":"c53b47ab-dab7-4f70-a114-e1e32ff91bbf","result_valid":true},"finished_at":"2026-07-23T10:03:10.533Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"bd6bfb47-687b-4d08-9a70-f019316e623a","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-holidays","trial":3,"started_at":"2026-07-23T10:03:10.533Z","status":"completed","discovery":{"result_tool_ids":["finnhub.stock.marketholiday.retrieve.v1.75d6c1f9","finnhub.stock.marketholiday.execute.v1","eodhd.exchange_details.retrieve.v1.7da32390","finnhub.stock.marketholiday.retrieve.v1","finnhub.stock.marketholiday.retrieve.v1.5c5bfe9a","massive_stocks.stocks.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.retrieve.v1","massive_stocks.options.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.list.v1","massive_stocks.options.marketstatus.retrieve.v1.3045ae6e"]},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"returned_tool_ids":["finnhub.stock.marketholiday.retrieve.v1"],"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"execution_id":"ee0ee3a7-f306-42ce-ac38-4b9d2471d2da","result_valid":true},"finished_at":"2026-07-23T10:03:37.293Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"c28bc1c8-94d0-4bed-a562-af06253fc011","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-23T10:03:37.293Z","status":"completed","discovery":{"result_tool_ids":["usgs_earthquake_catalog.event.events.list.v1","usgs_earthquake_catalog.fdsnws.event.query.retrieve.v1.025549c9","usgs_earthquake_catalog.earthquakes.query.v1","usgs_earthquake_catalog.fdsnws.event.query.v1.025549c9","usgs_earthquake_catalog.events.query.v1.025549c9","usgs_earthquake.catalogs.list.v1","usgs_earthquake.contributors.list.v1","usgs-earthquake-event-api.event.query.list.v1","usgs_earthquake.parameters.list.v1","usgs_earthquake.version.retrieve.v1"]},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"returned_tool_ids":["usgs_earthquake_catalog.event.events.list.v1"],"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"execution_id":"20baa4c2-30c0-4c54-873d-29b623d63766","result_valid":true},"finished_at":"2026-07-23T10:04:19.806Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"9eff8a69-cb9a-4931-ae30-4230a29c6b2c","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-23T10:04:19.806Z","status":"completed","discovery":{"result_tool_ids":["usgs_earthquake_catalog.event.events.list.v1","usgs_earthquake_catalog.fdsnws.event.query.retrieve.v1.025549c9","usgs_earthquake_catalog.earthquakes.query.v1","usgs_earthquake_catalog.fdsnws.event.query.v1.025549c9","usgs_earthquake_catalog.events.query.v1.025549c9","usgs_earthquake.catalogs.list.v1","usgs_earthquake.contributors.list.v1","usgs-earthquake-event-api.event.query.list.v1","usgs_earthquake.parameters.list.v1","usgs_earthquake.version.retrieve.v1"]},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"returned_tool_ids":["usgs_earthquake_catalog.event.events.list.v1"],"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"execution_id":"1953bab8-8a42-4e52-acf5-4043deadc1d3","result_valid":true},"finished_at":"2026-07-23T10:04:54.838Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"81d9ac77-2d8f-4363-be1a-09193951a1b1","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T10:04:54.838Z","status":"completed","discovery":{"result_tool_ids":["usgs_earthquake_catalog.event.events.list.v1","usgs_earthquake_catalog.fdsnws.event.query.retrieve.v1.025549c9","usgs_earthquake_catalog.earthquakes.query.v1","usgs_earthquake_catalog.fdsnws.event.query.v1.025549c9","usgs_earthquake_catalog.events.query.v1.025549c9","usgs_earthquake.catalogs.list.v1","usgs_earthquake.contributors.list.v1","usgs-earthquake-event-api.event.query.list.v1","usgs_earthquake.parameters.list.v1","usgs_earthquake.version.retrieve.v1"]},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"returned_tool_ids":["usgs_earthquake_catalog.event.events.list.v1"],"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"execution_id":"29f479b7-a1f9-45fa-8539-fb5ebbd208dc","result_valid":true},"finished_at":"2026-07-23T10:05:33.311Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"abc19032-2dc3-46fc-a861-05132c7eda65","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"company-profile","trial":1,"started_at":"2026-07-23T10:05:33.311Z","status":"completed","discovery":{"result_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2","bainiu_api.company.detail.execute.v1.f6b528e7","bainiu_api.openapi.company.alter.execute.v1.8fa090d1","tradefeeds.compinfo.retrieve.v1.c2314765","financialmodelingprep.stable.companynotes.retrieve.v1.930aa468","financialmodelingprep.stable.secfilingscompanysearch.symbol.retrieve.v1.5cf7397d","bainiu_api.openapi.common.company_info_deep_search.execute.v1.8e5f009d","financialmodelingprep.search.cusip.retrieve.v1.51d0c621","bainiu_api.openapi.common.company_lianxi_public.execute.v1.bd6261e8","financialmodelingprep.stable.secfilingscompanysearch.cik.retrieve.v1.a1087fe5"]},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"returned_tool_ids":["tradefeeds.compinfo.retrieve.v1.c2314765"],"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"execution_id":"c2cc2904-3341-4f5a-b3b5-33471d8bab04","result_valid":false},"finished_at":"2026-07-23T10:06:00.664Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"66977b1e-3302-469e-9e30-590d4b8cd542","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"company-profile","trial":2,"started_at":"2026-07-23T10:06:00.664Z","status":"completed","discovery":{"result_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2","bainiu_api.company.detail.execute.v1.f6b528e7","bainiu_api.openapi.company.alter.execute.v1.8fa090d1","tradefeeds.compinfo.retrieve.v1.c2314765","financialmodelingprep.stable.companynotes.retrieve.v1.930aa468","financialmodelingprep.stable.secfilingscompanysearch.symbol.retrieve.v1.5cf7397d","bainiu_api.openapi.common.company_info_deep_search.execute.v1.8e5f009d","financialmodelingprep.search.cusip.retrieve.v1.51d0c621","bainiu_api.openapi.common.company_lianxi_public.execute.v1.bd6261e8","financialmodelingprep.stable.secfilingscompanysearch.cik.retrieve.v1.a1087fe5"]},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"returned_tool_ids":["tradefeeds.compinfo.retrieve.v1.c2314765"],"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"execution_id":"9162f0c8-7d8e-41a5-bc0f-3c8670164b72","result_valid":false},"finished_at":"2026-07-23T10:06:50.608Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"b3001ec6-f8da-4dcc-b420-a40d0991337b","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"company-profile","trial":3,"started_at":"2026-07-23T10:06:50.608Z","status":"completed","discovery":{"result_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2","bainiu_api.company.detail.execute.v1.f6b528e7","bainiu_api.openapi.company.alter.execute.v1.8fa090d1","tradefeeds.compinfo.retrieve.v1.c2314765","financialmodelingprep.stable.companynotes.retrieve.v1.930aa468","financialmodelingprep.stable.secfilingscompanysearch.symbol.retrieve.v1.5cf7397d","bainiu_api.openapi.common.company_info_deep_search.execute.v1.8e5f009d","financialmodelingprep.search.cusip.retrieve.v1.51d0c621","bainiu_api.openapi.common.company_lianxi_public.execute.v1.bd6261e8","financialmodelingprep.stable.secfilingscompanysearch.cik.retrieve.v1.a1087fe5"]},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"returned_tool_ids":["tradefeeds.compinfo.retrieve.v1.c2314765"],"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"execution_id":"85544350-ce6a-4431-9e82-c4e5665bf8ab","result_valid":false},"finished_at":"2026-07-23T10:07:33.561Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"7e26fd5b-b2f6-40bb-90dc-0d32665ffadd","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"sports-standings","trial":1,"started_at":"2026-07-23T10:07:33.561Z","status":"completed","discovery":{"result_tool_ids":["api_sports.standings.list.v1.98a153bd","api_sports.standings.list.v1.39568ba4","api_sports.standings.groups.list.v1.a077a680","api_sports.standings.groups.list.v1.0d8a40cf","api_sports.standings.stages.list.v1.5ba13a89","api_sports.standings.stages.list.v1.838f769a","api_sports.standings.retrieve.v1.ff019893","api_sports.standings.groups.list.v1.bed67ce0","api_sports.standings.groups.list.v1.57393fb3","api_sports.standings.stages.list.v1.607034e1"]},"selection":{"tool_id":"api_sports.standings.list.v1.98a153bd"},"inspection":{"returned_tool_ids":["api_sports.standings.list.v1.98a153bd"],"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"execution_id":"de1481bd-672a-4d93-a07b-1896bcd48cc0","result_valid":true},"finished_at":"2026-07-23T10:08:06.053Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"936fb637-9c45-40c3-bfe5-33fb0bf5bc9a","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"sports-standings","trial":2,"started_at":"2026-07-23T10:08:06.053Z","status":"completed","discovery":{"result_tool_ids":["api_sports.standings.list.v1.98a153bd","api_sports.standings.list.v1.39568ba4","api_sports.standings.groups.list.v1.a077a680","api_sports.standings.groups.list.v1.0d8a40cf","api_sports.standings.stages.list.v1.5ba13a89","api_sports.standings.stages.list.v1.838f769a","api_sports.standings.retrieve.v1.ff019893","api_sports.standings.groups.list.v1.bed67ce0","api_sports.standings.groups.list.v1.57393fb3","api_sports.standings.stages.list.v1.607034e1"]},"selection":{"tool_id":"api_sports.standings.list.v1.98a153bd"},"inspection":{"returned_tool_ids":["api_sports.standings.list.v1.98a153bd"],"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"execution_id":"5610b885-0407-4350-b26e-fc772427fc32","result_valid":true},"finished_at":"2026-07-23T10:08:47.521Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"85b215db-7bde-4512-aad4-781a905c7c38","model":"gpt-5.6-sol","metadata":{"lane":"pinned-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"sports-standings","trial":3,"started_at":"2026-07-23T10:08:47.521Z","status":"completed","discovery":{"result_tool_ids":["api_sports.standings.list.v1.98a153bd","api_sports.standings.list.v1.39568ba4","api_sports.standings.groups.list.v1.a077a680","api_sports.standings.groups.list.v1.0d8a40cf","api_sports.standings.stages.list.v1.5ba13a89","api_sports.standings.stages.list.v1.838f769a","api_sports.standings.retrieve.v1.ff019893","api_sports.standings.groups.list.v1.bed67ce0","api_sports.standings.groups.list.v1.57393fb3","api_sports.standings.stages.list.v1.607034e1"]},"selection":{"tool_id":"api_sports.standings.list.v1.98a153bd"},"inspection":{"returned_tool_ids":["api_sports.standings.list.v1.98a153bd"],"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"execution_id":"1cd42924-35d4-40e0-930a-d7d56a26587c","result_valid":true},"finished_at":"2026-07-23T10:09:22.117Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"23187609-52d7-44b6-b60e-91f68d32777c","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":1,"started_at":"2026-07-23T09:40:25.828Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"28e564a72f02a0b6744c9086e328ba4b829e39246dca5821577adcb20a5f3a72","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:40:51.455Z","source_record_sha256":"0394fb5b9e153aa501a8d781643292841d812c05c0986a4661ed3866e242ab03"} +{"schema_version":1,"benchmark_version":"v1","run_id":"d2bfdd97-1ac6-4388-aab6-dc04bd389b4d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":2,"started_at":"2026-07-23T09:40:51.455Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"28e564a72f02a0b6744c9086e328ba4b829e39246dca5821577adcb20a5f3a72","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:41:19.367Z","source_record_sha256":"a132a571a86762281fbf49fdd7506aaa19fc15a736d6837127da51c0cab30f16"} +{"schema_version":1,"benchmark_version":"v1","run_id":"9dc77ec7-6fea-42e6-8786-f8970a33da5d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":3,"started_at":"2026-07-23T09:41:19.367Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"28e564a72f02a0b6744c9086e328ba4b829e39246dca5821577adcb20a5f3a72","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:41:45.578Z","source_record_sha256":"9b8eb9c3f5b12d221b1a3051a045f8cb17bdf481f796131e344089ac7ad58e16"} +{"schema_version":1,"benchmark_version":"v1","run_id":"98d4488a-8f0e-48fe-8743-ed08df8940f8","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-23T09:41:45.578Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"65423dcd48e9ba551f0500805a38535f2428d65ef165a26c778463c01921a519","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:42:10.683Z","source_record_sha256":"7dc8a173d225a7254741f7a0591960ebcf3c3ff7c8a7d18ba52e38dae92fef57"} +{"schema_version":1,"benchmark_version":"v1","run_id":"450cf5b6-8404-4be2-b73e-c28a6ae3227d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-23T09:42:10.683Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"65423dcd48e9ba551f0500805a38535f2428d65ef165a26c778463c01921a519","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:42:35.790Z","source_record_sha256":"088cb3c49cd699c6f9a9033404a71866daac47829c9a2ec696e3f5aed6bb6e89"} +{"schema_version":1,"benchmark_version":"v1","run_id":"456b1477-ec0a-4772-887c-8971ac437a95","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-23T09:42:35.790Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"65423dcd48e9ba551f0500805a38535f2428d65ef165a26c778463c01921a519","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:43:04.020Z","source_record_sha256":"6f10781492c58eeb6c351dfa9db214028bc0558b31b1e442e0568aa5f9611edf"} +{"schema_version":1,"benchmark_version":"v1","run_id":"850f6ae9-de46-4e92-a722-81edd34dd191","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-23T09:43:04.020Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"b98b9cb4476c56c6553ff1ea2bfd5e1976911c8c14142f76ef97c691d368414f","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":100,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:43:40.323Z","source_record_sha256":"18b9cf40ca54aac96152a60268192d8144aac03c1ab87fa134dd57bfffd5ce1e"} +{"schema_version":1,"benchmark_version":"v1","run_id":"378c4fcc-1e86-4149-8b38-82d8230e5d11","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-23T09:43:40.323Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"b98b9cb4476c56c6553ff1ea2bfd5e1976911c8c14142f76ef97c691d368414f","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:44:16.949Z","source_record_sha256":"f36668206832ea60ec935daeee36c7f48d9b4fad9c4a2740d6b347e3c6cda748"} +{"schema_version":1,"benchmark_version":"v1","run_id":"beb58cdc-281f-42cf-a388-9d9f30fa74be","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-23T09:44:16.949Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"b98b9cb4476c56c6553ff1ea2bfd5e1976911c8c14142f76ef97c691d368414f","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:44:53.121Z","source_record_sha256":"07f3fe927e57423d7eae4fa52ee878da564f4142ea03fe803b0b3307fb1a319d"} +{"schema_version":1,"benchmark_version":"v1","run_id":"b602eea9-1fa5-4b43-adf2-59adff282a87","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":1,"started_at":"2026-07-23T09:44:53.121Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"4ca8303398d379bc94e129f5da06d77fc7aebb56b424abeea7e4a3feb3910636","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20AI","from":"2026-07-16","to":"2026-07-23T23:59:59+08:00","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:45:25.145Z","source_record_sha256":"06a9d53c3fec8a8ec22cd6fb65cca82d18632ad97ceefbff36c9c18572724017"} +{"schema_version":1,"benchmark_version":"v1","run_id":"bfc0ab6d-9a15-4768-a672-e466f0d23414","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":2,"started_at":"2026-07-23T09:45:25.145Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"4ca8303398d379bc94e129f5da06d77fc7aebb56b424abeea7e4a3feb3910636","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20AI","from":"2026-07-16","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:46:05.132Z","source_record_sha256":"406f86260cf08286855e783c689cd90f3e1f976e84b0277cbff522aa6526c029"} +{"schema_version":1,"benchmark_version":"v1","run_id":"c2bab891-e16c-49ce-af35-9ba5220ef929","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":3,"started_at":"2026-07-23T09:46:05.132Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"4ca8303398d379bc94e129f5da06d77fc7aebb56b424abeea7e4a3feb3910636","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20%22generative%20AI%22","from":"2026-07-20","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:46:35.462Z","source_record_sha256":"e0be7b97e788ce36f9643c04d2f4911b3bba211890d0915b07f01e579b6ff811"} +{"schema_version":1,"benchmark_version":"v1","run_id":"7c3b1d86-cdf3-42aa-94d0-bfa2024284ec","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T09:46:35.462Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7935de8cea6f84dce9a4df111df427079b97af7dd653fecdcba159947c48d136","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:47:02.518Z","source_record_sha256":"24485ca6593e17388a3346e0f9209b4846153324491a991e8ea7f222a144ab4a"} +{"schema_version":1,"benchmark_version":"v1","run_id":"0dfd6d1c-ca90-4916-8b8f-4ae4d3e7f273","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T09:47:02.518Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7935de8cea6f84dce9a4df111df427079b97af7dd653fecdcba159947c48d136","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:47:32.328Z","source_record_sha256":"f7107e2e7957bb86f20b63cd1c0eb6a4de8d5158aa8f68d2f1d8a3ba01fe3179"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ce6fbc71-c894-461a-adb1-4a607e7d4bdc","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T09:47:32.328Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7935de8cea6f84dce9a4df111df427079b97af7dd653fecdcba159947c48d136","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:47:58.422Z","source_record_sha256":"f2cfd462aceb106b5154662f6f3e86d2e0d7afcc77db75ad43ba5d04ccf1aa56"} +{"schema_version":1,"benchmark_version":"v1","run_id":"b43508b4-6a54-487e-b3fd-dd511201af19","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T09:47:58.422Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"0c9d6a51ec70c89cafcbd7ccfca456093271999f98a8c42b6423e996fc6f48a4","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:48:24.635Z","source_record_sha256":"4da0464a226f7e73e86a16071182c1ecf42f06113bf5d0c77ee7068624a913db"} +{"schema_version":1,"benchmark_version":"v1","run_id":"704ee017-af2e-469b-a055-de7b367c3ffb","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T09:48:24.635Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"0c9d6a51ec70c89cafcbd7ccfca456093271999f98a8c42b6423e996fc6f48a4","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:48:55.300Z","source_record_sha256":"fb66f6302a91edfe07cfb60666e18fc797143c7c62980963704afd6dfdffc5fb"} +{"schema_version":1,"benchmark_version":"v1","run_id":"495bda63-87f8-4ada-bcd2-a2a4076f6431","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T09:48:55.300Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"0c9d6a51ec70c89cafcbd7ccfca456093271999f98a8c42b6423e996fc6f48a4","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:49:26.689Z","source_record_sha256":"d0640e38ed193059bf9d325cd9f756d76ff72556413bce3721d34f795afd0f0f"} +{"schema_version":1,"benchmark_version":"v1","run_id":"10a88d8d-1053-4d50-bab5-4b7114dd0ee7","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T09:49:26.689Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3dcb23db74a0c80244544838d4968450a29fa18516b5d8c8e17b0a38af6919cd","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:49:55.414Z","source_record_sha256":"35054cbc3c8f2f9e2e047104c7fce1e4b687d34d2a9fe24367477a5158e97af4"} +{"schema_version":1,"benchmark_version":"v1","run_id":"252d6b99-8605-4a66-aaec-d2b63a4fe36f","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T09:49:55.414Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3dcb23db74a0c80244544838d4968450a29fa18516b5d8c8e17b0a38af6919cd","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:50:31.198Z","source_record_sha256":"c5ed77b70f27ba4bba45540ec1d3ace713757cd7f600ec78f5aae4a0dd2b244b"} +{"schema_version":1,"benchmark_version":"v1","run_id":"3b2b6bf9-ad89-4298-92a7-3622fb115a13","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T09:50:31.198Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3dcb23db74a0c80244544838d4968450a29fa18516b5d8c8e17b0a38af6919cd","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:51:04.973Z","source_record_sha256":"ea2151bd097ea5a08d2fb3134f458e21af30728d07a85b1b08d3df56f4b816af"} +{"schema_version":1,"benchmark_version":"v1","run_id":"aaac2eb2-646e-42ff-ae32-1326ee9682b4","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-23T09:51:04.973Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2798b2551f89d750e90bc78b840801a716942fd2cc7241b33575fc13b5e1cc98","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T09:51:37.056Z","source_record_sha256":"6236fb0c65c08f023b41589cf7aed9f3194cb6782081175abfa6382589e9a535"} +{"schema_version":1,"benchmark_version":"v1","run_id":"3e947ba6-ca5a-47dc-abe3-404e9464239b","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-23T09:51:37.056Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"954802b741ae1296ad98d5a92faa78b0b02b7549ad05ec06ce4544a982fca525","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T09:52:06.521Z","source_record_sha256":"8ea3c9edf5c876f7243d43804235e2665a74f7cc290af4b8ba61e605a3ca21a5"} +{"schema_version":1,"benchmark_version":"v1","run_id":"69e303d3-d9ea-4f86-9fa3-0b4bd9c3087c","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-23T09:52:06.521Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"954802b741ae1296ad98d5a92faa78b0b02b7549ad05ec06ce4544a982fca525","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T09:52:38.206Z","source_record_sha256":"8d56e8fc444982b3c01b8ecf2017638d7194161bcebbf1ab788bc5169290a2d5"} +{"schema_version":1,"benchmark_version":"v1","run_id":"16873a4e-2cd3-47d9-845e-0067aeb15a42","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":1,"started_at":"2026-07-23T09:52:38.206Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2099bfa8887678c84902e7ec565c63edeef86ac5127e75b290713f8ff3a4f197","selection_grounded":true},"selection":{"tool_id":"coinmarketcap.tools.price-conversion.execute.v1.2b99a4df"},"inspection":{"selection_grounded":true,"required_parameters":["amount"]},"parameters":{"amount":1,"id":"1","convert":"USD"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:53:13.658Z","source_record_sha256":"ee67bc1787845c7185516013d6c9a0f7e5ac4e6116d9ef80bb8fb9cbb0d926ec"} +{"schema_version":1,"benchmark_version":"v1","run_id":"190b85e5-c226-468b-91d5-99a08a96768d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":2,"started_at":"2026-07-23T09:53:13.658Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2099bfa8887678c84902e7ec565c63edeef86ac5127e75b290713f8ff3a4f197","selection_grounded":true},"selection":{"tool_id":"coinmarketcap.tools.price-conversion.execute.v1.2b99a4df"},"inspection":{"selection_grounded":true,"required_parameters":["amount"]},"parameters":{"amount":1,"id":"1","convert":"USD"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:53:52.934Z","source_record_sha256":"d9583032847d73d7c87252026831b092ad84839a82152af3ba4f8958db2484e8"} +{"schema_version":1,"benchmark_version":"v1","run_id":"533afeba-6917-4bdb-83f5-6314d89c1207","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":3,"started_at":"2026-07-23T09:53:52.934Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2099bfa8887678c84902e7ec565c63edeef86ac5127e75b290713f8ff3a4f197","selection_grounded":true},"selection":{"tool_id":"coinmarketcap.tools.price-conversion.execute.v1.2b99a4df"},"inspection":{"selection_grounded":true,"required_parameters":["amount"]},"parameters":{"amount":1,"id":"1","convert":"USD"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:54:29.273Z","source_record_sha256":"c26269c83b8733a84b0b0fa0afd38a548f64e455d12753e98f3064158726eb8d"} +{"schema_version":1,"benchmark_version":"v1","run_id":"4e7bf26c-1907-441c-8103-49071ebcef84","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":1,"started_at":"2026-07-23T09:54:29.273Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9eb927c2b3d94415d2534d541127a937adb20aca4f0cefdc0b7c6948c064c9f9","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:54:58.710Z","source_record_sha256":"cbf18b1fae7e417ae99b83844c38d0c8ade8811fb732e5ccecde52d12d1ae414"} +{"schema_version":1,"benchmark_version":"v1","run_id":"7c57619b-bf15-4504-8587-08a778c4c502","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":2,"started_at":"2026-07-23T09:54:58.710Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9eb927c2b3d94415d2534d541127a937adb20aca4f0cefdc0b7c6948c064c9f9","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:55:30.042Z","source_record_sha256":"deeb3853b2c38377a75619dfad3b69859fb62bf5e4e5ebc5212f9464b79c9a08"} +{"schema_version":1,"benchmark_version":"v1","run_id":"b0c765a8-abef-4d88-b1d1-583d7a7083f5","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":3,"started_at":"2026-07-23T09:55:30.042Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9eb927c2b3d94415d2534d541127a937adb20aca4f0cefdc0b7c6948c064c9f9","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:56:01.974Z","source_record_sha256":"d4d546dc07a8cfebca12c2b75db4e5ba54f00289b241423a33213091bf58efa1"} +{"schema_version":1,"benchmark_version":"v1","run_id":"3573e6ae-8c2b-4c45-9b9f-a7dbdaf459d8","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":1,"started_at":"2026-07-23T09:56:01.974Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d664c4f2fd3f9d9e0d0d8f66a450ec57e2aa075d57bba3883ac678920ca072c6","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:56:42.264Z","source_record_sha256":"edd01cfdc6a189fb72df609f9a303603b01a19fad462ba18acb337cc42591b97"} +{"schema_version":1,"benchmark_version":"v1","run_id":"7835af93-e41c-428a-995f-4447571133fe","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":2,"started_at":"2026-07-23T09:56:42.264Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d664c4f2fd3f9d9e0d0d8f66a450ec57e2aa075d57bba3883ac678920ca072c6","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:57:10.056Z","source_record_sha256":"4f1e3e4bf201c9a85fe1a4c523b17c3b8e06c57dc637a19767f1872cc9de5d15"} +{"schema_version":1,"benchmark_version":"v1","run_id":"73d62bf3-09c9-4c6f-af68-cbe62631e4c3","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":3,"started_at":"2026-07-23T09:57:10.056Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d664c4f2fd3f9d9e0d0d8f66a450ec57e2aa075d57bba3883ac678920ca072c6","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:57:36.396Z","source_record_sha256":"3be7b15dcd9a6869bf6202dee1c262faa553589f0f1d34ed4fc3b0d3d860b9ef"} +{"schema_version":1,"benchmark_version":"v1","run_id":"0e125c12-7a7b-4c87-9e64-54a8be79f774","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":1,"started_at":"2026-07-23T09:57:36.396Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8d10f08954c40da564662fec151d75440ec8391aefdf74cfc8e2c32a1ee26dbb","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:58:04.308Z","source_record_sha256":"40c87aa07d78b9b835112c13584563870d4b64eb7a23230de8085d7a344f1851"} +{"schema_version":1,"benchmark_version":"v1","run_id":"7a2ca54b-b346-4343-b6fb-54a264c5b778","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":2,"started_at":"2026-07-23T09:58:04.308Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8d10f08954c40da564662fec151d75440ec8391aefdf74cfc8e2c32a1ee26dbb","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:58:29.894Z","source_record_sha256":"b0b04c429f3abe332a69253f6d6e831d1702da29097ab1285883cf4945269647"} +{"schema_version":1,"benchmark_version":"v1","run_id":"e98dc7fd-0ed4-4487-a753-c6c197254603","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":3,"started_at":"2026-07-23T09:58:29.894Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8d10f08954c40da564662fec151d75440ec8391aefdf74cfc8e2c32a1ee26dbb","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:58:58.074Z","source_record_sha256":"4dc549878c034c8440e01e24b20ced68fd25617caef32d35519f5ee2750ec0d3"} +{"schema_version":1,"benchmark_version":"v1","run_id":"a22d10aa-0f23-4f76-b2a2-692576b4b2a2","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T09:58:58.074Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected","message":"Adapter invocation failed"},"finished_at":"2026-07-23T09:59:40.997Z","source_record_sha256":"8e0f1a8dd393b2e3e3f8480ede541b2effe403ed92acfa0e2fab9f4269ed9246"} +{"schema_version":1,"benchmark_version":"v1","run_id":"97361151-f794-478a-8b90-e8e544c484f7","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T09:59:40.997Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:00:15.027Z","source_record_sha256":"f9b49fcdc5f1e3d00fdb7185a218da04e37fc6e60da566fc3aa2e9073ab191a1"} +{"schema_version":1,"benchmark_version":"v1","run_id":"e877975e-2fff-4ec9-a487-0bddba738e4f","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T10:00:15.027Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected","message":"Adapter invocation failed"},"finished_at":"2026-07-23T10:00:55.710Z","source_record_sha256":"d6bd4b821637a43316f544a37fc975c0b59797e77cd24ef7fae03280ddaa3721"} +{"schema_version":1,"benchmark_version":"v1","run_id":"d9fe796d-cd3d-4d0c-8bb3-0cd1fc7e128d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":1,"started_at":"2026-07-23T10:00:55.710Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"406f82f0151e2aab38d28e5369d6e3b6b8f1e5a40414bd6ea7cafc3eb9f40259","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:01:21.601Z","source_record_sha256":"61bcf3bc4829f5e23fb25aa989ab9e25e8bdc1b9066d6e8e7dae7ec7d3fe39a4"} +{"schema_version":1,"benchmark_version":"v1","run_id":"e3d0d1fd-e32c-4701-b87a-7d2907aae773","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":2,"started_at":"2026-07-23T10:01:21.601Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"406f82f0151e2aab38d28e5369d6e3b6b8f1e5a40414bd6ea7cafc3eb9f40259","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:01:46.789Z","source_record_sha256":"0c98fe2b9fab21c84a6018f2c17577e36c6f1db959d8959006e8a27a1d509c9d"} +{"schema_version":1,"benchmark_version":"v1","run_id":"109d0ded-433f-4ef6-a7ed-f8cf19b897be","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":3,"started_at":"2026-07-23T10:01:46.789Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"413e70c9b613711904d82925006e89c88e507640e6650646084db3a76f64b8ce","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:02:17.125Z","source_record_sha256":"25f730ed07cea7449205d61271acef2544d40eae7a33b9a76bc2ed2d6eb53a7d"} +{"schema_version":1,"benchmark_version":"v1","run_id":"e55b1799-5e11-481c-a18f-e4be01b4e744","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":1,"started_at":"2026-07-23T10:02:17.125Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5d7380bab4c7ce4e2289dc764fc2cae8ba0d5cbe190f62fc09dee13517796d3d","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:02:44.222Z","source_record_sha256":"b47fbff96f2cbd4f32ae3c1d02eae4ad3bb17da02f61fd7f1d9018cef02ce863"} +{"schema_version":1,"benchmark_version":"v1","run_id":"a88a3226-9842-4032-ac67-fcae35d59a03","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":2,"started_at":"2026-07-23T10:02:44.222Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5d7380bab4c7ce4e2289dc764fc2cae8ba0d5cbe190f62fc09dee13517796d3d","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:03:10.533Z","source_record_sha256":"c44ab638aca59dd5eb6650b581ed3e0bc1b085e81a01b9e06fa1ebde07edc1ad"} +{"schema_version":1,"benchmark_version":"v1","run_id":"bd6bfb47-687b-4d08-9a70-f019316e623a","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":3,"started_at":"2026-07-23T10:03:10.533Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5d7380bab4c7ce4e2289dc764fc2cae8ba0d5cbe190f62fc09dee13517796d3d","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:03:37.293Z","source_record_sha256":"f1f1cec153e7696804111aaa35815a177088c70905bb2e81effa38b9f7dc0afa"} +{"schema_version":1,"benchmark_version":"v1","run_id":"c28bc1c8-94d0-4bed-a562-af06253fc011","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-23T10:03:37.293Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:04:19.806Z","source_record_sha256":"558fe12898a5153ca789f01f8d5dff4733850bbd892e4e13ceaecd52a9909ccd"} +{"schema_version":1,"benchmark_version":"v1","run_id":"9eff8a69-cb9a-4931-ae30-4230a29c6b2c","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-23T10:04:19.806Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:04:54.838Z","source_record_sha256":"222b9822b0bf52fdd42c8e4ef55761abc6715c95ef4b0a67a99937a60c59da23"} +{"schema_version":1,"benchmark_version":"v1","run_id":"81d9ac77-2d8f-4363-be1a-09193951a1b1","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T10:04:54.838Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:05:33.311Z","source_record_sha256":"8b485bf1ce1cf1c78376b3b005c724bc529b3ddd6789b540138dbae1c9a0303d"} +{"schema_version":1,"benchmark_version":"v1","run_id":"abc19032-2dc3-46fc-a861-05132c7eda65","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":1,"started_at":"2026-07-23T10:05:33.311Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f9568410d64286b9cc625ae8f4a38b3b350dee873f626cc44bb01cd92e5e7652","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T10:06:00.664Z","source_record_sha256":"9841f36f25b9775a95a122bfd640f89bbe141c6a0b8cda802a39479dd4b7a510"} +{"schema_version":1,"benchmark_version":"v1","run_id":"66977b1e-3302-469e-9e30-590d4b8cd542","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":2,"started_at":"2026-07-23T10:06:00.664Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f9568410d64286b9cc625ae8f4a38b3b350dee873f626cc44bb01cd92e5e7652","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T10:06:50.608Z","source_record_sha256":"5233af85fb24de9f413a64d9c076732b4b2f658cb5ee949299dc62dfe2a97544"} +{"schema_version":1,"benchmark_version":"v1","run_id":"b3001ec6-f8da-4dcc-b420-a40d0991337b","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":3,"started_at":"2026-07-23T10:06:50.608Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f9568410d64286b9cc625ae8f4a38b3b350dee873f626cc44bb01cd92e5e7652","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T10:07:33.561Z","source_record_sha256":"a5bad4a51d0d339497fed6f904985b0ebb406c88b9a791b07a9a735845003e09"} +{"schema_version":1,"benchmark_version":"v1","run_id":"7e26fd5b-b2f6-40bb-90dc-0d32665ffadd","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":1,"started_at":"2026-07-23T10:07:33.561Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66bd65e09a5a8cadf11017997e8362fae34ffb37649c18058a462de424f6e78b","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.98a153bd"},"inspection":{"selection_grounded":true,"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:08:06.053Z","source_record_sha256":"4f5f11c913c72853033c5cc8b76b8a599895f5000a28ca566ecbcfdfa5aedca0"} +{"schema_version":1,"benchmark_version":"v1","run_id":"936fb637-9c45-40c3-bfe5-33fb0bf5bc9a","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":2,"started_at":"2026-07-23T10:08:06.053Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66bd65e09a5a8cadf11017997e8362fae34ffb37649c18058a462de424f6e78b","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.98a153bd"},"inspection":{"selection_grounded":true,"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:08:47.521Z","source_record_sha256":"e7a26c6bcdbf34636feff685eb07deebf1ce0c47f985051dbd2b896f753ab3e3"} +{"schema_version":1,"benchmark_version":"v1","run_id":"85b215db-7bde-4512-aad4-781a905c7c38","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":3,"started_at":"2026-07-23T10:08:47.521Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66bd65e09a5a8cadf11017997e8362fae34ffb37649c18058a462de424f6e78b","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.98a153bd"},"inspection":{"selection_grounded":true,"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:09:22.117Z","source_record_sha256":"4af6c1eee5e48b043149b271dc644e1f7e6b575823f62f617d7479e9044e3313"} diff --git a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.summary.json b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.summary.json index 32a309d..a4a61cd 100644 --- a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.summary.json +++ b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.summary.json @@ -1,11 +1,11 @@ { "schema_version": 1, "methodology": "discover-call-v1", - "generated_at": "2026-07-23T10:09:22.150Z", + "generated_at": "2026-07-23T11:10:34.270Z", "models": [ { "model": "gpt-5.6-sol", - "lane": "pinned-model", + "lane": "configured-model", "tasks": 18, "trials_per_task": 3, "runs": 54, @@ -16,11 +16,11 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0.8333, "call_success_rate": 0.8846, - "result_valid_rate": 0.8846, + "result_nonempty_rate": 0.8846, "workflow_success_rate": 0.7222, - "workflow_success_wilson_95": [ - 0.5911, - 0.8238 + "workflow_success_task_cluster_bootstrap_95": [ + 0.5, + 0.8889 ], "failures_by_stage": { "adapter": 2 @@ -43,18 +43,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -69,18 +79,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -95,18 +115,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -121,18 +151,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -147,18 +187,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -173,18 +223,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -199,18 +259,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -225,18 +295,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -251,18 +331,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -277,18 +367,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -303,18 +403,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -329,18 +439,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -355,18 +475,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -381,18 +511,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -407,18 +547,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -433,18 +583,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": false, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -459,18 +619,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": false, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -485,18 +655,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": false, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -511,18 +691,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -537,18 +727,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -563,18 +763,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -589,18 +799,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": false, - "result_valid": false, + "result_nonempty": false, "workflow_success": false, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -615,18 +835,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": false, - "result_valid": false, + "result_nonempty": false, "workflow_success": false, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -641,18 +871,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": false, - "result_valid": false, + "result_nonempty": false, "workflow_success": false, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -667,18 +907,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": false, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -693,18 +943,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": false, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -719,18 +979,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": false, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -745,18 +1015,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -771,18 +1051,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -797,18 +1087,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -823,18 +1123,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -849,18 +1159,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -875,18 +1195,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -901,18 +1231,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -927,18 +1267,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -953,18 +1303,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -979,18 +1339,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": null, - "result_valid": null, + "result_nonempty": null, "workflow_success": false, "error_stage": "adapter", "error_reason": "tool_use_rejected", "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1005,18 +1375,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": false, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1031,18 +1411,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": null, - "result_valid": null, + "result_nonempty": null, "workflow_success": false, "error_stage": "adapter", "error_reason": "tool_use_rejected", "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1057,18 +1447,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1083,18 +1483,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1109,18 +1519,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1135,18 +1555,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1161,18 +1591,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1187,18 +1627,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1213,18 +1663,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1239,18 +1699,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1265,18 +1735,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1291,18 +1771,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": false, - "result_valid": false, + "result_nonempty": false, "workflow_success": false, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1317,18 +1807,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": false, - "result_valid": false, + "result_nonempty": false, "workflow_success": false, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1343,18 +1843,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": false, - "result_valid": false, + "result_nonempty": false, "workflow_success": false, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1369,18 +1879,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1395,18 +1915,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1421,18 +1951,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "pinned-model", + "lane": "configured-model", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } } ] diff --git a/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.runs.jsonl b/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.runs.jsonl index 05ce48b..209c806 100644 --- a/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.runs.jsonl +++ b/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.runs.jsonl @@ -1,54 +1,54 @@ -{"schema_version":1,"benchmark_version":"v1","run_id":"8ba500d0-14cb-46a7-a347-c9ba9be3a3a8","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"weather-london","trial":1,"started_at":"2026-07-23T07:52:46.344Z","status":"completed","discovery":{"result_tool_ids":["weather_gov.gridpoints.forecast.retrieve.v1.40a9a3ce","weather_gov.gridpoints.forecast.retrieve.v3.29bbec80","weather_gov.gridpoints.forecast.hourly.retrieve.v3.808ea240","amap_webservice.weather.weatherinfo.retrieve.v3","weather_gov.icons.retrieve.v3.ad0b4d80","visualcrossing.timeline.retrieve.v1","weather_gov.products.retrieve.v3.43ee57b8","weather_gov.products.retrieve.v1.18657145","weather_gov.gridpoints.retrieve.v1.5bc9e3c1","weather_gov.gridpoints.retrieve.v3.a0c89365"]},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"returned_tool_ids":["visualcrossing.timeline.retrieve.v1"],"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"execution_id":"f3dfb93b-4c10-43b3-983e-124d7ab9942c","result_valid":true},"finished_at":"2026-07-23T07:52:51.394Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"1f240f6b-6c51-4970-94c8-86a99dd6a36e","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"weather-london","trial":2,"started_at":"2026-07-23T07:52:51.394Z","status":"completed","discovery":{"result_tool_ids":["weather_gov.gridpoints.forecast.retrieve.v1.40a9a3ce","weather_gov.gridpoints.forecast.retrieve.v3.29bbec80","weather_gov.gridpoints.forecast.hourly.retrieve.v3.808ea240","amap_webservice.weather.weatherinfo.retrieve.v3","weather_gov.icons.retrieve.v3.ad0b4d80","visualcrossing.timeline.retrieve.v1","weather_gov.products.retrieve.v3.43ee57b8","weather_gov.products.retrieve.v1.18657145","weather_gov.gridpoints.retrieve.v1.5bc9e3c1","weather_gov.gridpoints.retrieve.v3.a0c89365"]},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"returned_tool_ids":["visualcrossing.timeline.retrieve.v1"],"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"execution_id":"853f33f6-5dec-4427-97e5-9edb5ed803c4","result_valid":true},"finished_at":"2026-07-23T07:52:55.748Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"891080de-9c5f-45d3-897f-94db7557d1c1","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"weather-london","trial":3,"started_at":"2026-07-23T07:52:55.748Z","status":"completed","discovery":{"result_tool_ids":["weather_gov.gridpoints.forecast.retrieve.v1.40a9a3ce","weather_gov.gridpoints.forecast.retrieve.v3.29bbec80","weather_gov.gridpoints.forecast.hourly.retrieve.v3.808ea240","amap_webservice.weather.weatherinfo.retrieve.v3","weather_gov.icons.retrieve.v3.ad0b4d80","visualcrossing.timeline.retrieve.v1","weather_gov.products.retrieve.v3.43ee57b8","weather_gov.products.retrieve.v1.18657145","weather_gov.gridpoints.retrieve.v1.5bc9e3c1","weather_gov.gridpoints.retrieve.v3.a0c89365"]},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"returned_tool_ids":["visualcrossing.timeline.retrieve.v1"],"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"execution_id":"265e970c-9989-4a41-bbd1-e39afe60000e","result_valid":true},"finished_at":"2026-07-23T07:52:59.070Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"f32129b5-3004-4733-90c2-6137c3a74d0f","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-23T07:52:59.070Z","status":"completed","discovery":{"result_tool_ids":["eodhd.live_v2.us_quote_delayed.retrieve.v1.f0e13d45","twelvedata.quote.retrieve.v1.affbefe3","massive_stocks.options.reference.conditions.list.v3.41a82e07","alphavantage.realtime_bulk_quotes.retrieve.v1","fiu_mcp_server.postv1stockquoteextend.create.v2.a30e86e7","hangseng_polysource.a_shares_live_quote.query.v2.10fe0581","mcp_gildata.usstockdailyquotes.v1","massive_stocks.stocks.marketstatus.retrieve.v1.3045ae6e","fiu_mcp_server.postv3stockquoteextend.create.v2.b35a7f02","financialmodelingprep.cusip.search.v1.ad19189f"]},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"returned_tool_ids":["twelvedata.quote.retrieve.v1.affbefe3"],"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"00eacb0a-ca6d-4379-b706-4b8ad85dd2e6","result_valid":true},"finished_at":"2026-07-23T07:53:02.034Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"28f35db3-bab5-4b00-9aa3-e29371acf680","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-23T07:53:02.035Z","status":"completed","discovery":{"result_tool_ids":["eodhd.live_v2.us_quote_delayed.retrieve.v1.f0e13d45","twelvedata.quote.retrieve.v1.affbefe3","massive_stocks.options.reference.conditions.list.v3.41a82e07","alphavantage.realtime_bulk_quotes.retrieve.v1","fiu_mcp_server.postv1stockquoteextend.create.v2.a30e86e7","hangseng_polysource.a_shares_live_quote.query.v2.10fe0581","mcp_gildata.usstockdailyquotes.v1","massive_stocks.stocks.marketstatus.retrieve.v1.3045ae6e","fiu_mcp_server.postv3stockquoteextend.create.v2.b35a7f02","financialmodelingprep.cusip.search.v1.ad19189f"]},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"returned_tool_ids":["twelvedata.quote.retrieve.v1.affbefe3"],"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"42dbe5fc-0e7b-4358-b908-36183fabc92b","result_valid":true},"finished_at":"2026-07-23T07:53:05.383Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"770c7467-c2b6-4090-8826-0182ccfae9da","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-23T07:53:05.383Z","status":"completed","discovery":{"result_tool_ids":["eodhd.live_v2.us_quote_delayed.retrieve.v1.f0e13d45","twelvedata.quote.retrieve.v1.affbefe3","massive_stocks.options.reference.conditions.list.v3.41a82e07","alphavantage.realtime_bulk_quotes.retrieve.v1","fiu_mcp_server.postv1stockquoteextend.create.v2.a30e86e7","hangseng_polysource.a_shares_live_quote.query.v2.10fe0581","mcp_gildata.usstockdailyquotes.v1","massive_stocks.stocks.marketstatus.retrieve.v1.3045ae6e","fiu_mcp_server.postv3stockquoteextend.create.v2.b35a7f02","financialmodelingprep.cusip.search.v1.ad19189f"]},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"returned_tool_ids":["twelvedata.quote.retrieve.v1.affbefe3"],"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"d8a6652f-f68e-49a6-b36f-e3acfd0409f9","result_valid":true},"finished_at":"2026-07-23T07:53:08.429Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"c4624c8d-fb7b-4d9e-9248-209a6679336b","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-23T07:53:08.429Z","status":"completed","discovery":{"result_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4","twelvedata.exchangerate.retrieve.v1.9eeb3b0d","coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","coinmarketcap.tools.priceconversion.retrieve.v2.24e991d9","bybitp2p.pay.fx.convert.v5.9221b0c4","coingecko_pro.exchangerates.retrieve.v3.0f6133ea","bybitp2p.payment.fx.convert.v5.7883b755","nbp_pl.exchangerates.rates.last.list.v1","coinmarketcap.tools.priceconversion.retrieve.v2.52e4cb2d","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d"]},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"returned_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4"],"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"835c61d6-148f-4a93-91d7-cdd862feb1e4","result_valid":true},"finished_at":"2026-07-23T07:53:11.737Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"4d565c9f-3a0c-4dc2-ad7c-4848b1e3f561","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-23T07:53:11.737Z","status":"completed","discovery":{"result_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4","twelvedata.exchangerate.retrieve.v1.9eeb3b0d","coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","coinmarketcap.tools.priceconversion.retrieve.v2.24e991d9","bybitp2p.pay.fx.convert.v5.9221b0c4","coingecko_pro.exchangerates.retrieve.v3.0f6133ea","bybitp2p.payment.fx.convert.v5.7883b755","nbp_pl.exchangerates.rates.last.list.v1","coinmarketcap.tools.priceconversion.retrieve.v2.52e4cb2d","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d"]},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"returned_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4"],"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"fa50a78b-1fdd-47ad-965a-1630fc88bafb","result_valid":true},"finished_at":"2026-07-23T07:53:14.963Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"c8ab76aa-1e1b-4c55-8f77-f583d283d344","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-23T07:53:14.963Z","status":"completed","discovery":{"result_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4","twelvedata.exchangerate.retrieve.v1.9eeb3b0d","coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","coinmarketcap.tools.priceconversion.retrieve.v2.24e991d9","bybitp2p.pay.fx.convert.v5.9221b0c4","coingecko_pro.exchangerates.retrieve.v3.0f6133ea","bybitp2p.payment.fx.convert.v5.7883b755","nbp_pl.exchangerates.rates.last.list.v1","coinmarketcap.tools.priceconversion.retrieve.v2.52e4cb2d","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d"]},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"returned_tool_ids":["twelvedata.currencyconversion.retrieve.v1.60fe48d4"],"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"a1ddb43f-7eb0-48aa-9ed8-f469952a4f62","result_valid":true},"finished_at":"2026-07-23T07:53:18.126Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"3a5d9a59-34e8-4209-b700-633819ddcedf","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"news-ai","trial":1,"started_at":"2026-07-23T07:53:18.126Z","status":"completed","discovery":{"result_tool_ids":["x_developer.2.news.search.retrieve.v1.19b3e678","newsdata.news.search.v1.b65ccc56","x_developer.2.tweets.search.recent.retrieve.v2.f424c8f9","qveris_finance.finance_news_aggregation_v1","newsapi.top_headlines.list.v2.38face94","linkup.search.v1","newsapi.everything.retrieve.v2.e77ab1df","brave_search.news.search.list.v1.6ab52eaa","jina_search.rerank.create.v2026.23dc486d","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic"]},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"returned_tool_ids":["newsapi.everything.retrieve.v2.e77ab1df"],"required_parameters":["q"]},"parameters":{"q":"artificial intelligence","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"execution_id":"9d02d147-e450-4fb9-ac2c-da720aba6f1d","result_valid":true},"finished_at":"2026-07-23T07:53:20.737Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"1190b998-aceb-484b-8e96-8f711b737082","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"news-ai","trial":2,"started_at":"2026-07-23T07:53:20.737Z","status":"completed","discovery":{"result_tool_ids":["x_developer.2.news.search.retrieve.v1.19b3e678","newsdata.news.search.v1.b65ccc56","x_developer.2.tweets.search.recent.retrieve.v2.f424c8f9","qveris_finance.finance_news_aggregation_v1","newsapi.top_headlines.list.v2.38face94","linkup.search.v1","newsapi.everything.retrieve.v2.e77ab1df","brave_search.news.search.list.v1.6ab52eaa","jina_search.rerank.create.v2026.23dc486d","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic"]},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"returned_tool_ids":["newsapi.everything.retrieve.v2.e77ab1df"],"required_parameters":["q"]},"parameters":{"q":"artificial intelligence","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"execution_id":"4eabeb8c-81d1-47fe-8885-a9514512ae19","result_valid":true},"finished_at":"2026-07-23T07:53:23.488Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"9e1336ba-27ab-4d05-b4e2-72f24e2d9fd2","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"news-ai","trial":3,"started_at":"2026-07-23T07:53:23.488Z","status":"completed","discovery":{"result_tool_ids":["x_developer.2.news.search.retrieve.v1.19b3e678","newsdata.news.search.v1.b65ccc56","x_developer.2.tweets.search.recent.retrieve.v2.f424c8f9","qveris_finance.finance_news_aggregation_v1","newsapi.top_headlines.list.v2.38face94","linkup.search.v1","newsapi.everything.retrieve.v2.e77ab1df","brave_search.news.search.list.v1.6ab52eaa","jina_search.rerank.create.v2026.23dc486d","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic"]},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"returned_tool_ids":["newsapi.everything.retrieve.v2.e77ab1df"],"required_parameters":["q"]},"parameters":{"q":"artificial intelligence","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"execution_id":"e139ec85-de0b-4fe1-b788-760f4474fe52","result_valid":true},"finished_at":"2026-07-23T07:53:26.274Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"7ddaaf21-9eb5-4cbf-9e9e-ad0cf2b4da30","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T07:53:26.275Z","status":"completed","discovery":{"result_tool_ids":["semantic_scholar.paper.search.bulk.execute.v1.239dfd66","qveris_academic.paper_aggregation_v1","semantic_scholar.paper.search.bulk.retrieve.v1.096de627","semantic_scholar.paper.search.match.retrieve.v1.b9f166f3","semantic_scholar.paper.search.match.retrieve.v1.d3fd06f6","semantic_scholar.paper.autocomplete.retrieve.v1.e16c9798","semantic_scholar.paper.autocomplete.list.v1.99d870f6","xiaosu.scholar.search.v1.f79d2374","semantic_scholar.author.search.query.v1.1c428051","semantic_scholar.author.search.list.v1.ea3671fb"]},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"returned_tool_ids":["qveris_academic.paper_aggregation_v1"],"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"execution_id":"4fdca88a-862e-4fa8-9ee1-79ae131b52e3","result_valid":true},"finished_at":"2026-07-23T07:53:30.277Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"aa7da099-31bf-4f14-94b5-04b148b81bde","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T07:53:30.278Z","status":"completed","discovery":{"result_tool_ids":["semantic_scholar.paper.search.bulk.execute.v1.239dfd66","qveris_academic.paper_aggregation_v1","semantic_scholar.paper.search.bulk.retrieve.v1.096de627","semantic_scholar.paper.search.match.retrieve.v1.b9f166f3","semantic_scholar.paper.search.match.retrieve.v1.d3fd06f6","semantic_scholar.paper.autocomplete.retrieve.v1.e16c9798","semantic_scholar.paper.autocomplete.list.v1.99d870f6","xiaosu.scholar.search.v1.f79d2374","semantic_scholar.author.search.query.v1.1c428051","semantic_scholar.author.search.list.v1.ea3671fb"]},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"returned_tool_ids":["qveris_academic.paper_aggregation_v1"],"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"execution_id":"99f73406-fcb5-46ba-b898-684c1ad6b735","result_valid":true},"finished_at":"2026-07-23T07:53:34.089Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"b42f18f4-c0fe-443f-9d07-cb4870e88af6","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T07:53:34.089Z","status":"completed","discovery":{"result_tool_ids":["semantic_scholar.paper.search.bulk.execute.v1.239dfd66","qveris_academic.paper_aggregation_v1","semantic_scholar.paper.search.bulk.retrieve.v1.096de627","semantic_scholar.paper.search.match.retrieve.v1.b9f166f3","semantic_scholar.paper.search.match.retrieve.v1.d3fd06f6","semantic_scholar.paper.autocomplete.retrieve.v1.e16c9798","semantic_scholar.paper.autocomplete.list.v1.99d870f6","xiaosu.scholar.search.v1.f79d2374","semantic_scholar.author.search.query.v1.1c428051","semantic_scholar.author.search.list.v1.ea3671fb"]},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"returned_tool_ids":["qveris_academic.paper_aggregation_v1"],"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"execution_id":"230b7448-b12b-4ce1-951e-3aef321fcd97","result_valid":true},"finished_at":"2026-07-23T07:53:37.784Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"ab798371-3cf3-4945-87c6-4240b8b9e36f","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T07:53:37.784Z","status":"failed","discovery":{"result_tool_ids":["api_sports.timezone.retrieve.v1.e993615d","api_sports.timezone.retrieve.v1.6792e6aa","api_sports.timezone.retrieve.v1.9b882d11","api_sports.timezone.list.v1.fafec612","api_sports.timezone.retrieve.v1.00ab54ff","api_sports.timezone.list.v1.bd878c58","api_sports.timezone.retrieve.v1.64cfc345","api_sports.timezone.list.v1.58366f46","api_sports.timezone.retrieve.v1.ae6ae8d7","api_sports.timezone.retrieve.v1.895e2257"]},"selection":{"tool_id":null},"inspection":{"returned_tool_ids":[],"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id","message":"Adapter did not return a non-empty tool_id"},"finished_at":"2026-07-23T07:53:39.282Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"3879f2c1-7e03-4819-b2b4-4dc353c9b6b6","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T07:53:39.282Z","status":"failed","discovery":{"result_tool_ids":["api_sports.timezone.retrieve.v1.e993615d","api_sports.timezone.retrieve.v1.6792e6aa","api_sports.timezone.retrieve.v1.9b882d11","api_sports.timezone.list.v1.fafec612","api_sports.timezone.retrieve.v1.00ab54ff","api_sports.timezone.list.v1.bd878c58","api_sports.timezone.retrieve.v1.64cfc345","api_sports.timezone.list.v1.58366f46","api_sports.timezone.retrieve.v1.ae6ae8d7","api_sports.timezone.retrieve.v1.895e2257"]},"selection":{"tool_id":null},"inspection":{"returned_tool_ids":[],"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id","message":"Adapter did not return a non-empty tool_id"},"finished_at":"2026-07-23T07:53:41.087Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"cb7160d0-c6ea-4e8b-a60e-bffc3e6747df","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T07:53:41.087Z","status":"failed","discovery":{"result_tool_ids":["api_sports.timezone.retrieve.v1.e993615d","api_sports.timezone.retrieve.v1.6792e6aa","api_sports.timezone.retrieve.v1.9b882d11","api_sports.timezone.list.v1.fafec612","api_sports.timezone.retrieve.v1.00ab54ff","api_sports.timezone.list.v1.bd878c58","api_sports.timezone.retrieve.v1.64cfc345","api_sports.timezone.list.v1.58366f46","api_sports.timezone.retrieve.v1.ae6ae8d7","api_sports.timezone.retrieve.v1.895e2257"]},"selection":{"tool_id":null},"inspection":{"returned_tool_ids":[],"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id","message":"Adapter did not return a non-empty tool_id"},"finished_at":"2026-07-23T07:53:42.486Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"c004662d-e17c-441a-8439-d18a4a1988c5","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T07:53:42.486Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.geocoder.reverse.v2","baidu_map.geocoder.query.v2","baidu_map.geocoding.reverse_geocode.v1","baidu_map.reversegeocoding.create.v1","baidu_map.reverse_geocoding.query.v2","baidu_map.geocoder.wgeoreverse.create.v1","amap_webservice.geocode.geo.retrieve.v3","baidu_map.geocoder.wgeobatch.create.v1","arcgis_ocean.geocoding.reverse_geocode.v1.4d40d93e","baidu_map.geoconv.convert.v1"]},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"returned_tool_ids":["amap_webservice.geocode.geo.retrieve.v3"],"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"06b8694c-6ab1-4f7b-8544-54d418e655fe","result_valid":true},"finished_at":"2026-07-23T07:53:46.409Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"aa9437b4-4a68-4467-9195-e421a33d242d","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T07:53:46.409Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.geocoder.reverse.v2","baidu_map.geocoder.query.v2","baidu_map.geocoding.reverse_geocode.v1","baidu_map.reversegeocoding.create.v1","baidu_map.reverse_geocoding.query.v2","baidu_map.geocoder.wgeoreverse.create.v1","amap_webservice.geocode.geo.retrieve.v3","baidu_map.geocoder.wgeobatch.create.v1","arcgis_ocean.geocoding.reverse_geocode.v1.4d40d93e","baidu_map.geoconv.convert.v1"]},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"returned_tool_ids":["amap_webservice.geocode.geo.retrieve.v3"],"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"81c8f609-3484-47ac-927a-cff6bb17c021","result_valid":true},"finished_at":"2026-07-23T07:53:51.677Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"c3632840-0ea8-46f3-9c0b-d721c14211ed","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T07:53:51.677Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.geocoder.reverse.v2","baidu_map.geocoder.query.v2","baidu_map.geocoding.reverse_geocode.v1","baidu_map.reversegeocoding.create.v1","baidu_map.reverse_geocoding.query.v2","baidu_map.geocoder.wgeoreverse.create.v1","amap_webservice.geocode.geo.retrieve.v3","baidu_map.geocoder.wgeobatch.create.v1","arcgis_ocean.geocoding.reverse_geocode.v1.4d40d93e","baidu_map.geoconv.convert.v1"]},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"returned_tool_ids":["amap_webservice.geocode.geo.retrieve.v3"],"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"execution_id":"98d922a3-7e54-40d6-8305-de4c6130323e","result_valid":true},"finished_at":"2026-07-23T07:53:55.919Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"a0d8e047-f8fa-4579-bec2-de6526299e1a","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-23T07:53:55.919Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.location.ip.retrieve.v1","decodo.isp_pay_per_ip.ip.retrieve.v1.bc2c6371","baidu_map.location.ip.execute.v1","virustotal.ip.info.retrieve.v3.ab34b0d1","amap_webservice.ip.retrieve.v3","weather_api.ip_lookup.retrieve.v1.e095d904","virustotal.ip_addresses.retrieve.v3.ab34b0d1","virustotal.ip.report.retrieve.v3.ab34b0d1","baidu_map.ip_location.retrieve.v1","virustotal.ip-address.report.retrieve.v2.79606e33"]},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"returned_tool_ids":["virustotal.ip.info.retrieve.v3.ab34b0d1"],"required_parameters":["ip"]},"parameters":{"ip":"8.8.8.8"},"call":{"attempted":true,"success":true,"execution_id":"85c9f4d0-b112-47f3-b500-8809ccedd99f","result_valid":true},"finished_at":"2026-07-23T07:53:58.844Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"bc8d8e9b-d160-465f-bda4-3b185ef33115","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-23T07:53:58.844Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.location.ip.retrieve.v1","decodo.isp_pay_per_ip.ip.retrieve.v1.bc2c6371","baidu_map.location.ip.execute.v1","virustotal.ip.info.retrieve.v3.ab34b0d1","amap_webservice.ip.retrieve.v3","weather_api.ip_lookup.retrieve.v1.e095d904","virustotal.ip_addresses.retrieve.v3.ab34b0d1","virustotal.ip.report.retrieve.v3.ab34b0d1","baidu_map.ip_location.retrieve.v1","virustotal.ip-address.report.retrieve.v2.79606e33"]},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"returned_tool_ids":["virustotal.ip.info.retrieve.v3.ab34b0d1"],"required_parameters":["ip"]},"parameters":{"ip":"8.8.8.8"},"call":{"attempted":true,"success":true,"execution_id":"a0581017-05ca-4296-818f-7f00105ca92f","result_valid":true},"finished_at":"2026-07-23T07:54:02.027Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"cd7516ab-c10d-4af9-8063-273a79f68af3","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-23T07:54:02.027Z","status":"completed","discovery":{"result_tool_ids":["baidu_map.location.ip.retrieve.v1","decodo.isp_pay_per_ip.ip.retrieve.v1.bc2c6371","baidu_map.location.ip.execute.v1","virustotal.ip.info.retrieve.v3.ab34b0d1","amap_webservice.ip.retrieve.v3","weather_api.ip_lookup.retrieve.v1.e095d904","virustotal.ip_addresses.retrieve.v3.ab34b0d1","virustotal.ip.report.retrieve.v3.ab34b0d1","baidu_map.ip_location.retrieve.v1","virustotal.ip-address.report.retrieve.v2.79606e33"]},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"returned_tool_ids":["virustotal.ip.info.retrieve.v3.ab34b0d1"],"required_parameters":["ip"]},"parameters":{"ip":"8.8.8.8"},"call":{"attempted":true,"success":true,"execution_id":"e1bcc9d4-9f48-45c2-8bd8-acdc868eb913","result_valid":true},"finished_at":"2026-07-23T07:54:05.637Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"be7b9cb2-88b4-4148-b1aa-4a17f035c27a","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"crypto-price","trial":1,"started_at":"2026-07-23T07:54:05.637Z","status":"completed","discovery":{"result_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","binance.avg_price.retrieve.v1.aef0de4f","etherscan.api.stats.ethprice.retrieve.v2.8eeca59e","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.251242d7","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.9dc24be4","binance.ticker.price.list.v3.8675eca0","binance.avgPrice.retrieve.v3.7b0852b4","deribit.public.get_index_price.retrieve.v2.d47dcdcb","binance.ticker.query.v1.0437113d","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d"]},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"returned_tool_ids":["binance.ticker.price.list.v3.8675eca0"],"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"execution_id":"6cab41f3-55c0-4bf9-9bd5-c99de6753874","result_valid":true},"finished_at":"2026-07-23T07:54:09.422Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"fa26dcf3-d647-434e-8fc4-cfb726806704","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"crypto-price","trial":2,"started_at":"2026-07-23T07:54:09.422Z","status":"completed","discovery":{"result_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","binance.avg_price.retrieve.v1.aef0de4f","etherscan.api.stats.ethprice.retrieve.v2.8eeca59e","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.251242d7","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.9dc24be4","binance.ticker.price.list.v3.8675eca0","binance.avgPrice.retrieve.v3.7b0852b4","deribit.public.get_index_price.retrieve.v2.d47dcdcb","binance.ticker.query.v1.0437113d","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d"]},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"returned_tool_ids":["binance.ticker.price.list.v3.8675eca0"],"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"execution_id":"3606598a-880a-486b-a236-0b07cae220a3","result_valid":true},"finished_at":"2026-07-23T07:54:12.396Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"5a70425d-4ab1-48a2-a484-b91463c77e34","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"crypto-price","trial":3,"started_at":"2026-07-23T07:54:12.396Z","status":"completed","discovery":{"result_tool_ids":["coinmarketcap.tools.price-conversion.execute.v1.2b99a4df","binance.avg_price.retrieve.v1.aef0de4f","etherscan.api.stats.ethprice.retrieve.v2.8eeca59e","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.251242d7","coinmarketcap.cryptocurrency.listings.latest.retrieve.v2.9dc24be4","binance.ticker.price.list.v3.8675eca0","binance.avgPrice.retrieve.v3.7b0852b4","deribit.public.get_index_price.retrieve.v2.d47dcdcb","binance.ticker.query.v1.0437113d","coinmarketcap.tools.priceconversion.retrieve.v1.53d0670d"]},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"returned_tool_ids":["binance.ticker.price.list.v3.8675eca0"],"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"execution_id":"2fde8e07-7939-4545-8321-6ce7d557f37a","result_valid":true},"finished_at":"2026-07-23T07:54:15.174Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"cc44232f-8d5d-40e4-87d0-914004721dfc","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"web-search","trial":1,"started_at":"2026-07-23T07:54:15.174Z","status":"completed","discovery":{"result_tool_ids":["linkup.search.v1","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic","cloudsway.smart_search_overseas","brave_search.images.search.list.v1","firecrawl.search.create.v2.a99b6f74","serpapi.locations.locations.list.v1","x_developer.2.news.search.retrieve.v1.19b3e678","x_developer.2.spaces.search.retrieve.v1.43a1f31c","tavily.map.execute.v1","brave_search.web.search.list.v1.f02ed09d"]},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"returned_tool_ids":["cloudsway.smart_search_overseas"],"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"execution_id":"263066a3-72ae-4a8d-9422-aaa015590464","result_valid":true},"finished_at":"2026-07-23T07:54:17.456Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"0ecbcede-88b3-403b-934f-8e1ce9909259","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"web-search","trial":2,"started_at":"2026-07-23T07:54:17.456Z","status":"completed","discovery":{"result_tool_ids":["linkup.search.v1","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic","cloudsway.smart_search_overseas","brave_search.images.search.list.v1","firecrawl.search.create.v2.a99b6f74","serpapi.locations.locations.list.v1","x_developer.2.news.search.retrieve.v1.19b3e678","x_developer.2.spaces.search.retrieve.v1.43a1f31c","tavily.map.execute.v1","brave_search.web.search.list.v1.f02ed09d"]},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"returned_tool_ids":["cloudsway.smart_search_overseas"],"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"execution_id":"99ba7ee3-2175-4e5b-b887-fd486bf4f03b","result_valid":true},"finished_at":"2026-07-23T07:54:20.037Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"7b1f0b31-d0a7-4fb7-b807-d8633bd42387","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"web-search","trial":3,"started_at":"2026-07-23T07:54:20.037Z","status":"completed","discovery":{"result_tool_ids":["linkup.search.v1","xiaosu.smartsearch.search.retrieve.v2.6c50f296_domestic","cloudsway.smart_search_overseas","brave_search.images.search.list.v1","firecrawl.search.create.v2.a99b6f74","serpapi.locations.locations.list.v1","x_developer.2.news.search.retrieve.v1.19b3e678","x_developer.2.spaces.search.retrieve.v1.43a1f31c","tavily.map.execute.v1","brave_search.web.search.list.v1.f02ed09d"]},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"returned_tool_ids":["cloudsway.smart_search_overseas"],"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"execution_id":"e8756117-a9dc-4130-8f78-ab65b1f14b72","result_valid":true},"finished_at":"2026-07-23T07:54:22.508Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"e85eed29-7b7c-4c24-875b-7c4ec6f2afc6","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"air-quality","trial":1,"started_at":"2026-07-23T07:54:22.508Z","status":"completed","discovery":{"result_tool_ids":["openweathermap.air_pollution.current.v2_5","openweathermap.air_pollution.retrieve.v2.5","openweathermap.air_pollution.history.v2_5","openweathermap.air_pollution.history.retrieve.v2.5","openweathermap.air_pollution.forecast.retrieve.v2.5","epa_envirofacts.qaannualperformanceevaluations.bystate.retrieve.v1.61a2aa7d","epa_envirofacts.transactionsqaannualperformanceevaluations.bystate.retrieve.v1.4143b8bf","epa_envirofacts.transactionssample.bysite.retrieve.v1.93889e72","epa_envirofacts.annualdata.bybox.retrieve.v1.4b74a0d6","epa_envirofacts.dailydata.bycounty.retrieve.v1.f1ee8d1b"]},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"returned_tool_ids":["openweathermap.air_pollution.current.v2_5"],"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"execution_id":"e73a69f2-623b-4c65-9994-cbfe8248e4d9","result_valid":true},"finished_at":"2026-07-23T07:54:25.490Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"203b9a62-2140-4edb-b40a-61c95d9c3146","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"air-quality","trial":2,"started_at":"2026-07-23T07:54:25.490Z","status":"completed","discovery":{"result_tool_ids":["openweathermap.air_pollution.current.v2_5","openweathermap.air_pollution.retrieve.v2.5","openweathermap.air_pollution.history.v2_5","openweathermap.air_pollution.history.retrieve.v2.5","openweathermap.air_pollution.forecast.retrieve.v2.5","epa_envirofacts.qaannualperformanceevaluations.bystate.retrieve.v1.61a2aa7d","epa_envirofacts.transactionsqaannualperformanceevaluations.bystate.retrieve.v1.4143b8bf","epa_envirofacts.transactionssample.bysite.retrieve.v1.93889e72","epa_envirofacts.annualdata.bybox.retrieve.v1.4b74a0d6","epa_envirofacts.dailydata.bycounty.retrieve.v1.f1ee8d1b"]},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"returned_tool_ids":["openweathermap.air_pollution.current.v2_5"],"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"execution_id":"8bddf105-bf44-485e-901a-77235d5de868","result_valid":true},"finished_at":"2026-07-23T07:54:27.929Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"95854287-bfed-486a-9aff-20d839999229","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"air-quality","trial":3,"started_at":"2026-07-23T07:54:27.929Z","status":"completed","discovery":{"result_tool_ids":["openweathermap.air_pollution.current.v2_5","openweathermap.air_pollution.retrieve.v2.5","openweathermap.air_pollution.history.v2_5","openweathermap.air_pollution.history.retrieve.v2.5","openweathermap.air_pollution.forecast.retrieve.v2.5","epa_envirofacts.qaannualperformanceevaluations.bystate.retrieve.v1.61a2aa7d","epa_envirofacts.transactionsqaannualperformanceevaluations.bystate.retrieve.v1.4143b8bf","epa_envirofacts.transactionssample.bysite.retrieve.v1.93889e72","epa_envirofacts.annualdata.bybox.retrieve.v1.4b74a0d6","epa_envirofacts.dailydata.bycounty.retrieve.v1.f1ee8d1b"]},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"returned_tool_ids":["openweathermap.air_pollution.current.v2_5"],"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"execution_id":"720179b8-bbdb-4eea-b58c-10c52d7da23a","result_valid":true},"finished_at":"2026-07-23T07:54:30.101Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"ff1c9565-ffe8-4a70-adc2-b5215c3410ff","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-news","trial":1,"started_at":"2026-07-23T07:54:30.101Z","status":"completed","discovery":{"result_tool_ids":["finnhub.news.retrieve.v1.2e72ea5d","finnhub.market.news.list.v1","finnhub.news.retrieve.v1","finnhub.news.retrieve.v1.51b2567e","finnhub.news.execute.v1","finnhub.companynews.retrieve.v1","alphavantage.news_sentiment.query.v1.467a92c0","finnhub.companynews.retrieve.v1.4d11f47b","finnhub.companynews.retrieve.v1.e428b704","finnhub.companynews.execute.v1"]},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"returned_tool_ids":["finnhub.market.news.list.v1"],"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"execution_id":"3672275c-75aa-4a0b-b4d2-036c94fbbaa2","result_valid":true},"finished_at":"2026-07-23T07:54:33.694Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"56a668ab-8a69-4277-8729-4a1717af668c","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-news","trial":2,"started_at":"2026-07-23T07:54:33.695Z","status":"completed","discovery":{"result_tool_ids":["finnhub.news.retrieve.v1.2e72ea5d","finnhub.market.news.list.v1","finnhub.news.retrieve.v1","finnhub.news.retrieve.v1.51b2567e","finnhub.news.execute.v1","finnhub.companynews.retrieve.v1","alphavantage.news_sentiment.query.v1.467a92c0","finnhub.companynews.retrieve.v1.4d11f47b","finnhub.companynews.retrieve.v1.e428b704","finnhub.companynews.execute.v1"]},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"returned_tool_ids":["finnhub.market.news.list.v1"],"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"execution_id":"0db2574e-d68c-4fd6-87d7-d2683b157b26","result_valid":true},"finished_at":"2026-07-23T07:54:37.509Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"2fd403d8-a11c-41e1-9390-d076ca5dfdb0","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-news","trial":3,"started_at":"2026-07-23T07:54:37.510Z","status":"completed","discovery":{"result_tool_ids":["finnhub.news.retrieve.v1.2e72ea5d","finnhub.market.news.list.v1","finnhub.news.retrieve.v1","finnhub.news.retrieve.v1.51b2567e","finnhub.news.execute.v1","finnhub.companynews.retrieve.v1","alphavantage.news_sentiment.query.v1.467a92c0","finnhub.companynews.retrieve.v1.4d11f47b","finnhub.companynews.retrieve.v1.e428b704","finnhub.companynews.execute.v1"]},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"returned_tool_ids":["finnhub.market.news.list.v1"],"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"execution_id":"262940d0-2688-431c-9ea7-9e19e83d7ba9","result_valid":true},"finished_at":"2026-07-23T07:54:41.063Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"8b3f5092-fa39-4426-a908-35e2af2c9312","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T07:54:41.063Z","status":"completed","discovery":{"result_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8","dataforseo.dataforseo_labs.google.domain_whois_overview.live.execute.v3","dataforseo.databases.domains.query.v3.c1720ec0","virustotal.domains.report.retrieve.v3.c5459740","bainiu_api.common.company_domain.query.v1.e978ca32","virustotal.domain_reports.retrieve.v3.c5459740","dataforseo.domain_analytics.technologies.domains_by_technology.live.create.v3.bfc3dc2d","virustotal.domains.retrieve.v3.c5459740","virustotal.domains.analyse.v3.c755c69e","dataforseo.domain_analytics.technologies.domains_by_html_terms.live.create.v3.d3760ae1"]},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"returned_tool_ids":["virustotal.domains.report.retrieve.v3.c5459740"],"required_parameters":["domain"]},"parameters":{"domain":"example.com"},"call":{"attempted":true,"success":true,"execution_id":"2aaef5cc-ccf1-4eff-96f9-07eaa355d05a","result_valid":true},"finished_at":"2026-07-23T07:54:44.215Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"d454cf08-3fbd-4a50-bb4a-9af533a5675d","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T07:54:44.215Z","status":"completed","discovery":{"result_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8","dataforseo.dataforseo_labs.google.domain_whois_overview.live.execute.v3","dataforseo.databases.domains.query.v3.c1720ec0","virustotal.domains.report.retrieve.v3.c5459740","bainiu_api.common.company_domain.query.v1.e978ca32","virustotal.domain_reports.retrieve.v3.c5459740","dataforseo.domain_analytics.technologies.domains_by_technology.live.create.v3.bfc3dc2d","virustotal.domains.retrieve.v3.c5459740","virustotal.domains.analyse.v3.c755c69e","dataforseo.domain_analytics.technologies.domains_by_html_terms.live.create.v3.d3760ae1"]},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"returned_tool_ids":["virustotal.domains.report.retrieve.v3.c5459740"],"required_parameters":["domain"]},"parameters":{"domain":"example.com"},"call":{"attempted":true,"success":true,"execution_id":"f90745cc-a050-4e1f-a2f5-bbc689b2721a","result_valid":true},"finished_at":"2026-07-23T07:54:47.714Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"d24fe5b1-5ffb-427b-a45a-37baffde5d36","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T07:54:47.714Z","status":"completed","discovery":{"result_tool_ids":["dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8","dataforseo.dataforseo_labs.google.domain_whois_overview.live.execute.v3","dataforseo.databases.domains.query.v3.c1720ec0","virustotal.domains.report.retrieve.v3.c5459740","bainiu_api.common.company_domain.query.v1.e978ca32","virustotal.domain_reports.retrieve.v3.c5459740","dataforseo.domain_analytics.technologies.domains_by_technology.live.create.v3.bfc3dc2d","virustotal.domains.retrieve.v3.c5459740","virustotal.domains.analyse.v3.c755c69e","dataforseo.domain_analytics.technologies.domains_by_html_terms.live.create.v3.d3760ae1"]},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"returned_tool_ids":["virustotal.domains.report.retrieve.v3.c5459740"],"required_parameters":["domain"]},"parameters":{"domain":"example.com"},"call":{"attempted":true,"success":true,"execution_id":"8f66f75e-1ccb-48fe-b37f-580d0e6d86f7","result_valid":true},"finished_at":"2026-07-23T07:54:50.804Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"66cecd06-6fce-4084-94a3-4997791e2303","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"image-search","trial":1,"started_at":"2026-07-23T07:54:50.804Z","status":"completed","discovery":{"result_tool_ids":["brave_search.images.search.list.v1","shutterstock.images.search.list.v2.69059569","shutterstock.images.search.list.v2.c2b44682","dataforseo.serp.google.search_by_image.task.create.v3","jina_search.embeddings.create.v2026.5c7cb661","jina_search.rerank.create.v2026.23dc486d","linkup.search.v1","shutterstock.images.search.list.v2.15f15845","shutterstock.images.search.suggestions.list.v2.f6850c10","shutterstock.images.search.suggestions.list.v2"]},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"returned_tool_ids":["brave_search.images.search.list.v1"],"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge"},"call":{"attempted":true,"success":true,"execution_id":"399c7b71-6f72-425b-89f4-96486128d89b","result_valid":true},"finished_at":"2026-07-23T07:54:54.158Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"ffdc3990-5991-4e84-bfb9-4997434ec7e8","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"image-search","trial":2,"started_at":"2026-07-23T07:54:54.158Z","status":"completed","discovery":{"result_tool_ids":["brave_search.images.search.list.v1","shutterstock.images.search.list.v2.69059569","shutterstock.images.search.list.v2.c2b44682","dataforseo.serp.google.search_by_image.task.create.v3","jina_search.embeddings.create.v2026.5c7cb661","jina_search.rerank.create.v2026.23dc486d","linkup.search.v1","shutterstock.images.search.list.v2.15f15845","shutterstock.images.search.suggestions.list.v2.f6850c10","shutterstock.images.search.suggestions.list.v2"]},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"returned_tool_ids":["brave_search.images.search.list.v1"],"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge"},"call":{"attempted":true,"success":true,"execution_id":"c632bc27-e454-4809-b1b9-76abb2c24b3f","result_valid":true},"finished_at":"2026-07-23T07:54:57.764Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"8fc69d41-be93-424e-ae30-754fb957f63b","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"image-search","trial":3,"started_at":"2026-07-23T07:54:57.764Z","status":"completed","discovery":{"result_tool_ids":["brave_search.images.search.list.v1","shutterstock.images.search.list.v2.69059569","shutterstock.images.search.list.v2.c2b44682","dataforseo.serp.google.search_by_image.task.create.v3","jina_search.embeddings.create.v2026.5c7cb661","jina_search.rerank.create.v2026.23dc486d","linkup.search.v1","shutterstock.images.search.list.v2.15f15845","shutterstock.images.search.suggestions.list.v2.f6850c10","shutterstock.images.search.suggestions.list.v2"]},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"returned_tool_ids":["brave_search.images.search.list.v1"],"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge"},"call":{"attempted":true,"success":true,"execution_id":"53c4f6b7-26d5-4fa8-a424-41dbee950069","result_valid":true},"finished_at":"2026-07-23T07:55:01.885Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"5cd557cc-6800-41f3-b397-f194865cc601","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-holidays","trial":1,"started_at":"2026-07-23T07:55:01.885Z","status":"completed","discovery":{"result_tool_ids":["finnhub.stock.marketholiday.retrieve.v1.75d6c1f9","finnhub.stock.marketholiday.retrieve.v1","finnhub.stock.marketholiday.execute.v1","eodhd.exchange_details.retrieve.v1.7da32390","finnhub.stock.marketholiday.retrieve.v1.5c5bfe9a","massive_stocks.stocks.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.retrieve.v1","massive_stocks.options.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.list.v1","massive_stocks.options.marketstatus.retrieve.v1.3045ae6e"]},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"returned_tool_ids":["finnhub.stock.marketholiday.retrieve.v1"],"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"execution_id":"8ee1342e-41a1-4127-874f-91a442534efe","result_valid":true},"finished_at":"2026-07-23T07:55:04.949Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"5c5c80cb-d46e-42c9-bbb7-c173a487851d","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-holidays","trial":2,"started_at":"2026-07-23T07:55:04.949Z","status":"completed","discovery":{"result_tool_ids":["finnhub.stock.marketholiday.retrieve.v1.75d6c1f9","finnhub.stock.marketholiday.retrieve.v1","finnhub.stock.marketholiday.execute.v1","eodhd.exchange_details.retrieve.v1.7da32390","finnhub.stock.marketholiday.retrieve.v1.5c5bfe9a","massive_stocks.stocks.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.retrieve.v1","massive_stocks.options.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.list.v1","massive_stocks.options.marketstatus.retrieve.v1.3045ae6e"]},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"returned_tool_ids":["finnhub.stock.marketholiday.retrieve.v1"],"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"execution_id":"9db6a044-57ff-4216-8d0e-84893a01cfd8","result_valid":true},"finished_at":"2026-07-23T07:55:07.898Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"e0657299-6f28-4851-8554-1b4d50126bfb","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"market-holidays","trial":3,"started_at":"2026-07-23T07:55:07.898Z","status":"completed","discovery":{"result_tool_ids":["finnhub.stock.marketholiday.retrieve.v1.75d6c1f9","finnhub.stock.marketholiday.retrieve.v1","finnhub.stock.marketholiday.execute.v1","eodhd.exchange_details.retrieve.v1.7da32390","finnhub.stock.marketholiday.retrieve.v1.5c5bfe9a","massive_stocks.stocks.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.retrieve.v1","massive_stocks.options.marketstatus.upcoming.retrieve.v1.353f0493","tradier.markets.calendar.list.v1","massive_stocks.options.marketstatus.retrieve.v1.3045ae6e"]},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"returned_tool_ids":["finnhub.stock.marketholiday.retrieve.v1"],"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"execution_id":"735442ad-1d5a-46a7-92b3-7166f78e1c8f","result_valid":true},"finished_at":"2026-07-23T07:55:11.714Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"53649f10-7f15-439b-b58e-c8e01a811995","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-23T07:55:11.714Z","status":"completed","discovery":{"result_tool_ids":["usgs_earthquake_catalog.event.events.list.v1","usgs_earthquake_catalog.fdsnws.event.query.retrieve.v1.025549c9","usgs_earthquake_catalog.earthquakes.query.v1","usgs_earthquake_catalog.fdsnws.event.query.v1.025549c9","usgs_earthquake_catalog.events.query.v1.025549c9","usgs_earthquake.catalogs.list.v1","usgs_earthquake.contributors.list.v1","usgs-earthquake-event-api.event.query.list.v1","usgs_earthquake.parameters.list.v1","usgs_earthquake.version.retrieve.v1"]},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"returned_tool_ids":["usgs_earthquake_catalog.event.events.list.v1"],"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"execution_id":"8db7bed3-07ad-48e8-ae80-b5ca5235bb49","result_valid":true},"finished_at":"2026-07-23T07:55:20.097Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"cdefdbdd-5005-472b-b3e0-d33e3ecc51c5","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-23T07:55:20.097Z","status":"completed","discovery":{"result_tool_ids":["usgs_earthquake_catalog.event.events.list.v1","usgs_earthquake_catalog.fdsnws.event.query.retrieve.v1.025549c9","usgs_earthquake_catalog.earthquakes.query.v1","usgs_earthquake_catalog.fdsnws.event.query.v1.025549c9","usgs_earthquake_catalog.events.query.v1.025549c9","usgs_earthquake.catalogs.list.v1","usgs_earthquake.contributors.list.v1","usgs-earthquake-event-api.event.query.list.v1","usgs_earthquake.parameters.list.v1","usgs_earthquake.version.retrieve.v1"]},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"returned_tool_ids":["usgs_earthquake_catalog.event.events.list.v1"],"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"execution_id":"1b7c3dc5-3ada-4646-ac8c-735dc5251824","result_valid":true},"finished_at":"2026-07-23T07:55:28.789Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"929db9db-2ef4-466d-a80b-2e14af1f4c7c","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T07:55:28.789Z","status":"completed","discovery":{"result_tool_ids":["usgs_earthquake_catalog.event.events.list.v1","usgs_earthquake_catalog.fdsnws.event.query.retrieve.v1.025549c9","usgs_earthquake_catalog.earthquakes.query.v1","usgs_earthquake_catalog.fdsnws.event.query.v1.025549c9","usgs_earthquake_catalog.events.query.v1.025549c9","usgs_earthquake.catalogs.list.v1","usgs_earthquake.contributors.list.v1","usgs-earthquake-event-api.event.query.list.v1","usgs_earthquake.parameters.list.v1","usgs_earthquake.version.retrieve.v1"]},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"returned_tool_ids":["usgs_earthquake_catalog.event.events.list.v1"],"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"execution_id":"d61b9ae7-e2ef-4632-b542-9a61aaa061f2","result_valid":true},"finished_at":"2026-07-23T07:55:36.177Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"711a0f3a-7f40-4b3d-ab18-0fd661803757","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"company-profile","trial":1,"started_at":"2026-07-23T07:55:36.177Z","status":"completed","discovery":{"result_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2","bainiu_api.company.detail.execute.v1.f6b528e7","bainiu_api.openapi.company.alter.execute.v1.8fa090d1","financialmodelingprep.stable.companynotes.retrieve.v1.930aa468","tradefeeds.compinfo.retrieve.v1.c2314765","financialmodelingprep.stable.secfilingscompanysearch.symbol.retrieve.v1.5cf7397d","financialmodelingprep.search.cusip.retrieve.v1.51d0c621","bainiu_api.openapi.common.company_info_deep_search.execute.v1.8e5f009d","finnhub.stock.profile2.retrieve.v1.59df26f6","bainiu_api.openapi.common.company_lianxi_public.execute.v1.bd6261e8"]},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"returned_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"],"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":true,"execution_id":"0938428f-f710-49e3-92d2-7aaf488c788b","result_valid":true},"finished_at":"2026-07-23T07:55:40.433Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"342d9249-b2e3-43e1-9db2-9d6584889277","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"company-profile","trial":2,"started_at":"2026-07-23T07:55:40.433Z","status":"completed","discovery":{"result_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2","bainiu_api.company.detail.execute.v1.f6b528e7","bainiu_api.openapi.company.alter.execute.v1.8fa090d1","financialmodelingprep.stable.companynotes.retrieve.v1.930aa468","tradefeeds.compinfo.retrieve.v1.c2314765","financialmodelingprep.stable.secfilingscompanysearch.symbol.retrieve.v1.5cf7397d","financialmodelingprep.search.cusip.retrieve.v1.51d0c621","bainiu_api.openapi.common.company_info_deep_search.execute.v1.8e5f009d","finnhub.stock.profile2.retrieve.v1.59df26f6","bainiu_api.openapi.common.company_lianxi_public.execute.v1.bd6261e8"]},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"returned_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"],"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":true,"execution_id":"174e2b74-0f18-4300-8dec-ecb8775cde8e","result_valid":true},"finished_at":"2026-07-23T07:55:43.901Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"063b8a50-9c2c-41dd-89df-4a639c354c74","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"company-profile","trial":3,"started_at":"2026-07-23T07:55:43.901Z","status":"completed","discovery":{"result_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2","bainiu_api.company.detail.execute.v1.f6b528e7","bainiu_api.openapi.company.alter.execute.v1.8fa090d1","financialmodelingprep.stable.companynotes.retrieve.v1.930aa468","tradefeeds.compinfo.retrieve.v1.c2314765","financialmodelingprep.stable.secfilingscompanysearch.symbol.retrieve.v1.5cf7397d","financialmodelingprep.search.cusip.retrieve.v1.51d0c621","bainiu_api.openapi.common.company_info_deep_search.execute.v1.8e5f009d","finnhub.stock.profile2.retrieve.v1.59df26f6","bainiu_api.openapi.common.company_lianxi_public.execute.v1.bd6261e8"]},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"returned_tool_ids":["financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"],"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":true,"execution_id":"fa6ff70f-0e6d-49de-8bae-23b085cb48fc","result_valid":true},"finished_at":"2026-07-23T07:55:47.413Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"34788c2f-c12d-46b8-9a88-d4285ab363de","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"sports-standings","trial":1,"started_at":"2026-07-23T07:55:47.413Z","status":"completed","discovery":{"result_tool_ids":["api_sports.standings.list.v1.98a153bd","api_sports.standings.list.v1.39568ba4","api_sports.standings.groups.list.v1.0d8a40cf","api_sports.standings.groups.list.v1.a077a680","api_sports.standings.stages.list.v1.5ba13a89","api_sports.standings.retrieve.v1.ff019893","api_sports.standings.groups.list.v1.57393fb3","api_sports.standings.stages.list.v1.838f769a","api-football.standings.list.v3.76f7703e","api_sports.standings.list.v1.5956be03"]},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"returned_tool_ids":["api_sports.standings.list.v1.39568ba4"],"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"execution_id":"b8c1f968-3226-47f9-8ba1-9e99406099ee","result_valid":true},"finished_at":"2026-07-23T07:55:51.371Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"1ae754a2-5fe1-4baa-85b8-e5c6e8664576","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"sports-standings","trial":2,"started_at":"2026-07-23T07:55:51.371Z","status":"completed","discovery":{"result_tool_ids":["api_sports.standings.list.v1.98a153bd","api_sports.standings.list.v1.39568ba4","api_sports.standings.groups.list.v1.0d8a40cf","api_sports.standings.groups.list.v1.a077a680","api_sports.standings.stages.list.v1.5ba13a89","api_sports.standings.retrieve.v1.ff019893","api_sports.standings.groups.list.v1.57393fb3","api_sports.standings.stages.list.v1.838f769a","api-football.standings.list.v3.76f7703e","api_sports.standings.list.v1.5956be03"]},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"returned_tool_ids":["api_sports.standings.list.v1.39568ba4"],"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"execution_id":"13f07f7b-c843-4cb8-abd3-cd10aca5ae09","result_valid":true},"finished_at":"2026-07-23T07:55:54.947Z"} -{"schema_version":1,"benchmark_version":"v1","run_id":"ddfc24d1-5ac5-4861-b959-7013b1160520","model":"oracle-v1","metadata":{"lane":"oracle","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true},"task_id":"sports-standings","trial":3,"started_at":"2026-07-23T07:55:54.947Z","status":"completed","discovery":{"result_tool_ids":["api_sports.standings.list.v1.98a153bd","api_sports.standings.list.v1.39568ba4","api_sports.standings.groups.list.v1.0d8a40cf","api_sports.standings.groups.list.v1.a077a680","api_sports.standings.stages.list.v1.5ba13a89","api_sports.standings.retrieve.v1.ff019893","api_sports.standings.groups.list.v1.57393fb3","api_sports.standings.stages.list.v1.838f769a","api-football.standings.list.v3.76f7703e","api_sports.standings.list.v1.5956be03"]},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"returned_tool_ids":["api_sports.standings.list.v1.39568ba4"],"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"execution_id":"6bdc5e3b-9d36-49d7-805d-8a5536dfc0a8","result_valid":true},"finished_at":"2026-07-23T07:55:58.608Z"} +{"schema_version":1,"benchmark_version":"v1","run_id":"8ba500d0-14cb-46a7-a347-c9ba9be3a3a8","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":1,"started_at":"2026-07-23T07:52:46.344Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"b4772dc9d4f828e88d87b4ace08a968439e097cee7c9613886d037cc95f7eb04","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:52:51.394Z","source_record_sha256":"0af61ea34d3519146acf43ad3a651907d4379f14dbc9311939a1e0f223b4f560"} +{"schema_version":1,"benchmark_version":"v1","run_id":"1f240f6b-6c51-4970-94c8-86a99dd6a36e","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":2,"started_at":"2026-07-23T07:52:51.394Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"b4772dc9d4f828e88d87b4ace08a968439e097cee7c9613886d037cc95f7eb04","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:52:55.748Z","source_record_sha256":"9fe1cf51eda32858b6eae402db46b71a4a67b87a7f3482a6fa3e747497931fe6"} +{"schema_version":1,"benchmark_version":"v1","run_id":"891080de-9c5f-45d3-897f-94db7557d1c1","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":3,"started_at":"2026-07-23T07:52:55.748Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"b4772dc9d4f828e88d87b4ace08a968439e097cee7c9613886d037cc95f7eb04","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:52:59.070Z","source_record_sha256":"de18dbbb42fd505950f46e6f353c3c611431bc728b3a6c50fcceaced1ecb20eb"} +{"schema_version":1,"benchmark_version":"v1","run_id":"f32129b5-3004-4733-90c2-6137c3a74d0f","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-23T07:52:59.070Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8c4e8e2ab56f10ce503086b98981d6be57f293b4043dec00a86237144a107a5b","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:02.034Z","source_record_sha256":"bceda6101f8eeaa37088617b0b9b7b5aa4280285a8b9db10657cfcbfe7a96a0e"} +{"schema_version":1,"benchmark_version":"v1","run_id":"28f35db3-bab5-4b00-9aa3-e29371acf680","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-23T07:53:02.035Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8c4e8e2ab56f10ce503086b98981d6be57f293b4043dec00a86237144a107a5b","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:05.383Z","source_record_sha256":"dc3d2d8c9423bb629806b01d70116a244e2edb3c225af885af5c54638aaf1e42"} +{"schema_version":1,"benchmark_version":"v1","run_id":"770c7467-c2b6-4090-8826-0182ccfae9da","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-23T07:53:05.383Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8c4e8e2ab56f10ce503086b98981d6be57f293b4043dec00a86237144a107a5b","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:08.429Z","source_record_sha256":"9450c107d0c58da627d2c3b1ed2f9a115d286340bfb5011576b0ca1091a6a0f0"} +{"schema_version":1,"benchmark_version":"v1","run_id":"c4624c8d-fb7b-4d9e-9248-209a6679336b","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-23T07:53:08.429Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"84ea39c886ae326a41eb2ef257097f3e0ad96d07404ab489516a62c468e23e0e","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:11.737Z","source_record_sha256":"7a7f0968645bd379de70740d7fd9a67a8b52b5bd637e9981a4cddace3fcd2c0a"} +{"schema_version":1,"benchmark_version":"v1","run_id":"4d565c9f-3a0c-4dc2-ad7c-4848b1e3f561","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-23T07:53:11.737Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"84ea39c886ae326a41eb2ef257097f3e0ad96d07404ab489516a62c468e23e0e","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:14.963Z","source_record_sha256":"76a8ffbec9cf85c811492d2a7b89ed017fe0e6a4bacdc435f335acb9424f123f"} +{"schema_version":1,"benchmark_version":"v1","run_id":"c8ab76aa-1e1b-4c55-8f77-f583d283d344","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-23T07:53:14.963Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"84ea39c886ae326a41eb2ef257097f3e0ad96d07404ab489516a62c468e23e0e","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:18.126Z","source_record_sha256":"a9ad392d5968fb6dc60ca31ee93bf238579d4fe58710f3512f0c279fe29aa197"} +{"schema_version":1,"benchmark_version":"v1","run_id":"3a5d9a59-34e8-4209-b700-633819ddcedf","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":1,"started_at":"2026-07-23T07:53:18.126Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"15c3ab376fd0853e249e998c30c4632d06a8cb26ae60ae74080fdeaeab085a29","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"artificial intelligence","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:20.737Z","source_record_sha256":"06d2a66d7ec4df06e6881e8568069ca35541904373ef7d9cd1df8a43b3433167"} +{"schema_version":1,"benchmark_version":"v1","run_id":"1190b998-aceb-484b-8e96-8f711b737082","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":2,"started_at":"2026-07-23T07:53:20.737Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"15c3ab376fd0853e249e998c30c4632d06a8cb26ae60ae74080fdeaeab085a29","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"artificial intelligence","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:23.488Z","source_record_sha256":"e7fa7c4d1a74281901b6142fee9b9f4124addb0f964f8a93a201664ea0e93784"} +{"schema_version":1,"benchmark_version":"v1","run_id":"9e1336ba-27ab-4d05-b4e2-72f24e2d9fd2","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":3,"started_at":"2026-07-23T07:53:23.488Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"15c3ab376fd0853e249e998c30c4632d06a8cb26ae60ae74080fdeaeab085a29","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"artificial intelligence","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:26.274Z","source_record_sha256":"e1eb12914dfef2709c88243eec74e97e1a6be81f60bbb60878f71aa16d238500"} +{"schema_version":1,"benchmark_version":"v1","run_id":"7ddaaf21-9eb5-4cbf-9e9e-ad0cf2b4da30","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T07:53:26.275Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:30.277Z","source_record_sha256":"7f876ab3626ab04941230d16c52d7ad76831f5c7eca25e27ced0795f330b21b5"} +{"schema_version":1,"benchmark_version":"v1","run_id":"aa7da099-31bf-4f14-94b5-04b148b81bde","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T07:53:30.278Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:34.089Z","source_record_sha256":"14497b8df3945d1204c9f850aff21659fed945adf62f0282570a7fd03b000b8d"} +{"schema_version":1,"benchmark_version":"v1","run_id":"b42f18f4-c0fe-443f-9d07-cb4870e88af6","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T07:53:34.089Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:37.784Z","source_record_sha256":"6671f9b91d3b83f7ad972c4764083b7aeedfbcc2e2f0483c454d8d2b9d90e2b0"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ab798371-3cf3-4945-87c6-4240b8b9e36f","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T07:53:37.784Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"97d9f951a9f193786150828de15857112d2439f3832ea0b92efcd5108ada92af","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id","message":"Adapter did not return a non-empty tool_id"},"finished_at":"2026-07-23T07:53:39.282Z","source_record_sha256":"334143d7288f3d7bd9e0e1542e391875145a4f8d296ac3deab16320d4de545ee"} +{"schema_version":1,"benchmark_version":"v1","run_id":"3879f2c1-7e03-4819-b2b4-4dc353c9b6b6","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T07:53:39.282Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"97d9f951a9f193786150828de15857112d2439f3832ea0b92efcd5108ada92af","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id","message":"Adapter did not return a non-empty tool_id"},"finished_at":"2026-07-23T07:53:41.087Z","source_record_sha256":"664918ea59c543cacbe4d218fc05fbd56995b603c5981355eb040dd1b64da7af"} +{"schema_version":1,"benchmark_version":"v1","run_id":"cb7160d0-c6ea-4e8b-a60e-bffc3e6747df","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T07:53:41.087Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"97d9f951a9f193786150828de15857112d2439f3832ea0b92efcd5108ada92af","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id","message":"Adapter did not return a non-empty tool_id"},"finished_at":"2026-07-23T07:53:42.486Z","source_record_sha256":"f020400865f1d1eee9552674edd440cdb237b26d9dbe01217fd1704109c7bc5c"} +{"schema_version":1,"benchmark_version":"v1","run_id":"c004662d-e17c-441a-8439-d18a4a1988c5","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T07:53:42.486Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f3a89453c750d6ddfee82c21bfd021c18cbefd53654879c10f93d3b6200c7d13","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:46.409Z","source_record_sha256":"81cc941960ad0e0e7d1ca50a6703be7197c26fa3190096771441100819fb438b"} +{"schema_version":1,"benchmark_version":"v1","run_id":"aa9437b4-4a68-4467-9195-e421a33d242d","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T07:53:46.409Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f3a89453c750d6ddfee82c21bfd021c18cbefd53654879c10f93d3b6200c7d13","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:51.677Z","source_record_sha256":"98c2f8f11295550da5dc09caaeb1cbba0e058a4e0672402acb9052882f0162ce"} +{"schema_version":1,"benchmark_version":"v1","run_id":"c3632840-0ea8-46f3-9c0b-d721c14211ed","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T07:53:51.677Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f3a89453c750d6ddfee82c21bfd021c18cbefd53654879c10f93d3b6200c7d13","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:55.919Z","source_record_sha256":"07899db418e2d3ff161a1468af71d0d8d9f0d5ecb0e3aaeda74c8501b5c858a3"} +{"schema_version":1,"benchmark_version":"v1","run_id":"a0d8e047-f8fa-4579-bec2-de6526299e1a","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-23T07:53:55.919Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"bed0baef996d3d7151e059d230ea273b29388e076cd6fd76c201699fe0e77d92","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true,"required_parameters":["ip"]},"parameters":{"ip":"8.8.8.8"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:58.844Z","source_record_sha256":"417a087f5fff31d941d1450f386d7f2b7cf26493e2b56a32a0c89aceba03110b"} +{"schema_version":1,"benchmark_version":"v1","run_id":"bc8d8e9b-d160-465f-bda4-3b185ef33115","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-23T07:53:58.844Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"bed0baef996d3d7151e059d230ea273b29388e076cd6fd76c201699fe0e77d92","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true,"required_parameters":["ip"]},"parameters":{"ip":"8.8.8.8"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:02.027Z","source_record_sha256":"7b3a08f02c7ebbc54497ee554d7c7508d0a36d537fe86a0da0e967108153a0a6"} +{"schema_version":1,"benchmark_version":"v1","run_id":"cd7516ab-c10d-4af9-8063-273a79f68af3","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-23T07:54:02.027Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"bed0baef996d3d7151e059d230ea273b29388e076cd6fd76c201699fe0e77d92","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true,"required_parameters":["ip"]},"parameters":{"ip":"8.8.8.8"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:05.637Z","source_record_sha256":"554942cdc248efa73636668cf133e807fe106f6ba81b17b6a3bbc0792846c1ae"} +{"schema_version":1,"benchmark_version":"v1","run_id":"be7b9cb2-88b4-4148-b1aa-4a17f035c27a","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":1,"started_at":"2026-07-23T07:54:05.637Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66d9bb9820728c627264f0ade0a1ca2560a70f6bcb8e608cc1ff1dcba98fc27c","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:09.422Z","source_record_sha256":"894f6a0e29bb8e829274f97ccda1360b23dddd9642e8967b73c7164c0fbd7438"} +{"schema_version":1,"benchmark_version":"v1","run_id":"fa26dcf3-d647-434e-8fc4-cfb726806704","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":2,"started_at":"2026-07-23T07:54:09.422Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66d9bb9820728c627264f0ade0a1ca2560a70f6bcb8e608cc1ff1dcba98fc27c","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:12.396Z","source_record_sha256":"7e6204b967be6bed5da44ba84fc66b245ec98f6ee5b919d4c80dcb436dc3a203"} +{"schema_version":1,"benchmark_version":"v1","run_id":"5a70425d-4ab1-48a2-a484-b91463c77e34","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":3,"started_at":"2026-07-23T07:54:12.396Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66d9bb9820728c627264f0ade0a1ca2560a70f6bcb8e608cc1ff1dcba98fc27c","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:15.174Z","source_record_sha256":"b90d6f2b430710e87f66e708b63dd2e3907a7110314961ef14dcef2e799b98f9"} +{"schema_version":1,"benchmark_version":"v1","run_id":"cc44232f-8d5d-40e4-87d0-914004721dfc","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":1,"started_at":"2026-07-23T07:54:15.174Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6ce1454e6f0eb9e506f81f2b5680e6c466aa63d6ebe9a2fd2dcbc8a9ef43cbfb","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:17.456Z","source_record_sha256":"30451fa2e418594b6e586f1d65e1d271d69c88a98333fe7fa180ec7a579bfc68"} +{"schema_version":1,"benchmark_version":"v1","run_id":"0ecbcede-88b3-403b-934f-8e1ce9909259","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":2,"started_at":"2026-07-23T07:54:17.456Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6ce1454e6f0eb9e506f81f2b5680e6c466aa63d6ebe9a2fd2dcbc8a9ef43cbfb","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:20.037Z","source_record_sha256":"cc6f64a7dada4a2d453e01e4c97a9dcf587205e82aa0063be7b0e758b231c75e"} +{"schema_version":1,"benchmark_version":"v1","run_id":"7b1f0b31-d0a7-4fb7-b807-d8633bd42387","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":3,"started_at":"2026-07-23T07:54:20.037Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6ce1454e6f0eb9e506f81f2b5680e6c466aa63d6ebe9a2fd2dcbc8a9ef43cbfb","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:22.508Z","source_record_sha256":"982d57090dc3a49756ae97a55ad4c8b2d881887f167b439982fab4c075a75ebc"} +{"schema_version":1,"benchmark_version":"v1","run_id":"e85eed29-7b7c-4c24-875b-7c4ec6f2afc6","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":1,"started_at":"2026-07-23T07:54:22.508Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:25.490Z","source_record_sha256":"439b63730d6b09940fdf2a8ab4bdefd01f4d0500f874f0f43f9bbf718521d81e"} +{"schema_version":1,"benchmark_version":"v1","run_id":"203b9a62-2140-4edb-b40a-61c95d9c3146","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":2,"started_at":"2026-07-23T07:54:25.490Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:27.929Z","source_record_sha256":"faa4bdd3fc214b2cb211097805f866c9e9328d33548e9ebb5418c3b0cc7ce47d"} +{"schema_version":1,"benchmark_version":"v1","run_id":"95854287-bfed-486a-9aff-20d839999229","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":3,"started_at":"2026-07-23T07:54:27.929Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:30.101Z","source_record_sha256":"2f8fd3729b4ecadac2c9e08867dcccb1c46ef5f3014f4cc01e030c26b08401d4"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ff1c9565-ffe8-4a70-adc2-b5215c3410ff","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":1,"started_at":"2026-07-23T07:54:30.101Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"a06d4241bd0f4717242da76bd0a368da5831e1734b134538a73c2bdaccb08f02","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:33.694Z","source_record_sha256":"6fd516f622f28c4308eb9a8fb0fc8ba0da6bf4e55ca0d0bccbaf314f8514c12d"} +{"schema_version":1,"benchmark_version":"v1","run_id":"56a668ab-8a69-4277-8729-4a1717af668c","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":2,"started_at":"2026-07-23T07:54:33.695Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"a06d4241bd0f4717242da76bd0a368da5831e1734b134538a73c2bdaccb08f02","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:37.509Z","source_record_sha256":"e9f371194aacbe14fdb754ae7ca81cfed9d449fe60bc2b0e7db8b4eced05d7e4"} +{"schema_version":1,"benchmark_version":"v1","run_id":"2fd403d8-a11c-41e1-9390-d076ca5dfdb0","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":3,"started_at":"2026-07-23T07:54:37.510Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"a06d4241bd0f4717242da76bd0a368da5831e1734b134538a73c2bdaccb08f02","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:41.063Z","source_record_sha256":"462b2a9d2956048e37863bb20115229e7cdf7eec0f0c38e926fa28e3e69f87d3"} +{"schema_version":1,"benchmark_version":"v1","run_id":"8b3f5092-fa39-4426-a908-35e2af2c9312","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T07:54:41.063Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"selection_grounded":true,"required_parameters":["domain"]},"parameters":{"domain":"example.com"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:44.215Z","source_record_sha256":"bb6cd41a0bc9404975ac0ce301c90dfa89db04f157f2baf52c386d078353248d"} +{"schema_version":1,"benchmark_version":"v1","run_id":"d454cf08-3fbd-4a50-bb4a-9af533a5675d","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T07:54:44.215Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"selection_grounded":true,"required_parameters":["domain"]},"parameters":{"domain":"example.com"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:47.714Z","source_record_sha256":"83adcd9d73f2cbcb2401cee3c0236f99274f1d94b4f5bcaae9f2b2b7fd0cea2d"} +{"schema_version":1,"benchmark_version":"v1","run_id":"d24fe5b1-5ffb-427b-a45a-37baffde5d36","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T07:54:47.714Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"selection_grounded":true,"required_parameters":["domain"]},"parameters":{"domain":"example.com"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:50.804Z","source_record_sha256":"f2c213a9cf5de6fb38b9a6b5c533ad4ec1a6782a088b14b16534373073244be9"} +{"schema_version":1,"benchmark_version":"v1","run_id":"66cecd06-6fce-4084-94a3-4997791e2303","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":1,"started_at":"2026-07-23T07:54:50.804Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"ee3ecf6abb4efbab753f21f53cf48773a835c92a3130aaa099de9ab94938c11f","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:54.158Z","source_record_sha256":"a890d50ba9be3b67f26af31bdfb78cef8f90c9114710fd6ea3de002d3571f13d"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ffdc3990-5991-4e84-bfb9-4997434ec7e8","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":2,"started_at":"2026-07-23T07:54:54.158Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"ee3ecf6abb4efbab753f21f53cf48773a835c92a3130aaa099de9ab94938c11f","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:57.764Z","source_record_sha256":"f47672c5937d11c4dbc47389860eab92a1e9b524f175e2660244ff5de2f9562e"} +{"schema_version":1,"benchmark_version":"v1","run_id":"8fc69d41-be93-424e-ae30-754fb957f63b","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":3,"started_at":"2026-07-23T07:54:57.764Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"ee3ecf6abb4efbab753f21f53cf48773a835c92a3130aaa099de9ab94938c11f","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:01.885Z","source_record_sha256":"79a25af369d8e25cfe3d8675f3de51c3167bb7b6a295b2ba0dd6cc2086ae602f"} +{"schema_version":1,"benchmark_version":"v1","run_id":"5cd557cc-6800-41f3-b397-f194865cc601","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":1,"started_at":"2026-07-23T07:55:01.885Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"94c7741de8644b5da225afc2ab92aaca833f1b9970bd8ef60ab25afcbd393b4f","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:04.949Z","source_record_sha256":"96a966391d65a0f6043821c49d8a00b24c60d3dca3f384168389f3d6fe7790f6"} +{"schema_version":1,"benchmark_version":"v1","run_id":"5c5c80cb-d46e-42c9-bbb7-c173a487851d","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":2,"started_at":"2026-07-23T07:55:04.949Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"94c7741de8644b5da225afc2ab92aaca833f1b9970bd8ef60ab25afcbd393b4f","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:07.898Z","source_record_sha256":"e30ad712f81d01a8e10cc3cbb7aca35ba8691162103607f77c9f2b989249df29"} +{"schema_version":1,"benchmark_version":"v1","run_id":"e0657299-6f28-4851-8554-1b4d50126bfb","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":3,"started_at":"2026-07-23T07:55:07.898Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"94c7741de8644b5da225afc2ab92aaca833f1b9970bd8ef60ab25afcbd393b4f","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:11.714Z","source_record_sha256":"ddb4dd8f9c7e67b446b0afbac933185d49b2f9027b079dfd511548552f3e6df3"} +{"schema_version":1,"benchmark_version":"v1","run_id":"53649f10-7f15-439b-b58e-c8e01a811995","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-23T07:55:11.714Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:20.097Z","source_record_sha256":"59f4bd89d66c54fe7486331376c9ba579108485c9a765ecc207405b698402cae"} +{"schema_version":1,"benchmark_version":"v1","run_id":"cdefdbdd-5005-472b-b3e0-d33e3ecc51c5","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-23T07:55:20.097Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:28.789Z","source_record_sha256":"9cc943cb8d10c628630fb2425caccb1a8c3093774d7b7dda357ac6cb074b8d01"} +{"schema_version":1,"benchmark_version":"v1","run_id":"929db9db-2ef4-466d-a80b-2e14af1f4c7c","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T07:55:28.789Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:36.177Z","source_record_sha256":"7cb4afef06406a0e19a1ee831041bc440418e7a2819a80a18360e9386f6d7668"} +{"schema_version":1,"benchmark_version":"v1","run_id":"711a0f3a-7f40-4b3d-ab18-0fd661803757","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":1,"started_at":"2026-07-23T07:55:36.177Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9ca02b19c7d86f46ad7a35b1f996cc0789a5a1cb92c6c567fbf2233aefa59ed4","selection_grounded":true},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:40.433Z","source_record_sha256":"bb8a1ce15aaca2982405e532624ca02946339e8355c0a4b192b75a9a5e886b81"} +{"schema_version":1,"benchmark_version":"v1","run_id":"342d9249-b2e3-43e1-9db2-9d6584889277","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":2,"started_at":"2026-07-23T07:55:40.433Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9ca02b19c7d86f46ad7a35b1f996cc0789a5a1cb92c6c567fbf2233aefa59ed4","selection_grounded":true},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:43.901Z","source_record_sha256":"f8e45b5268313edd9fcf9dc8fc42facdac41bbf8aa973d87f286cab763f06ab8"} +{"schema_version":1,"benchmark_version":"v1","run_id":"063b8a50-9c2c-41dd-89df-4a639c354c74","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":3,"started_at":"2026-07-23T07:55:43.901Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9ca02b19c7d86f46ad7a35b1f996cc0789a5a1cb92c6c567fbf2233aefa59ed4","selection_grounded":true},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:47.413Z","source_record_sha256":"25401223352ec2b8591b28d187ddd4a8215e1ceec71e81d878195b48b3e11951"} +{"schema_version":1,"benchmark_version":"v1","run_id":"34788c2f-c12d-46b8-9a88-d4285ab363de","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":1,"started_at":"2026-07-23T07:55:47.413Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8533341a13a1599cae14cf51f2a2e2642be5fae6d57aa42b445fc87cf9803a1f","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"selection_grounded":true,"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:51.371Z","source_record_sha256":"2c20d520e1e846149c959212418c6111000c94379f2adac4e9f4a1f98c013e7c"} +{"schema_version":1,"benchmark_version":"v1","run_id":"1ae754a2-5fe1-4baa-85b8-e5c6e8664576","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":2,"started_at":"2026-07-23T07:55:51.371Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8533341a13a1599cae14cf51f2a2e2642be5fae6d57aa42b445fc87cf9803a1f","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"selection_grounded":true,"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:54.947Z","source_record_sha256":"a66bede429e003a5177bcab371c06cfb944263d91f57f05c2ebddbd991e58cb7"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ddfc24d1-5ac5-4861-b959-7013b1160520","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":3,"started_at":"2026-07-23T07:55:54.947Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8533341a13a1599cae14cf51f2a2e2642be5fae6d57aa42b445fc87cf9803a1f","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"selection_grounded":true,"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:58.608Z","source_record_sha256":"f0eef74f6476cb2e6b010b7b0bef148fd16ead166a062cf4c11e3fcf8e5f2921"} diff --git a/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.summary.json b/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.summary.json index 9b58557..4d72a09 100644 --- a/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.summary.json +++ b/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.summary.json @@ -1,11 +1,11 @@ { "schema_version": 1, "methodology": "discover-call-v1", - "generated_at": "2026-07-23T07:55:58.643Z", + "generated_at": "2026-07-23T11:10:34.385Z", "models": [ { "model": "oracle-v1", - "lane": "oracle", + "lane": "reference", "tasks": 18, "trials_per_task": 3, "runs": 54, @@ -16,11 +16,11 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0.9444, "call_success_rate": 1, - "result_valid_rate": 1, + "result_nonempty_rate": 1, "workflow_success_rate": 0.9444, - "workflow_success_wilson_95": [ - 0.8489, - 0.9809 + "workflow_success_task_cluster_bootstrap_95": [ + 0.8333, + 1 ], "failures_by_stage": { "select": 3 @@ -43,18 +43,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -69,18 +79,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -95,18 +115,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -121,18 +151,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -147,18 +187,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -173,18 +223,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -199,18 +259,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -225,18 +295,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -251,18 +331,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -277,18 +367,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -303,18 +403,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -329,18 +439,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -355,18 +475,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -381,18 +511,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -407,18 +547,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -433,18 +583,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": null, - "result_valid": null, + "result_nonempty": null, "workflow_success": false, "error_stage": "select", "error_reason": "missing_tool_id", "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -459,18 +619,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": null, - "result_valid": null, + "result_nonempty": null, "workflow_success": false, "error_stage": "select", "error_reason": "missing_tool_id", "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -485,18 +655,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": null, - "result_valid": null, + "result_nonempty": null, "workflow_success": false, "error_stage": "select", "error_reason": "missing_tool_id", "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -511,18 +691,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -537,18 +727,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -563,18 +763,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -589,18 +799,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -615,18 +835,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -641,18 +871,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -667,18 +907,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -693,18 +943,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -719,18 +979,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -745,18 +1015,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -771,18 +1051,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -797,18 +1087,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -823,18 +1123,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -849,18 +1159,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -875,18 +1195,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -901,18 +1231,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -927,18 +1267,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -953,18 +1303,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -979,18 +1339,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1005,18 +1375,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1031,18 +1411,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1057,18 +1447,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1083,18 +1483,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1109,18 +1519,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1135,18 +1555,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1161,18 +1591,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1187,18 +1627,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1213,18 +1663,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1239,18 +1699,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1265,18 +1735,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1291,18 +1771,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1317,18 +1807,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1343,18 +1843,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1369,18 +1879,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1395,18 +1915,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } }, { @@ -1421,18 +1951,28 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_valid": true, + "result_nonempty": true, "workflow_success": true, "error_stage": null, "error_reason": null, "metadata": { - "lane": "oracle", + "lane": "reference", "adapter_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1", "toolkit_revision": "5006c2a0789e2352944ffd87b1a154b96ec0566f", "task_set_sha256": "d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866", "api_base_url": "https://qveris.ai/api/v1", "discovery_limit": 10, - "execute": true + "execute": true, + "model_revision": "unreported", + "api_revision": "unreported", + "catalog_revision": "unreported", + "catalog_observation_sha256": "dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591", + "runtime": { + "node": "unreported", + "platform": "unreported", + "arch": "unreported" + }, + "publication_policy": "public-artifact-v1" } } ] diff --git a/benchmarks/discover-call/results/README.md b/benchmarks/discover-call/results/README.md index a8c79b2..cd1fe3c 100644 --- a/benchmarks/discover-call/results/README.md +++ b/benchmarks/discover-call/results/README.md @@ -1,12 +1,14 @@ # Published benchmark results -## 2026-07-23 — v3 comparison lanes +## 2026-07-23 — v3 diagnostic comparison -The v3 baseline runs the deterministic Oracle and pinned-model lanes over the -same 18 tasks, three trials per task, Top 10 discovery limit, and real QVeris -calls. No failed trial was removed or selectively rerun. +This diagnostic run compares the curated reference route and a configured model +over the same 18 tasks, three trials per task, Top 10 discovery limit, and real +QVeris calls. No failed trial was removed or selectively rerun. It is not the +official quality baseline because v3 has a known Bitcoin constraint-scoring +false negative. -| Metric | Oracle | `gpt-5.6-sol` pinned model | +| Metric | Curated reference route | `gpt-5.6-sol` configured model | | --- | ---: | ---: | | Runs | 54 | 54 | | Completed and executed | 51 / 54 | 52 / 54 | @@ -17,10 +19,13 @@ calls. No failed trial was removed or selectively rerun. | Call success among attempted calls | 100% (51 / 51) | 88.46% (46 / 52) | | Non-empty result among attempted calls | 100% (51 / 51) | 88.46% (46 / 52) | | Strict workflow success | 94.44% (51 / 54) | 72.22% (39 / 54) | -| Workflow success, 95% Wilson interval | 84.89%–98.09% | 59.11%–82.38% | +| Workflow success, 95% task-cluster bootstrap | 83.33%–100% | 50.00%–88.89% | -The strict routing gap is **22.22 percentage points**: Oracle 94.44% minus -pinned model 72.22%. +The v3 strict benchmark gap is **22.22 percentage points**: curated reference +route 94.44% minus configured model 72.22%. It includes three Bitcoin false +negatives: successful non-empty calls used the provider-specific `id=1`, while +v3 accepted only values containing `BTC`. Therefore this gap is diagnostic, +not a publishable model-routing delta. Shared reproducibility metadata: @@ -29,19 +34,22 @@ Shared reproducibility metadata: - task-set SHA-256: `d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866` - API base URL: `https://qveris.ai/api/v1` +- API revision: `unreported` (the legacy runner did not capture it) +- catalog revision: `unreported` - discovery limit: `10` -Oracle metadata: +Curated-reference metadata: -- model: `oracle-v1` -- lane: `oracle` +- legacy model identifier: `oracle-v1` +- published lane: `reference` - adapter revision: `5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1` -Pinned-model metadata: +Configured-model metadata: - model: `gpt-5.6-sol` -- lane: `pinned-model` +- lane: `configured-model` +- provider model revision: `unreported`; no immutable snapshot claim is made - reasoning effort: `medium` - Codex CLI: `0.144.1` - adapter revision: @@ -54,15 +62,15 @@ Artifacts: - [`2026-07-23-gpt-5.6-sol-v3.runs.jsonl`](2026-07-23-gpt-5.6-sol-v3.runs.jsonl) - [`2026-07-23-gpt-5.6-sol-v3.summary.json`](2026-07-23-gpt-5.6-sol-v3.summary.json) -### v3 interpretation +### v3 diagnostic interpretation -The Oracle's three strict failures are all `timezone-tokyo`: the fixed -discovery query returned no Top 10 capability that accepts a city or -coordinates. All 51 Oracle-selected calls succeeded and returned non-empty +The curated reference route's three strict failures are all `timezone-tokyo`: +the fixed discovery query returned no Top 10 capability that accepts a city or +coordinates. All 51 reference-selected calls succeeded and returned non-empty results. -The pinned model also missed Tokyo semantically in all three trials: it selected -a successful timezone-list capability with no Tokyo input. Six well-formed +The configured model also missed Tokyo semantically in all three trials: it +selected a successful timezone-list capability with no Tokyo input. Six well-formed calls returned `success: false`: three selected `weather_api.ip_lookup.retrieve.v1.e095d904`, and three selected `tradefeeds.compinfo.retrieve.v1.c2314765`. Two domain-intelligence @@ -74,8 +82,8 @@ Three cryptocurrency calls succeeded and returned non-empty Bitcoin results via CoinMarketCap's provider-specific numeric `id=1`, but the deterministic task constraint expects a parameter value containing `BTC`; these remain strict constraint misses rather than being silently reclassified after the run. This -known identifier-mapping limitation should be addressed in a future immutable -task-set revision. +known identifier-mapping limitation is fixed in immutable `tasks/v4.jsonl`. +The v3 records remain unchanged in meaning. ## 2026-07-23 — `gpt-5.6-sol` historical v2 baseline @@ -93,7 +101,7 @@ No failed trial was removed or selectively rerun. | Constraint accuracy | 75.93% | | Call success among attempted calls | 88.00% (44 / 50) | | Strict workflow success | 64.81% (35 / 54) | -| Workflow success, 95% Wilson interval | 51.48%–76.18% | +| Workflow success, 95% task-cluster bootstrap | 42.59%–85.19% | Reproducibility metadata: @@ -107,6 +115,8 @@ Reproducibility metadata: - task-set SHA-256: `67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba` - API base URL: `https://qveris.ai/api/v1` +- API revision: `unreported` (the legacy runner did not capture it) +- catalog revision: `unreported` - discovery limit: `10` Artifacts: @@ -135,3 +145,8 @@ current v2 alias scorer does not split into separate base/quote constraints, three URL-encoded news queries, and three timezone-list selections that executed successfully but did not accept Tokyo as an input. These details are why the strict workflow metric should not be interpreted as API availability alone. + +All checked-in run files are sanitized public artifacts produced under +`public-artifact-v1`. They omit execution/search/connection identifiers and the +ordered discovery catalog. Raw operational records are not stored in this +repository. diff --git a/benchmarks/discover-call/src/api.mjs b/benchmarks/discover-call/src/api.mjs index df63592..53bf9f9 100644 --- a/benchmarks/discover-call/src/api.mjs +++ b/benchmarks/discover-call/src/api.mjs @@ -12,6 +12,8 @@ export function createApiClient({ const key = typeof apiKey === 'string' ? apiKey.trim() : ''; if (!key || /[\r\n]/.test(key)) throw new Error('QVERIS_API_KEY is required'); const resolvedBaseUrl = normalizeBaseUrl(baseUrl); + const observedApiRevisions = new Set(); + const observedCatalogRevisions = new Set(); if (!Number.isInteger(maxRetries) || maxRetries < 0 || maxRetries > 10) { throw new Error('maxRetries must be an integer from 0 to 10'); } @@ -39,6 +41,8 @@ export function createApiClient({ await Promise.resolve(response.body?.cancel?.()).catch(() => undefined); await sleep(retryDelayMs(response.headers.get('retry-after'), attempt)); } + observeRevision(response.headers, 'x-qveris-api-version', observedApiRevisions); + observeRevision(response.headers, 'x-qveris-catalog-version', observedCatalogRevisions); if (!response.ok) { await Promise.resolve(response.body?.cancel?.()).catch(() => undefined); throw apiError(`HTTP ${response.status}`); @@ -80,9 +84,24 @@ export function createApiClient({ parameters, search_id: discoveryId ?? null, }), + observedRevisions: () => ({ + api_revision: singleRevision(observedApiRevisions), + catalog_revision: singleRevision(observedCatalogRevisions), + }), }; } +function observeRevision(headers, name, revisions) { + const value = headers?.get?.(name); + if (typeof value === 'string' && value.trim()) revisions.add(value.trim()); +} + +function singleRevision(revisions) { + if (revisions.size === 0) return 'unreported'; + if (revisions.size === 1) return [...revisions][0]; + return `mixed:${[...revisions].sort().join(',')}`; +} + function retryDelayMs(retryAfter, attempt) { const seconds = Number(retryAfter); if (retryAfter !== null && Number.isFinite(seconds) && seconds >= 0) { diff --git a/benchmarks/discover-call/src/harness.mjs b/benchmarks/discover-call/src/harness.mjs index 03a5e09..2324e68 100644 --- a/benchmarks/discover-call/src/harness.mjs +++ b/benchmarks/discover-call/src/harness.mjs @@ -41,24 +41,43 @@ export function validateTask(task) { ) { throw new Error(`Task ${task.id}: constraint ${constraint.id} has invalid normalizers`); } + if (constraint.alias_values !== undefined) { + if (!isPlainObject(constraint.alias_values)) { + throw new Error(`Task ${task.id}: constraint ${constraint.id} has invalid alias_values`); + } + for (const [alias, values] of Object.entries(constraint.alias_values)) { + if ( + !constraint.aliases.includes(alias) || + !Array.isArray(values) || + values.length === 0 || + values.some((value) => !['string', 'number', 'boolean'].includes(typeof value)) + ) { + throw new Error(`Task ${task.id}: constraint ${constraint.id} has invalid alias_values`); + } + } + } + } + if (task.reference !== undefined && task.oracle !== undefined) { + throw new Error(`Task ${task.id}: use reference or legacy oracle, not both`); } - if (task.oracle !== undefined) { - if (!task.oracle || typeof task.oracle !== 'object' || !Array.isArray(task.oracle.candidates)) { - throw new Error(`Task ${task.id}: oracle.candidates must be an array`); + const reference = task.reference ?? task.oracle; + if (reference !== undefined) { + if (!reference || typeof reference !== 'object' || !Array.isArray(reference.candidates)) { + throw new Error(`Task ${task.id}: reference.candidates must be an array`); } if ( - task.oracle.candidates.length === 0 && - (typeof task.oracle.unavailable_reason !== 'string' || !task.oracle.unavailable_reason) + reference.candidates.length === 0 && + (typeof reference.unavailable_reason !== 'string' || !reference.unavailable_reason) ) { - throw new Error(`Task ${task.id}: an empty oracle needs unavailable_reason`); + throw new Error(`Task ${task.id}: an empty reference needs unavailable_reason`); } - for (const candidate of task.oracle.candidates) { + for (const candidate of reference.candidates) { if ( typeof candidate?.tool_id !== 'string' || !candidate.tool_id || !isPlainObject(candidate.parameters) ) { - throw new Error(`Task ${task.id}: every oracle candidate needs tool_id and parameters`); + throw new Error(`Task ${task.id}: every reference candidate needs tool_id and parameters`); } } } @@ -160,8 +179,7 @@ async function runTrial({ task, model, trial, execute, limit, api, invokeAdapter parameters: record.parameters, }); record.call.success = response?.success === true; - record.call.execution_id = response?.execution_id ?? null; - record.call.result_valid = response?.success === true ? hasMeaningfulResult(response?.result) : false; + record.call.result_nonempty = response?.success === true ? hasNonemptyResult(response?.result) : false; } record.status = 'completed'; @@ -307,7 +325,7 @@ function omitNullParameters(parameters) { return Object.fromEntries(Object.entries(parameters).filter(([, value]) => value !== null)); } -function hasMeaningfulResult(result) { +function hasNonemptyResult(result) { if (result === null || result === undefined) return false; if (typeof result === 'string') return result.trim().length > 0; if (Array.isArray(result)) return result.length > 0; diff --git a/benchmarks/discover-call/src/publication.mjs b/benchmarks/discover-call/src/publication.mjs new file mode 100644 index 0000000..63c555b --- /dev/null +++ b/benchmarks/discover-call/src/publication.mjs @@ -0,0 +1,180 @@ +import { createHash } from 'node:crypto'; + +export function sanitizePublicRecords(records, policy) { + validatePolicy(policy); + const approvedToolIds = new Set(policy.approved_selected_tool_ids); + const catalogObservationSha256 = + commonMetadataValue(records, 'catalog_observation_sha256') ?? + sha256( + JSON.stringify( + records.map((record) => ({ + task_id: record.task_id, + trial: record.trial, + result_tool_ids: array(record.discovery?.result_tool_ids), + })), + ), + ); + return records.map((record) => + sanitizePublicRecord(record, policy, approvedToolIds, catalogObservationSha256), + ); +} + +export function sanitizePublicRecord( + record, + policy, + approvedToolIds = new Set(policy.approved_selected_tool_ids), + catalogObservationSha256 = record.metadata?.catalog_observation_sha256 ?? 'unreported', +) { + const selectedToolId = record.selection?.tool_id ?? null; + if (selectedToolId && !approvedToolIds.has(selectedToolId)) { + throw new Error(`Selected tool is not approved for public artifacts: ${selectedToolId}`); + } + + const resultToolIds = array(record.discovery?.result_tool_ids); + const returnedToolIds = array(record.inspection?.returned_tool_ids); + const selectionGrounded = selectedToolId ? resultToolIds.includes(selectedToolId) : false; + const inspectionGrounded = selectedToolId ? returnedToolIds.includes(selectedToolId) : false; + const call = { + attempted: record.call?.attempted === true, + success: record.call?.success ?? null, + }; + if (typeof record.call?.result_nonempty === 'boolean') { + call.result_nonempty = record.call.result_nonempty; + } else if (typeof record.call?.result_valid === 'boolean') { + call.result_nonempty = record.call.result_valid; + } + + const sanitized = { + schema_version: record.schema_version, + benchmark_version: record.benchmark_version, + run_id: record.run_id, + model: record.model, + metadata: publicMetadata(record.metadata, policy, catalogObservationSha256), + task_id: record.task_id, + trial: record.trial, + started_at: record.started_at, + status: record.status, + discovery: { + result_count: resultToolIds.length, + snapshot_sha256: sha256(JSON.stringify(resultToolIds)), + selection_grounded: selectionGrounded, + }, + selection: { tool_id: selectedToolId }, + inspection: { + selection_grounded: inspectionGrounded, + required_parameters: array(record.inspection?.required_parameters), + }, + parameters: policy.publish_parameters === true ? record.parameters : null, + call, + ...(record.error ? { error: record.error } : {}), + finished_at: record.finished_at, + source_record_sha256: sha256(JSON.stringify(record)), + }; + assertNoForbiddenFields(sanitized, new Set(policy.forbidden_fields)); + return sanitized; +} + +export function validatePolicy(policy) { + if (!policy || typeof policy !== 'object' || policy.schema_version !== 1) { + throw new Error('Publication policy schema_version must be 1'); + } + if (typeof policy.policy_id !== 'string' || !policy.policy_id) { + throw new Error('Publication policy needs a policy_id'); + } + if (policy.catalog_visibility_default !== 'private') { + throw new Error('Public artifact policy must default catalog visibility to private'); + } + if ( + !Array.isArray(policy.approved_selected_tool_ids) || + policy.approved_selected_tool_ids.some((toolId) => typeof toolId !== 'string' || !toolId) + ) { + throw new Error('Publication policy needs approved_selected_tool_ids'); + } + if (!Array.isArray(policy.forbidden_fields) || !policy.forbidden_fields.includes('execution_id')) { + throw new Error('Publication policy must forbid execution_id'); + } + if ( + policy.legacy_lane_rewrites !== undefined && + (!plainObject(policy.legacy_lane_rewrites) || + Object.values(policy.legacy_lane_rewrites).some((lane) => typeof lane !== 'string' || !lane)) + ) { + throw new Error('Publication policy legacy_lane_rewrites must map strings to strings'); + } +} + +export function validatePublicRecords(records, policy) { + validatePolicy(policy); + const approvedToolIds = new Set(policy.approved_selected_tool_ids); + const forbidden = new Set(policy.forbidden_fields); + for (const record of records) { + assertNoForbiddenFields(record, forbidden); + if (record.metadata?.publication_policy !== policy.policy_id) { + throw new Error(`Public artifact is missing publication policy ${policy.policy_id}`); + } + const selectedToolId = record.selection?.tool_id; + if (selectedToolId && !approvedToolIds.has(selectedToolId)) { + throw new Error(`Public artifact contains an unapproved selected tool: ${selectedToolId}`); + } + if ( + !Number.isInteger(record.discovery?.result_count) || + typeof record.discovery?.snapshot_sha256 !== 'string' || + record.discovery.snapshot_sha256.length !== 64 + ) { + throw new Error('Public artifact discovery needs a count and SHA-256 snapshot'); + } + if (typeof record.source_record_sha256 !== 'string' || record.source_record_sha256.length !== 64) { + throw new Error('Public artifact needs source_record_sha256'); + } + } +} + +function publicMetadata(metadata = {}, policy, catalogObservationSha256) { + const lane = policy.legacy_lane_rewrites?.[metadata.lane] ?? metadata.lane ?? 'model'; + return { + ...metadata, + lane, + model_revision: metadata.model_revision ?? 'unreported', + api_revision: metadata.api_revision ?? 'unreported', + catalog_revision: metadata.catalog_revision ?? 'unreported', + catalog_observation_sha256: catalogObservationSha256, + runtime: { + node: metadata.runtime?.node ?? 'unreported', + platform: metadata.runtime?.platform ?? 'unreported', + arch: metadata.runtime?.arch ?? 'unreported', + }, + publication_policy: policy.policy_id, + }; +} + +function commonMetadataValue(records, field) { + const values = new Set( + records + .map((record) => record.metadata?.[field]) + .filter((value) => typeof value === 'string' && value && value !== 'pending'), + ); + return values.size === 1 ? [...values][0] : null; +} + +function assertNoForbiddenFields(value, forbidden, path = '$') { + if (Array.isArray(value)) { + value.forEach((item, index) => assertNoForbiddenFields(item, forbidden, `${path}[${index}]`)); + return; + } + if (!value || typeof value !== 'object') return; + for (const [key, child] of Object.entries(value)) { + if (forbidden.has(key)) throw new Error(`Forbidden public artifact field at ${path}.${key}`); + assertNoForbiddenFields(child, forbidden, `${path}.${key}`); + } +} + +function sha256(value) { + return createHash('sha256').update(value).digest('hex'); +} + +function array(value) { + return Array.isArray(value) ? value : []; +} + +function plainObject(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value); +} diff --git a/benchmarks/discover-call/src/publish.mjs b/benchmarks/discover-call/src/publish.mjs new file mode 100644 index 0000000..6a5a90c --- /dev/null +++ b/benchmarks/discover-call/src/publish.mjs @@ -0,0 +1,62 @@ +#!/usr/bin/env node + +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { readJsonLines, writeJsonLines } from './io.mjs'; +import { sanitizePublicRecords } from './publication.mjs'; +import { scoreRecords } from './scoring.mjs'; + +const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..'); + +async function main(argv = process.argv.slice(2)) { + const options = parseArgs(argv); + const [tasks, records, policy] = await Promise.all([ + readJsonLines(resolve(options.tasks)), + readJsonLines(resolve(options.runs)), + readFile(resolve(options.policy), 'utf8').then(JSON.parse), + ]); + const publicRecords = sanitizePublicRecords(records, policy); + const summary = scoreRecords(tasks, publicRecords); + const outputRuns = resolve(options.outputRuns); + const outputSummary = resolve(options.outputSummary); + await Promise.all([mkdir(dirname(outputRuns), { recursive: true }), mkdir(dirname(outputSummary), { recursive: true })]); + await writeJsonLines(outputRuns, publicRecords); + await writeFile(outputSummary, JSON.stringify(summary, null, 2) + '\n', { encoding: 'utf8', mode: 0o600 }); + process.stdout.write(`Wrote ${publicRecords.length} sanitized records and summary\n`); +} + +function parseArgs(argv) { + const options = { + tasks: resolve(ROOT, 'tasks/v1.jsonl'), + policy: resolve(ROOT, 'publication-policy.json'), + }; + for (let index = 0; index < argv.length; index++) { + const arg = argv[index]; + if (arg === '--tasks') options.tasks = value(argv, ++index, arg); + else if (arg === '--runs') options.runs = value(argv, ++index, arg); + else if (arg === '--policy') options.policy = value(argv, ++index, arg); + else if (arg === '--output-runs') options.outputRuns = value(argv, ++index, arg); + else if (arg === '--output-summary') options.outputSummary = value(argv, ++index, arg); + else throw new Error(`Unknown argument: ${arg}`); + } + for (const name of ['runs', 'outputRuns', 'outputSummary']) { + if (!options[name]) throw new Error(`--${camelToKebab(name)} is required`); + } + return options; +} + +function value(argv, index, flag) { + if (!argv[index]) throw new Error(`${flag} requires a value`); + return argv[index]; +} + +function camelToKebab(value) { + return value.replace(/[A-Z]/g, (character) => `-${character.toLowerCase()}`); +} + +main().catch((error) => { + process.stderr.write(`${error.message}\n`); + process.exitCode = 1; +}); diff --git a/benchmarks/discover-call/src/run.mjs b/benchmarks/discover-call/src/run.mjs index f6b2039..fc4136a 100644 --- a/benchmarks/discover-call/src/run.mjs +++ b/benchmarks/discover-call/src/run.mjs @@ -33,6 +33,24 @@ export async function main(argv = process.argv.slice(2)) { }); const output = resolve(options.output); await mkdir(dirname(output), { recursive: true }); + const metadata = { + lane: options.lane, + model_revision: options.modelRevision, + adapter_revision: options.adapterRevision, + toolkit_revision: toolkitRevision, + task_set_sha256: taskSetSha256, + api_base_url: api.baseUrl, + api_revision: 'pending', + catalog_revision: 'pending', + catalog_observation_sha256: 'pending', + runtime: { + node: process.version, + platform: process.platform, + arch: process.arch, + }, + discovery_limit: options.limit, + execute: options.execute, + }; const records = await runBenchmark({ tasks, model: options.model, @@ -41,16 +59,13 @@ export async function main(argv = process.argv.slice(2)) { limit: options.limit, api, invokeAdapter, - metadata: { - lane: options.lane, - adapter_revision: options.adapterRevision, - toolkit_revision: toolkitRevision, - task_set_sha256: taskSetSha256, - api_base_url: api.baseUrl, - discovery_limit: options.limit, - execute: options.execute, - }, + metadata, }); + Object.assign( + metadata, + api.observedRevisions?.() ?? { api_revision: 'unreported', catalog_revision: 'unreported' }, + ); + metadata.catalog_observation_sha256 = catalogObservationSha256(records); await writeJsonLines(output, records); process.stdout.write(`Wrote ${records.length} benchmark records to ${output}\n`); } @@ -121,6 +136,7 @@ function parseArgs(argv) { execute: false, limit: 10, lane: 'model', + modelRevision: 'unreported', adapterArgs: [], adapterTimeoutMs: 120_000, }; @@ -133,6 +149,7 @@ function parseArgs(argv) { else if (arg === '--adapter-revision') options.adapterRevision = nextValue(argv, ++index, arg); else if (arg === '--toolkit-revision') options.toolkitRevision = nextValue(argv, ++index, arg); else if (arg === '--model') options.model = nextValue(argv, ++index, arg); + else if (arg === '--model-revision') options.modelRevision = nextValue(argv, ++index, arg); else if (arg === '--tasks') options.tasks = nextValue(argv, ++index, arg); else if (arg === '--output') options.output = nextValue(argv, ++index, arg); else if (arg === '--trials') options.trials = integer(nextValue(argv, ++index, arg), arg); @@ -144,6 +161,9 @@ function parseArgs(argv) { if (!options.help && !options.model) throw new Error('--model is required'); if (!options.help && !options.adapter) throw new Error('--adapter is required'); if (!options.help && !options.adapterRevision) throw new Error('--adapter-revision is required'); + if (!options.help && options.lane === 'pinned-model' && options.modelRevision === 'unreported') { + throw new Error('--model-revision is required for --lane pinned-model'); + } return options; } @@ -160,12 +180,23 @@ function integer(value, flag) { } function lane(value) { - if (!['model', 'oracle', 'pinned-model', 'current-model'].includes(value)) { - throw new Error('--lane must be model, oracle, pinned-model, or current-model'); + if (!['model', 'oracle', 'reference', 'configured-model', 'pinned-model', 'current-model'].includes(value)) { + throw new Error( + '--lane must be model, oracle, reference, configured-model, pinned-model, or current-model', + ); } return value; } +function catalogObservationSha256(records) { + const observation = records.map((record) => ({ + task_id: record.task_id, + trial: record.trial, + result_tool_ids: record.discovery?.result_tool_ids ?? [], + })); + return createHash('sha256').update(JSON.stringify(observation)).digest('hex'); +} + function adapterError(message, reason) { const error = new Error(message); error.benchmarkStage = 'adapter'; @@ -175,7 +206,7 @@ function adapterError(message, reason) { function adapterReasonFromStderr(stderr) { const match = stderr.match( - /QVERIS_BENCHMARK_ADAPTER_ERROR=(tool_use_rejected|model_failed|invalid_events|invalid_output|cli_failed|start_failed|output_limit|missing_oracle|missing_oracle_candidate|unsupported_stage)/, + /QVERIS_BENCHMARK_ADAPTER_ERROR=(tool_use_rejected|model_failed|invalid_events|invalid_output|cli_failed|start_failed|output_limit|missing_reference|missing_reference_candidate|missing_oracle|missing_oracle_candidate|unsupported_stage)/, ); return match?.[1] ?? null; } @@ -191,7 +222,7 @@ function resolveToolkitRevision(explicit) { } function helpText() { - return `QVeris discover→call benchmark\n\nUsage:\n node src/run.mjs --model MODEL --adapter COMMAND --adapter-revision REVISION [options]\n\nOptions:\n --adapter-arg VALUE Repeatable adapter argument (no shell parsing)\n --adapter-revision VALUE Immutable adapter source/config revision (required)\n --toolkit-revision VALUE Toolkit revision (auto-detected in git/GitHub Actions)\n --tasks PATH Task-set JSONL (default: tasks/v1.jsonl)\n --output PATH Run-record JSONL (default: benchmark-runs.jsonl)\n --trials N Trials per task (default: 3)\n --limit N Discover result limit (default: 10)\n --lane VALUE model, oracle, pinned-model, or current-model\n --execute Perform billed call requests (required for workflow success)\n --adapter-timeout-ms N Per-stage adapter timeout (default: 120000)\n`; + return `QVeris discover→call benchmark\n\nUsage:\n node src/run.mjs --model MODEL --adapter COMMAND --adapter-revision REVISION [options]\n\nOptions:\n --adapter-arg VALUE Repeatable adapter argument (no shell parsing)\n --adapter-revision VALUE Immutable adapter source/config revision (required)\n --toolkit-revision VALUE Toolkit revision (auto-detected in git/GitHub Actions)\n --model-revision VALUE Provider snapshot/backend revision; required for pinned-model\n --tasks PATH Task-set JSONL (default: tasks/v1.jsonl)\n --output PATH Run-record JSONL (default: benchmark-runs.jsonl)\n --trials N Trials per task (default: 3)\n --limit N Discover result limit (default: 10)\n --lane VALUE reference, configured-model, pinned-model, current-model, or legacy value\n --execute Perform billed call requests (required for workflow success)\n --adapter-timeout-ms N Per-stage adapter timeout (default: 120000)\n`; } if (process.argv[1] === fileURLToPath(import.meta.url)) { diff --git a/benchmarks/discover-call/src/scoring.mjs b/benchmarks/discover-call/src/scoring.mjs index cda224e..4c4843c 100644 --- a/benchmarks/discover-call/src/scoring.mjs +++ b/benchmarks/discover-call/src/scoring.mjs @@ -54,10 +54,14 @@ function validateCompleteModelRun(model, tasks, records) { // it with runner metadata is rejected. const comparableMetadata = [ 'lane', + 'model_revision', 'adapter_revision', 'toolkit_revision', 'task_set_sha256', 'api_base_url', + 'api_revision', + 'catalog_revision', + 'catalog_observation_sha256', 'discovery_limit', 'execute', ]; @@ -80,14 +84,29 @@ export function scoreRecord(task, record) { const required = array(record.inspection?.required_parameters); const parameters = plainObject(record.parameters) ? record.parameters : {}; const completed = record.status === 'completed'; - const selectionGrounded = selected ? discovered.includes(selected) : false; - const inspectionGrounded = selected ? inspected.includes(selected) : false; + const selectionGrounded = + typeof record.discovery?.selection_grounded === 'boolean' + ? record.discovery.selection_grounded + : selected + ? discovered.includes(selected) + : false; + const inspectionGrounded = + typeof record.inspection?.selection_grounded === 'boolean' + ? record.inspection.selection_grounded + : selected + ? inspected.includes(selected) + : false; const requiredParameterAccuracy = required.length === 0 ? 1 : mean(required.map((name) => hasValue(parameters[name]))); const constraintAccuracy = mean(task.constraints.map((constraint) => constraintSatisfied(parameters, constraint))); const executed = record.call?.attempted === true; const callSuccess = executed ? record.call?.success === true : null; - const resultValid = executed && callSuccess ? record.call?.result_valid !== false : executed ? false : null; + const resultNonempty = + executed && callSuccess + ? record.call?.result_nonempty ?? record.call?.result_valid ?? true + : executed + ? false + : null; const workflowSuccess = completed && selectionGrounded && @@ -95,7 +114,7 @@ export function scoreRecord(task, record) { requiredParameterAccuracy === 1 && constraintAccuracy === 1 && callSuccess === true && - resultValid === true; + resultNonempty === true; return { run_id: record.run_id, @@ -109,7 +128,7 @@ export function scoreRecord(task, record) { required_parameter_accuracy: round(requiredParameterAccuracy), constraint_accuracy: round(constraintAccuracy), call_success: callSuccess, - result_valid: resultValid, + result_nonempty: resultNonempty, workflow_success: workflowSuccess, error_stage: record.error?.stage ?? null, error_reason: record.error?.reason_code ?? null, @@ -120,7 +139,7 @@ export function scoreRecord(task, record) { function aggregateModel(model, records) { const executed = records.filter((record) => record.executed); const workflowWins = records.filter((record) => record.workflow_success).length; - const interval = wilsonInterval(workflowWins, records.length); + const interval = taskClusterBootstrapInterval(records); const taskCount = new Set(records.map((record) => record.task_id)).size; const failuresByStage = {}; const failuresByReason = {}; @@ -143,9 +162,9 @@ function aggregateModel(model, records) { required_parameter_accuracy: round(mean(records.map((record) => record.required_parameter_accuracy))), constraint_accuracy: round(mean(records.map((record) => record.constraint_accuracy))), call_success_rate: executed.length ? round(mean(executed.map((record) => record.call_success))) : null, - result_valid_rate: executed.length ? round(mean(executed.map((record) => record.result_valid))) : null, + result_nonempty_rate: executed.length ? round(mean(executed.map((record) => record.result_nonempty))) : null, workflow_success_rate: round(workflowWins / records.length), - workflow_success_wilson_95: interval.map(round), + workflow_success_task_cluster_bootstrap_95: interval.map(round), failures_by_stage: failuresByStage, failures_by_reason: failuresByReason, }; @@ -155,11 +174,13 @@ function constraintSatisfied(parameters, constraint) { for (const alias of constraint.aliases) { if (!hasValue(parameters[alias])) continue; const actual = parameters[alias]; - const expected = constraint.value; - if (constraint.match === 'contains') { - if (normalize(actual, constraint).includes(normalize(expected, constraint))) return 1; - } else if (normalize(actual, constraint) === normalize(expected, constraint)) { - return 1; + const expectedValues = [constraint.value, ...array(constraint.alias_values?.[alias])]; + for (const expected of expectedValues) { + if (constraint.match === 'contains') { + if (normalize(actual, constraint).includes(normalize(expected, constraint))) return 1; + } else if (normalize(actual, constraint) === normalize(expected, constraint)) { + return 1; + } } } for (const alias of array(constraint.composite_aliases)) { @@ -204,13 +225,36 @@ function mean(values) { return values.reduce((sum, value) => sum + Number(value), 0) / values.length; } -function wilsonInterval(successes, total, z = 1.96) { - if (total === 0) return [0, 0]; - const proportion = successes / total; - const denominator = 1 + (z * z) / total; - const center = (proportion + (z * z) / (2 * total)) / denominator; - const margin = (z / denominator) * Math.sqrt((proportion * (1 - proportion)) / total + (z * z) / (4 * total * total)); - return [Math.max(0, center - margin), Math.min(1, center + margin)]; +function taskClusterBootstrapInterval(records, iterations = 20_000) { + const byTask = new Map(); + for (const record of records) { + const values = byTask.get(record.task_id) || []; + values.push(Number(record.workflow_success)); + byTask.set(record.task_id, values); + } + const taskRates = [...byTask.values()].map(mean); + if (taskRates.length <= 1) { + const value = taskRates[0] ?? 0; + return [value, value]; + } + + let state = 0x51f15e; + const randomIndex = () => { + state = (Math.imul(state, 1_664_525) + 1_013_904_223) >>> 0; + return Math.floor((state / 2 ** 32) * taskRates.length); + }; + const estimates = []; + for (let iteration = 0; iteration < iterations; iteration++) { + let total = 0; + for (let task = 0; task < taskRates.length; task++) total += taskRates[randomIndex()]; + estimates.push(total / taskRates.length); + } + estimates.sort((left, right) => left - right); + return [percentile(estimates, 0.025), percentile(estimates, 0.975)]; +} + +function percentile(sorted, probability) { + return sorted[Math.min(sorted.length - 1, Math.max(0, Math.floor(probability * sorted.length)))]; } function round(value) { diff --git a/benchmarks/discover-call/src/validate.mjs b/benchmarks/discover-call/src/validate.mjs index a7d7df4..5144b6c 100644 --- a/benchmarks/discover-call/src/validate.mjs +++ b/benchmarks/discover-call/src/validate.mjs @@ -1,9 +1,10 @@ -import { readdir } from 'node:fs/promises'; +import { readFile, readdir } from 'node:fs/promises'; import { resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { validateTask } from './harness.mjs'; import { readJsonLines } from './io.mjs'; +import { validatePolicy, validatePublicRecords } from './publication.mjs'; import { scoreRecords } from './scoring.mjs'; const root = resolve(fileURLToPath(new URL('..', import.meta.url))); @@ -30,4 +31,21 @@ for (const file of taskFiles) { const v1 = await readJsonLines(resolve(root, 'tasks/v1.jsonl')); const fixture = await readJsonLines(resolve(root, 'fixtures/sample-runs.jsonl')); scoreRecords(v1, fixture); -process.stdout.write(`Validated ${counts.join(', ')} and ${fixture.length} fixture records\n`); + +const policy = JSON.parse(await readFile(resolve(root, 'publication-policy.json'), 'utf8')); +validatePolicy(policy); +const resultFiles = (await readdir(resolve(root, 'results'))) + .filter((name) => /\.runs\.jsonl$/.test(name)) + .sort(); +for (const file of resultFiles) { + const version = file.match(/-v(\d+)\.runs\.jsonl$/)?.[1]; + if (!version) throw new Error(`${file}: result filename must identify its task-set version`); + const tasks = await readJsonLines(resolve(root, `tasks/v${version}.jsonl`)); + const records = await readJsonLines(resolve(root, 'results', file)); + validatePublicRecords(records, policy); + scoreRecords(tasks, records); +} + +process.stdout.write( + `Validated ${counts.join(', ')}, ${fixture.length} fixture records, and ${resultFiles.length} public result files\n`, +); diff --git a/benchmarks/discover-call/tasks/v4.jsonl b/benchmarks/discover-call/tasks/v4.jsonl new file mode 100644 index 0000000..ce5ed1b --- /dev/null +++ b/benchmarks/discover-call/tasks/v4.jsonl @@ -0,0 +1,18 @@ +{"id":"weather-london","prompt":"Find a weather forecast capability and request the forecast for London.","discover_query":"weather forecast API","constraints":[{"id":"location","aliases":["city","location","place","query"],"value":"London"}],"reference":{"candidates":[{"tool_id":"visualcrossing.timeline.retrieve.v1","parameters":{"location":"London","unitGroup":"metric","contentType":"json"}}]}} +{"id":"stock-aapl","prompt":"Find a market-data capability and request the latest price for AAPL.","discover_query":"stock quote market data API","constraints":[{"id":"symbol","aliases":["symbol","ticker","code"],"value":"AAPL"}],"reference":{"candidates":[{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3","parameters":{"symbol":"AAPL","format":"JSON"}}]}} +{"id":"exchange-usd-eur","prompt":"Find a foreign-exchange capability and request a USD to EUR conversion.","discover_query":"currency exchange conversion API","constraints":[{"id":"base_currency","aliases":["base","base_currency","from","from_currency"],"composite_aliases":["symbol","pair"],"value":"USD"},{"id":"quote_currency","aliases":["quote","quote_currency","to","to_currency","target_currency"],"composite_aliases":["symbol","pair"],"value":"EUR"}],"reference":{"candidates":[{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4","parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"}}]}} +{"id":"news-ai","prompt":"Find a news-search capability and search for recent artificial intelligence news.","discover_query":"recent news search API","constraints":[{"id":"query","aliases":["query","q","keyword","keywords","topic"],"value":"artificial intelligence","match":"contains","normalizers":["url_decode"]}],"reference":{"candidates":[{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df","parameters":{"q":"artificial intelligence","language":"en","sortBy":"publishedAt","pageSize":20,"page":1}}]}} +{"id":"papers-climate","prompt":"Find a scholarly-search capability and search for papers about climate risk.","discover_query":"academic paper search API","constraints":[{"id":"query","aliases":["query","q","keyword","keywords","topic"],"value":"climate risk","match":"contains"}],"reference":{"candidates":[{"tool_id":"qveris_academic.paper_aggregation_v1","parameters":{"query":"climate risk","limit":10}}]}} +{"id":"timezone-tokyo","prompt":"Find a timezone capability and request timezone information for Tokyo.","discover_query":"timezone lookup API","constraints":[{"id":"location","aliases":["city","location","place","query"],"value":"Tokyo"}],"reference":{"candidates":[],"unavailable_reason":"The fixed discovery query currently returns no top-10 capability that accepts a city or coordinates for a Tokyo timezone lookup."}} +{"id":"geocode-address","prompt":"Find a geocoding capability and convert the street address '北京市海淀区上地十街10号' to coordinates.","discover_query":"geocoding convert address to coordinates API","constraints":[{"id":"address","aliases":["address","q","query","location"],"value":"上地十街","match":"contains"}],"reference":{"candidates":[{"tool_id":"amap_webservice.geocode.geo.retrieve.v3","parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"}}]}} +{"id":"ip-geolocation","prompt":"Find an IP geolocation capability and look up details for the IP address 8.8.8.8.","discover_query":"IP address geolocation lookup API","constraints":[{"id":"ip","aliases":["ip","q","query","ip_address","address"],"value":"8.8.8.8"}],"reference":{"candidates":[{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1","parameters":{"ip":"8.8.8.8"}},{"tool_id":"virustotal.ip.report.retrieve.v3.ab34b0d1","parameters":{"ip":"8.8.8.8"}}]}} +{"id":"crypto-price","prompt":"Find a cryptocurrency market-data capability and request the latest price for Bitcoin (BTC).","discover_query":"cryptocurrency price API","constraints":[{"id":"symbol","aliases":["symbol","symbols","slug","id","pair","base_currency"],"alias_values":{"id":["1"]},"value":"BTC","match":"contains"}],"reference":{"candidates":[{"tool_id":"binance.ticker.price.list.v3.8675eca0","parameters":{"symbol":"BTCUSDT"}}]}} +{"id":"web-search","prompt":"Find a web search capability and search the web for 'model context protocol'.","discover_query":"web search API","constraints":[{"id":"query","aliases":["q","query","keyword","keywords","text"],"value":"model context protocol","match":"contains","normalizers":["url_decode"]}],"reference":{"candidates":[{"tool_id":"cloudsway.smart_search_overseas","parameters":{"q":"model context protocol","count":"10"}}]}} +{"id":"air-quality","prompt":"Find an air quality capability and request the current air pollution for latitude 51.5, longitude -0.12.","discover_query":"air quality index API","constraints":[{"id":"latitude","aliases":["lat","latitude"],"value":51.5},{"id":"longitude","aliases":["lon","lng","longitude"],"value":-0.12}],"reference":{"candidates":[{"tool_id":"openweathermap.air_pollution.current.v2_5","parameters":{"lat":51.5,"lon":-0.12}}]}} +{"id":"market-news","prompt":"Find a financial news capability and request general market news.","discover_query":"stock market news API","constraints":[{"id":"category","aliases":["category","topic","q","query"],"value":"general","match":"contains"}],"reference":{"candidates":[{"tool_id":"finnhub.market.news.list.v1","parameters":{"category":"general"}}]}} +{"id":"domain-intel","prompt":"Find a domain intelligence capability and look up the domain example.com.","discover_query":"domain whois lookup API","constraints":[{"id":"domain","aliases":["domain","domain_name","q","query","target","host"],"value":"example.com","match":"contains"}],"reference":{"candidates":[{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740","parameters":{"domain":"example.com"}},{"tool_id":"virustotal.domain_reports.retrieve.v3.c5459740","parameters":{"domain":"example.com"}}]}} +{"id":"image-search","prompt":"Find an image search capability and search for images of the Golden Gate Bridge.","discover_query":"image search API","constraints":[{"id":"query","aliases":["q","query","keyword","keywords"],"value":"golden gate","match":"contains","normalizers":["url_decode"]}],"reference":{"candidates":[{"tool_id":"brave_search.images.search.list.v1","parameters":{"q":"Golden Gate Bridge"}}]}} +{"id":"market-holidays","prompt":"Find a market calendar capability and request stock market holidays for the exchange code US.","discover_query":"stock market holiday calendar API","constraints":[{"id":"exchange","aliases":["exchange","market","exchange_code"],"value":"US"}],"reference":{"candidates":[{"tool_id":"finnhub.stock.marketholiday.retrieve.v1","parameters":{"exchange":"US"}}]}} +{"id":"earthquake-catalog","prompt":"Find an earthquake catalog capability and query recent events in geojson format.","discover_query":"recent earthquakes data API","constraints":[{"id":"format","aliases":["format","output","outputformat"],"value":"geojson"}],"reference":{"candidates":[{"tool_id":"usgs_earthquake_catalog.event.events.list.v1","parameters":{"format":"geojson"}}]}} +{"id":"company-profile","prompt":"Find a company fundamentals capability and look up the company profile for CIK 0000320193.","discover_query":"company information lookup API","constraints":[{"id":"cik","aliases":["cik","symbol","company","q","query"],"value":"320193","match":"contains"}],"reference":{"candidates":[{"tool_id":"mcp_gildata.uscompanybasicinfo.v1","parameters":{"query":"0000320193"}},{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2","parameters":{"cik":"0000320193"}}]}} +{"id":"sports-standings","prompt":"Find a sports data capability and request league standings for league 39 in season 2025.","discover_query":"sports league standings API","constraints":[{"id":"league","aliases":["league","league_id"],"value":39},{"id":"season","aliases":["season","year"],"value":2025}],"reference":{"candidates":[{"tool_id":"api_sports.standings.list.v1.39568ba4","parameters":{"league":39,"season":"2025"}}]}} diff --git a/benchmarks/discover-call/test/adapter.test.mjs b/benchmarks/discover-call/test/adapter.test.mjs index 9e50de0..596c8da 100644 --- a/benchmarks/discover-call/test/adapter.test.mjs +++ b/benchmarks/discover-call/test/adapter.test.mjs @@ -12,8 +12,8 @@ import { } from '../adapters/codex-cli.mjs'; const adapterPath = resolve(fileURLToPath(new URL('../adapters/first-result.mjs', import.meta.url))); -const oracleAdapterPath = resolve(fileURLToPath(new URL('../adapters/oracle.mjs', import.meta.url))); -const v3TasksPath = resolve(fileURLToPath(new URL('../tasks/v3.jsonl', import.meta.url))); +const referenceAdapterPath = resolve(fileURLToPath(new URL('../adapters/reference.mjs', import.meta.url))); +const v4TasksPath = resolve(fileURLToPath(new URL('../tasks/v4.jsonl', import.meta.url))); test('process adapter exchanges one JSON object without shell parsing', async () => { const invoke = createProcessAdapter({ command: process.execPath, args: [adapterPath], timeoutMs: 5_000 }); @@ -92,10 +92,10 @@ test('process adapter classifies timeouts', async () => { }); }); -test('Oracle adapter selects only configured candidates present in discovery', async () => { +test('reference adapter selects only configured candidates present in discovery', async () => { const invoke = createProcessAdapter({ command: process.execPath, - args: [oracleAdapterPath, v3TasksPath], + args: [referenceAdapterPath, v4TasksPath], timeoutMs: 5_000, }); assert.deepEqual( diff --git a/benchmarks/discover-call/test/api.test.mjs b/benchmarks/discover-call/test/api.test.mjs index 1c8284a..3728e48 100644 --- a/benchmarks/discover-call/test/api.test.mjs +++ b/benchmarks/discover-call/test/api.test.mjs @@ -89,3 +89,19 @@ test('cancels unsuccessful response bodies before reporting API errors', async ( await assert.rejects(client.discover({ query: 'weather', limit: 5 }), /HTTP 400/); assert.equal(cancelled, true); }); + +test('records API revisions without inventing an absent catalog revision', async () => { + const client = createApiClient({ + apiKey: 'test-key', + fetchImpl: async () => + new Response(JSON.stringify({ results: [] }), { + headers: { 'x-qveris-api-version': '2026-07-22.1' }, + }), + }); + + await client.discover({ query: 'weather', limit: 5 }); + assert.deepEqual(client.observedRevisions(), { + api_revision: '2026-07-22.1', + catalog_revision: 'unreported', + }); +}); diff --git a/benchmarks/discover-call/test/harness.test.mjs b/benchmarks/discover-call/test/harness.test.mjs index 450512c..8f790b4 100644 --- a/benchmarks/discover-call/test/harness.test.mjs +++ b/benchmarks/discover-call/test/harness.test.mjs @@ -64,8 +64,8 @@ test('orchestrates discover, select, inspect, parameterize, and call', async () assert.deepEqual(records[0].inspection.required_parameters, ['city']); assert.deepEqual(records[0].parameters, { city: 'London' }); assert.equal(records[0].call.success, true); - assert.equal(records[0].call.result_valid, true); - assert.equal(records[0].call.execution_id, 'exec-1'); + assert.equal(records[0].call.result_nonempty, true); + assert.equal('execution_id' in records[0].call, false); assert.equal(records[0].metadata.adapter_revision, 'adapter-sha'); }); @@ -160,6 +160,23 @@ test('validates task constraints and trial bounds', async () => { /normalizers/, ); assert.throws(() => validateTask({ ...task, oracle: { candidates: [] } }), /unavailable_reason/); + assert.doesNotThrow(() => + validateTask({ + ...task, + constraints: [{ ...task.constraints[0], alias_values: { city: ['LON'] } }], + reference: { + candidates: [{ tool_id: 'weather.forecast', parameters: { city: 'LON' } }], + }, + }), + ); + assert.throws( + () => + validateTask({ + ...task, + constraints: [{ ...task.constraints[0], alias_values: { unknown: ['LON'] } }], + }), + /alias_values/, + ); await assert.rejects( runBenchmark({ tasks: [task], model: 'model-a', trials: 0, api: {}, invokeAdapter() {} }), /trials/, diff --git a/benchmarks/discover-call/test/publication.test.mjs b/benchmarks/discover-call/test/publication.test.mjs new file mode 100644 index 0000000..9969aa6 --- /dev/null +++ b/benchmarks/discover-call/test/publication.test.mjs @@ -0,0 +1,82 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { sanitizePublicRecords, validatePublicRecords } from '../src/publication.mjs'; + +const policy = { + schema_version: 1, + policy_id: 'test-public-v1', + catalog_visibility_default: 'private', + publish_parameters: true, + legacy_lane_rewrites: { 'pinned-model': 'configured-model' }, + approved_selected_tool_ids: ['weather.tool'], + forbidden_fields: [ + 'execution_id', + 'search_id', + 'connection_id', + 'remaining_credits', + 'result_tool_ids', + 'returned_tool_ids', + ], +}; + +test('publishes grounded attestations and hashes instead of operational identifiers or catalog rows', () => { + const [record] = sanitizePublicRecords( + [ + { + schema_version: 1, + benchmark_version: 'v4', + run_id: 'run-1', + model: 'model-a', + metadata: { lane: 'configured-model', api_revision: '2026-07-22.1' }, + task_id: 'weather', + trial: 1, + status: 'completed', + discovery: { + search_id: 'search-1', + result_tool_ids: ['other.tool', 'weather.tool'], + }, + selection: { tool_id: 'weather.tool' }, + inspection: { returned_tool_ids: ['weather.tool'], required_parameters: ['city'] }, + parameters: { city: 'Shanghai' }, + call: { + attempted: true, + success: true, + execution_id: 'execution-1', + result_nonempty: true, + }, + }, + ], + policy, + ); + + assert.equal(record.discovery.result_count, 2); + assert.equal(record.discovery.snapshot_sha256.length, 64); + assert.equal(record.discovery.selection_grounded, true); + assert.equal(record.inspection.selection_grounded, true); + assert.equal(record.call.result_nonempty, true); + assert.equal(record.metadata.publication_policy, policy.policy_id); + assert.equal(record.metadata.model_revision, 'unreported'); + assert.equal(record.metadata.catalog_revision, 'unreported'); + assert.equal(record.source_record_sha256.length, 64); + for (const forbidden of policy.forbidden_fields) { + assert.equal(JSON.stringify(record).includes(`"${forbidden}"`), false); + } + assert.doesNotThrow(() => validatePublicRecords([record], policy)); +}); + +test('refuses to publish an unapproved selected tool', () => { + assert.throws( + () => + sanitizePublicRecords( + [ + { + selection: { tool_id: 'private.tool' }, + discovery: { result_tool_ids: ['private.tool'] }, + }, + ], + policy, + ), + /not approved/, + ); +}); diff --git a/benchmarks/discover-call/test/scoring.test.mjs b/benchmarks/discover-call/test/scoring.test.mjs index 3895925..8d064ed 100644 --- a/benchmarks/discover-call/test/scoring.test.mjs +++ b/benchmarks/discover-call/test/scoring.test.mjs @@ -79,7 +79,7 @@ test('v3 constraints support composite parameters and explicit URL decoding', () discovery: { result_tool_ids: ['tool'] }, selection: { tool_id: 'tool' }, inspection: { returned_tool_ids: ['tool'], required_parameters: [] }, - call: { attempted: true, success: true, result_valid: true }, + call: { attempted: true, success: true, result_nonempty: true }, }; assert.equal( @@ -98,7 +98,7 @@ test('v3 constraints support composite parameters and explicit URL decoding', () ); }); -test('requires a meaningful result when result validity is recorded', () => { +test('requires a non-empty result when result non-emptiness is recorded', () => { const record = { run_id: 'run-empty', task_id: task.id, @@ -109,11 +109,11 @@ test('requires a meaningful result when result validity is recorded', () => { selection: { tool_id: 'weather.forecast' }, inspection: { returned_tool_ids: ['weather.forecast'], required_parameters: ['city'] }, parameters: { city: 'London' }, - call: { attempted: true, success: true, result_valid: false }, + call: { attempted: true, success: true, result_nonempty: false }, }; const result = scoreRecord(task, record); assert.equal(result.call_success, true); - assert.equal(result.result_valid, false); + assert.equal(result.result_nonempty, false); assert.equal(result.workflow_success, false); }); @@ -173,7 +173,39 @@ test('aggregates per model with parameter and workflow rates', () => { failureReasons: {}, }, ); - assert.equal(summary.models[0].workflow_success_wilson_95.length, 2); + assert.equal(summary.models[0].workflow_success_task_cluster_bootstrap_95.length, 2); +}); + +test('scores aliases with task-versioned accepted values', () => { + const btcTask = { + id: 'crypto-price', + prompt: 'BTC price', + constraints: [ + { + id: 'symbol', + aliases: ['symbol', 'id'], + value: 'BTC', + alias_values: { id: ['1'] }, + }, + ], + }; + const result = scoreRecord(btcTask, { + run_id: 'run-btc', + task_id: btcTask.id, + model: 'model-a', + trial: 1, + status: 'completed', + discovery: { selection_grounded: true }, + selection: { tool_id: 'crypto.tool' }, + inspection: { selection_grounded: true, required_parameters: ['id'] }, + parameters: { id: 1 }, + call: { attempted: true, success: true, result_nonempty: true }, + }); + + assert.equal(result.selection_grounded, true); + assert.equal(result.inspection_grounded, true); + assert.equal(result.constraint_accuracy, 1); + assert.equal(result.workflow_success, true); }); test('rejects run records for unknown tasks', () => { From 8a65e9fb0e37f6cfe334d035fb4a112ccb3dbd5e Mon Sep 17 00:00:00 2001 From: Linfang Wang Date: Thu, 23 Jul 2026 19:52:50 +0800 Subject: [PATCH 09/19] docs(benchmark): publish sanitized v4 baseline --- .../discover-call/PUBLICATION_POLICY.md | 7 +- benchmarks/discover-call/README.md | 5 +- .../discover-call/publication-policy.json | 1 + ...07-23-gpt-5.6-sol-configured-v4.runs.jsonl | 54 + ...-23-gpt-5.6-sol-configured-v4.summary.json | 1979 +++++++++++++++++ .../2026-07-23-reference-v1-v4.runs.jsonl | 54 + .../2026-07-23-reference-v1-v4.summary.json | 1979 +++++++++++++++++ benchmarks/discover-call/results/README.md | 63 + benchmarks/discover-call/src/publication.mjs | 22 +- .../discover-call/test/publication.test.mjs | 26 +- docs/en-US/discover-call-benchmark.md | 156 +- docs/zh-CN/discover-call-benchmark.md | 95 +- 12 files changed, 4301 insertions(+), 140 deletions(-) create mode 100644 benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.runs.jsonl create mode 100644 benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.summary.json create mode 100644 benchmarks/discover-call/results/2026-07-23-reference-v1-v4.runs.jsonl create mode 100644 benchmarks/discover-call/results/2026-07-23-reference-v1-v4.summary.json diff --git a/benchmarks/discover-call/PUBLICATION_POLICY.md b/benchmarks/discover-call/PUBLICATION_POLICY.md index b1ee7c8..0749124 100644 --- a/benchmarks/discover-call/PUBLICATION_POLICY.md +++ b/benchmarks/discover-call/PUBLICATION_POLICY.md @@ -35,9 +35,10 @@ raw tool results, private prompts, tenant-private tools, or the unfiltered ordered discovery catalog. The publication command enforces `publication-policy.json`. It replaces the -ordered discovery list with its count and SHA-256 digest, preserves grounded -selection as an attestation, and refuses to publish a selected tool unless its -ID is explicitly approved. +ordered discovery list with its count and SHA-256 digest and preserves grounded +selection as an attestation. An approved selected tool keeps its ID; an +unapproved selected tool is represented only by a SHA-256 digest, so a benchmark +run cannot silently expand the public catalog. Provider authorization and namespace ownership must be confirmed before adding a tool ID to the public allowlist. Removing an approval requires regenerating all diff --git a/benchmarks/discover-call/README.md b/benchmarks/discover-call/README.md index 1166372..0133438 100644 --- a/benchmarks/discover-call/README.md +++ b/benchmarks/discover-call/README.md @@ -157,7 +157,7 @@ npm run publish -- \ Keep raw records outside the public repository. Publication removes operational identifiers and the ordered discovery catalog, adds count/digest attestations, -and rejects selected tools that are not explicitly approved. +and hashes selected tools that are not explicitly approved. Validate the checked-in task set, fixtures, and scorer: @@ -276,7 +276,8 @@ discovery-limit, or execution settings. The catalog and artifact visibility boundary is defined in [`PUBLICATION_POLICY.md`](PUBLICATION_POLICY.md). Until discovery reports per-tool visibility, bulk catalog content defaults to private; only approved -selected tool IDs may appear in public benchmark records. +selected tool IDs may appear in public benchmark records. Other selected tools +are represented only by a digest. The checked-in fixture validates the scorer only. It is synthetic and must not be presented as product performance. diff --git a/benchmarks/discover-call/publication-policy.json b/benchmarks/discover-call/publication-policy.json index e88b774..ba6c7f3 100644 --- a/benchmarks/discover-call/publication-policy.json +++ b/benchmarks/discover-call/publication-policy.json @@ -3,6 +3,7 @@ "policy_id": "public-artifact-v1", "catalog_visibility_default": "private", "publish_parameters": true, + "unapproved_selected_tool_handling": "hash", "legacy_lane_rewrites": { "oracle": "reference", "pinned-model": "configured-model" diff --git a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.runs.jsonl b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.runs.jsonl new file mode 100644 index 0000000..f7410e2 --- /dev/null +++ b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.runs.jsonl @@ -0,0 +1,54 @@ +{"schema_version":1,"benchmark_version":"v1","run_id":"371aaf4e-0615-4710-88dc-5c67e8f30685","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":1,"started_at":"2026-07-23T11:21:32.795Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"244bb1be6e597bd95075f0125dca8e2be2687f8607960916cb7a36ebbb25a5c9","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:21:55.843Z","source_record_sha256":"19b58e231fede0c078aec557ebbf6485480f275bca4fee4439579831fa306ee6"} +{"schema_version":1,"benchmark_version":"v1","run_id":"6c8d58e4-e962-49b6-8f06-1fb22482cf5f","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":2,"started_at":"2026-07-23T11:21:55.843Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"244bb1be6e597bd95075f0125dca8e2be2687f8607960916cb7a36ebbb25a5c9","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:22:19.345Z","source_record_sha256":"c377bc6f28eaa91c5faa5ab0abcfc8b74961cf294aa859842f16a03959b24162"} +{"schema_version":1,"benchmark_version":"v1","run_id":"0e761080-a3c5-4e82-82e2-cb93f97e962f","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":3,"started_at":"2026-07-23T11:22:19.345Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"244bb1be6e597bd95075f0125dca8e2be2687f8607960916cb7a36ebbb25a5c9","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:22:42.136Z","source_record_sha256":"a37ae7222aff18ebba3449bf0a8d50b1e11dc6e92beda09be6e12b73a877e8f2"} +{"schema_version":1,"benchmark_version":"v1","run_id":"f36b9c33-7310-420d-bc8f-26963eb5b722","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-23T11:22:42.136Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7090bc0d2bb54692f07a94a9c33a11631fe09798990dd9238790662776c22977","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:23:05.424Z","source_record_sha256":"a535f4847de398dce635099257343d97919151812fbb3ae2e801e2fbd1f493f1"} +{"schema_version":1,"benchmark_version":"v1","run_id":"478620bd-257e-4f3f-9faf-e5695f06fe8d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-23T11:23:05.424Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7090bc0d2bb54692f07a94a9c33a11631fe09798990dd9238790662776c22977","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON","eod":false,"timezone":"Exchange"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:23:31.485Z","source_record_sha256":"6778249cf3d5816cfaf1ec55f0de21bd325a88963280aea3c4d9902b3dcd77cd"} +{"schema_version":1,"benchmark_version":"v1","run_id":"0a54cf62-5bd6-4137-bc8b-0a1e687b5b1b","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-23T11:23:31.486Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7090bc0d2bb54692f07a94a9c33a11631fe09798990dd9238790662776c22977","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:23:53.420Z","source_record_sha256":"8275132e39abda084cdc23d716b7d032f91a5f28e80f9607dbe4b4d743a2d9bc"} +{"schema_version":1,"benchmark_version":"v1","run_id":"b93e6ec7-aa16-4fa5-b3f9-00e9d729a2f3","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-23T11:23:53.420Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2c68180e93be4ac81c6dbcab29687374ce54f07ddafe04848159bf8cfb882657","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":100,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:24:17.564Z","source_record_sha256":"18bf6bd0c62dfd783cf0de106292da5f50086c41b3d79c5c0430703f09f28624"} +{"schema_version":1,"benchmark_version":"v1","run_id":"db14c6bf-4780-47db-9af9-e7d8aa2988ff","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-23T11:24:17.564Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2c68180e93be4ac81c6dbcab29687374ce54f07ddafe04848159bf8cfb882657","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":100,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:24:41.890Z","source_record_sha256":"0f458900b9044b56fa89b726fdbf76bd9c494e46a236f5eb142b82f15f2e71b2"} +{"schema_version":1,"benchmark_version":"v1","run_id":"886e6db9-fa73-4f8e-9bee-1df89f6cd2ca","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-23T11:24:41.890Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2c68180e93be4ac81c6dbcab29687374ce54f07ddafe04848159bf8cfb882657","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:25:06.537Z","source_record_sha256":"6b42685637dad3c7dda90776bb768e967c3fb3fae6c3feaa40037d8ba69e0b60"} +{"schema_version":1,"benchmark_version":"v1","run_id":"58e58f43-855f-4ebf-a00e-451235a45f93","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":1,"started_at":"2026-07-23T11:25:06.537Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3cd090fbd5820604d10766221f77d1503a3d3271d85af28370d853e9dfbe34a1","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20AI","from":"2026-07-16","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:25:32.429Z","source_record_sha256":"8c07e0c9547070cfe87b68a2a6f1627fab3e453e495c0cce48b928eb7a51e51f"} +{"schema_version":1,"benchmark_version":"v1","run_id":"512dc6cf-6366-4973-b4aa-fea5ddad95ff","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":2,"started_at":"2026-07-23T11:25:32.429Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3cd090fbd5820604d10766221f77d1503a3d3271d85af28370d853e9dfbe34a1","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20AI","from":"2026-07-16","to":"2026-07-23T23:59:59+08:00","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:26:02.529Z","source_record_sha256":"02cf9ccaaa1239778a7e9d8715121b96e9a940d8a3072d4aec14a0d443c7cac3"} +{"schema_version":1,"benchmark_version":"v1","run_id":"42fd8322-85bc-4cd3-bff3-99382b9ee47d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":3,"started_at":"2026-07-23T11:26:02.529Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3cd090fbd5820604d10766221f77d1503a3d3271d85af28370d853e9dfbe34a1","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20OpenAI%20OR%20Anthropic%20OR%20%22machine%20learning%22","from":"2026-07-16","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:26:29.513Z","source_record_sha256":"67ebb682dfbc685b78cdb37f36ca4ba68a4a9677abe4e07f820f0d5d4cbba5cd"} +{"schema_version":1,"benchmark_version":"v1","run_id":"45e748b3-bd65-4f13-a226-9305abcb16ac","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T11:26:29.513Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d2b69a74b7ca36c6145b2f658dff7eae70ca9bdc663ebb85438798ecfdfbd517","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:26:54.396Z","source_record_sha256":"32eb9620da428b7fcd3e84022035dd5143e1ac02219e1144e743406fbff80901"} +{"schema_version":1,"benchmark_version":"v1","run_id":"44d32825-78ff-4068-a63c-42c43b528904","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T11:26:54.396Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d2b69a74b7ca36c6145b2f658dff7eae70ca9bdc663ebb85438798ecfdfbd517","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:27:19.885Z","source_record_sha256":"6c16e9d4850f6481daed57e2925f3fa62e4574552eda0019289d08d7efe7b49a"} +{"schema_version":1,"benchmark_version":"v1","run_id":"2a9c4066-3f0f-4bc9-ace4-c89d9914f323","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T11:27:19.885Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d2b69a74b7ca36c6145b2f658dff7eae70ca9bdc663ebb85438798ecfdfbd517","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:27:42.379Z","source_record_sha256":"83e747801cb18bf90368e5ff3c1797f615b3733a361a472614933fbc95412c59"} +{"schema_version":1,"benchmark_version":"v1","run_id":"973bb4c3-fd62-4528-910b-d4e9d9f13335","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T11:27:42.379Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"cc42608bbca8d2cfc12e11d4ace5f987cf4158fab8f32026c3e26d0e41e070aa","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:28:04.893Z","source_record_sha256":"6b210ca05391e2857e4a5f40d58014fc877f15f7adf3e4bffe231515010052c1"} +{"schema_version":1,"benchmark_version":"v1","run_id":"c4cdd1fd-1b64-4aab-b0f1-bf410779e959","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T11:28:04.894Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"cc42608bbca8d2cfc12e11d4ace5f987cf4158fab8f32026c3e26d0e41e070aa","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:28:34.255Z","source_record_sha256":"ba64b5942d022dde3eb7130aa7b16b1192cd3a0863da475c646f762fd801a91c"} +{"schema_version":1,"benchmark_version":"v1","run_id":"3df5595f-fcd1-4403-ac3b-27d50069f69b","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T11:28:34.255Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"cc42608bbca8d2cfc12e11d4ace5f987cf4158fab8f32026c3e26d0e41e070aa","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:28:58.524Z","source_record_sha256":"2084e7785dea619498038bc7f5b1aa917251af8fccf8ed45b99331fab5a43938"} +{"schema_version":1,"benchmark_version":"v1","run_id":"62beba38-b80d-42a2-8bfa-d628349af67e","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T11:28:58.524Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"ee2a0bbf7535b51d34f66e007ce04a9df26e23ab59529da23e5e644a32833f97","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:29:27.009Z","source_record_sha256":"87a076866c31c19ab3a6ba6aaa82dcd2373b2278dd99dc56c0613d9c13a218ff"} +{"schema_version":1,"benchmark_version":"v1","run_id":"78e43571-51cc-40a9-8b57-8f9c6b890794","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T11:29:27.009Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"ee2a0bbf7535b51d34f66e007ce04a9df26e23ab59529da23e5e644a32833f97","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:29:53.909Z","source_record_sha256":"340312f4dd31b0de6018de3f37ca4eab6f42bc62284d77f730cfc9ef7f15461c"} +{"schema_version":1,"benchmark_version":"v1","run_id":"67272715-b8f0-4ad6-8671-39c16bfe9f04","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T11:29:53.909Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"ee2a0bbf7535b51d34f66e007ce04a9df26e23ab59529da23e5e644a32833f97","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:30:20.862Z","source_record_sha256":"6a50a43d4ab1809ce760dc8735afc6967ce3a2ac55d872637fb5d153bb2f7749"} +{"schema_version":1,"benchmark_version":"v1","run_id":"d4ddae0d-2f59-4ecc-ab94-06727ec58cca","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-23T11:30:20.862Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"223bd05e0fcf12f37867246279bb93c17de568e2fb09922157c1600b83ce9e41","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T11:30:41.307Z","source_record_sha256":"8952e54a57b7bcef0c388f060b6e135b13337f5a62d7411670794fd651ca545e"} +{"schema_version":1,"benchmark_version":"v1","run_id":"d18e99eb-cc46-4095-a8c5-8372f149e061","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-23T11:30:41.307Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"223bd05e0fcf12f37867246279bb93c17de568e2fb09922157c1600b83ce9e41","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T11:31:02.728Z","source_record_sha256":"f629a1ec2d0328840f5be6d19470d8c937f866e296cc2be42ac66f519da5cc89"} +{"schema_version":1,"benchmark_version":"v1","run_id":"5568fd3b-32a1-4c2a-8d22-b4c22d69dd9d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-23T11:31:02.728Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"223bd05e0fcf12f37867246279bb93c17de568e2fb09922157c1600b83ce9e41","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T11:31:32.626Z","source_record_sha256":"24ebbb700feac211b1280355832cf27d5fbfecdac09a112646d1b4ca76fd21ac"} +{"schema_version":1,"benchmark_version":"v1","run_id":"80454152-d4cc-484e-8492-33d5fb8f06b9","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":1,"started_at":"2026-07-23T11:31:32.626Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2099bfa8887678c84902e7ec565c63edeef86ac5127e75b290713f8ff3a4f197","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:31:57.655Z","source_record_sha256":"7cf07a8098fc7de2900eee2c9471d41a5eeb8dec92903448c513bdc0e102d845"} +{"schema_version":1,"benchmark_version":"v1","run_id":"a703667c-b85b-4479-81f4-ea1af3f68891","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":2,"started_at":"2026-07-23T11:31:57.655Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66d9bb9820728c627264f0ade0a1ca2560a70f6bcb8e608cc1ff1dcba98fc27c","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:32:21.294Z","source_record_sha256":"d869eeb25eb2ddb2ba07e3c5abc1d375330ee466b7f89fe5fe9a6c6d1d91b19d"} +{"schema_version":1,"benchmark_version":"v1","run_id":"49211d01-07a0-43e3-82c3-b7fe5a567e04","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":3,"started_at":"2026-07-23T11:32:21.294Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66d9bb9820728c627264f0ade0a1ca2560a70f6bcb8e608cc1ff1dcba98fc27c","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:32:44.251Z","source_record_sha256":"fa084b186970020b9e6a71a1548b8362734847844104781dee93ed3e03002d8f"} +{"schema_version":1,"benchmark_version":"v1","run_id":"8dd15e90-d3e9-4ca3-8a64-dd163696948e","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":1,"started_at":"2026-07-23T11:32:44.251Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5e68e6de1d0883ee6bc55cb00d388d9a48144cac87bf641c9821f40515e7ea32","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:33:10.768Z","source_record_sha256":"d4d3af4888539d89a5f7ee1ac44781bf0493fe29b8c8ed69c54939ece33e1de0"} +{"schema_version":1,"benchmark_version":"v1","run_id":"9e62f261-75fb-4f20-9413-d7d048b239a5","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":2,"started_at":"2026-07-23T11:33:10.768Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5e68e6de1d0883ee6bc55cb00d388d9a48144cac87bf641c9821f40515e7ea32","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:33:35.333Z","source_record_sha256":"9d43f86f5725ac6b93f5b4160928570eb7823f158deb174594bc1666c0239029"} +{"schema_version":1,"benchmark_version":"v1","run_id":"618d54d7-28e3-41ea-8012-b8c054079799","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":3,"started_at":"2026-07-23T11:33:35.333Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5e68e6de1d0883ee6bc55cb00d388d9a48144cac87bf641c9821f40515e7ea32","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:33:57.236Z","source_record_sha256":"4f7bd85dec08098ae63d92fa1790041056e03428f2e763e15fcdbe27facc6d08"} +{"schema_version":1,"benchmark_version":"v1","run_id":"d3692a53-51a6-4970-ace5-06f8517985ae","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":1,"started_at":"2026-07-23T11:33:57.236Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:34:18.423Z","source_record_sha256":"007e7efd2a7ac75ec56044b2cdc4d9b083a58ce9e360a071b6c1fad953748480"} +{"schema_version":1,"benchmark_version":"v1","run_id":"26276fa6-e245-49b3-855b-60a4ea25b85b","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":2,"started_at":"2026-07-23T11:34:18.423Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:34:39.915Z","source_record_sha256":"264d4a0eb360bae5ab942c0fa5fdd5a29134af30bc82caf54e78efbd8bd2c81d"} +{"schema_version":1,"benchmark_version":"v1","run_id":"98acd9e1-dda2-4d51-84ad-6019ffcb21b3","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":3,"started_at":"2026-07-23T11:34:39.915Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:35:00.421Z","source_record_sha256":"fd6d10826ed0789d8a3421f44979c2507485b29181ae8648a512ec8b92b84b82"} +{"schema_version":1,"benchmark_version":"v1","run_id":"58c60360-c513-442d-b08e-f3c729fc5542","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":1,"started_at":"2026-07-23T11:35:00.421Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1fa372fa0e73a010285d8cae90eb1dc1b04ca995386b24d01d04998e09b2f09f","selection_grounded":true},"selection":{"tool_id":null,"tool_id_sha256":"aa938efb4e16c513e85cee2e773d507bc478cb8543b68333ab921785516c7a2e"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:35:22.330Z","source_record_sha256":"59258c6c137545703d2b6ee4fd34db1f67b130aac2b9ffcc15ff326bce2b74a9"} +{"schema_version":1,"benchmark_version":"v1","run_id":"a23aaa73-481a-4b61-b883-c8280ddcc8e0","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":2,"started_at":"2026-07-23T11:35:22.330Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1fa372fa0e73a010285d8cae90eb1dc1b04ca995386b24d01d04998e09b2f09f","selection_grounded":true},"selection":{"tool_id":null,"tool_id_sha256":"aa938efb4e16c513e85cee2e773d507bc478cb8543b68333ab921785516c7a2e"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:35:43.366Z","source_record_sha256":"af69472b0970e418b33cb16e5e055559d872ad62dca122c8f09021ea0425e118"} +{"schema_version":1,"benchmark_version":"v1","run_id":"57461645-9d30-4864-8a68-952d6d99a112","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":3,"started_at":"2026-07-23T11:35:43.366Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1fa372fa0e73a010285d8cae90eb1dc1b04ca995386b24d01d04998e09b2f09f","selection_grounded":true},"selection":{"tool_id":null,"tool_id_sha256":"aa938efb4e16c513e85cee2e773d507bc478cb8543b68333ab921785516c7a2e"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:36:08.708Z","source_record_sha256":"e54b101e094e8c07304b3e62a0d7f0c31ec354414df19fa337ad640f12d3977a"} +{"schema_version":1,"benchmark_version":"v1","run_id":"062ac004-ca1d-4938-a2ee-f56389cc5d8e","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T11:36:08.708Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"bcbd727391a8751121de802694245a7b97b9063d15176e9d3077129ca89c1f23","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected","message":"Adapter invocation failed"},"finished_at":"2026-07-23T11:36:48.187Z","source_record_sha256":"e1cc5dbe6957ce04839841829c7e6cb828c4a5c86ed5523f3f74021f6c56e65f"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ea6ee15f-1a67-457d-bf48-e7045eaff0f6","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T11:36:48.187Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"bcbd727391a8751121de802694245a7b97b9063d15176e9d3077129ca89c1f23","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected","message":"Adapter invocation failed"},"finished_at":"2026-07-23T11:37:37.695Z","source_record_sha256":"d4cf42d286d5f51b1f00d85fdf776f606ff182d2187b06d000af45d5357557b9"} +{"schema_version":1,"benchmark_version":"v1","run_id":"13e9d4a9-c237-4cfe-8b66-2b4bc6727157","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T11:37:37.695Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"bcbd727391a8751121de802694245a7b97b9063d15176e9d3077129ca89c1f23","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected","message":"Adapter invocation failed"},"finished_at":"2026-07-23T11:38:18.107Z","source_record_sha256":"928570b21b09c2fd63f8ae7ee1faf5f9b3b0b66ce7de966f60ec6f95c0d98eb6"} +{"schema_version":1,"benchmark_version":"v1","run_id":"46fa3b68-38ad-4728-9252-32e08dd52ee7","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":1,"started_at":"2026-07-23T11:38:18.107Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"406f82f0151e2aab38d28e5369d6e3b6b8f1e5a40414bd6ea7cafc3eb9f40259","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":10,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:38:46.770Z","source_record_sha256":"b0691ac1115ede5d59b2c49e6c7ad87c6c440e8ad911982666f36d32ffa00b5e"} +{"schema_version":1,"benchmark_version":"v1","run_id":"19c84ab6-eab7-4ed1-b536-20d6db38801c","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":2,"started_at":"2026-07-23T11:38:46.770Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"406f82f0151e2aab38d28e5369d6e3b6b8f1e5a40414bd6ea7cafc3eb9f40259","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:39:13.722Z","source_record_sha256":"3326539c1c392a2b9f51ea323d1c669cbaa2134e4a71361b2a194a1bca2932d4"} +{"schema_version":1,"benchmark_version":"v1","run_id":"d2438aee-241f-4be1-8c92-0a21375a39cd","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":3,"started_at":"2026-07-23T11:39:13.722Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"406f82f0151e2aab38d28e5369d6e3b6b8f1e5a40414bd6ea7cafc3eb9f40259","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:39:36.630Z","source_record_sha256":"57f8f030306de487ac1e7f3de74fbca58e026038c7612b4bf239b44e747a5322"} +{"schema_version":1,"benchmark_version":"v1","run_id":"e8832d33-357f-4f68-925a-d4381142678b","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":1,"started_at":"2026-07-23T11:39:36.631Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2057edc3ba2996ada07d11b091f42383501397f2bd7499d626a300e8022fe784","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:40:01.350Z","source_record_sha256":"8cb15016b1a0c653593e161998b315894277c9cb902fa5c330a3063d1160a0d7"} +{"schema_version":1,"benchmark_version":"v1","run_id":"e87ea408-9168-46df-acbd-a61a85f7ca9e","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":2,"started_at":"2026-07-23T11:40:01.350Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2057edc3ba2996ada07d11b091f42383501397f2bd7499d626a300e8022fe784","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:40:23.845Z","source_record_sha256":"0d878dd10de5848935a490bf4a52317f61c12593c5f73f2aa339a4ec25863952"} +{"schema_version":1,"benchmark_version":"v1","run_id":"e0ac9896-d511-4ba3-90e8-21f01cfde6bc","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":3,"started_at":"2026-07-23T11:40:23.845Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2057edc3ba2996ada07d11b091f42383501397f2bd7499d626a300e8022fe784","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:40:46.507Z","source_record_sha256":"7fbe8c452d77153276d8bc178a8e9ff6998a55ecdfc493ecc0a37a86400ab94f"} +{"schema_version":1,"benchmark_version":"v1","run_id":"5b03766d-4b69-4b3a-acb7-ad89437a239a","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-23T11:40:46.507Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:41:12.868Z","source_record_sha256":"d3033fec8aaaa09d0a46ac19779117f41f0aa04f75a1d20e9cb483295c208615"} +{"schema_version":1,"benchmark_version":"v1","run_id":"62fa2743-433e-4a20-887c-74feafdda446","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-23T11:41:12.868Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:41:40.786Z","source_record_sha256":"d8cbbadffea583a4baab90221b7bc5b3b934c0d2890a78cbd3a85dcc9e2905f2"} +{"schema_version":1,"benchmark_version":"v1","run_id":"90981cc4-2bb6-457f-b8eb-950da1d98923","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T11:41:40.786Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:42:07.441Z","source_record_sha256":"f046d6ef60c603401007987ef63d16bf71e7a3f98e1f70365a07db4c53c65917"} +{"schema_version":1,"benchmark_version":"v1","run_id":"e63213b2-c4c0-4f6c-9d7a-a761069870e2","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":1,"started_at":"2026-07-23T11:42:07.441Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8a3148a19d493f9d95e740965080d759189237c39909aa19fc69be443262784f","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T11:42:29.049Z","source_record_sha256":"32003fe6a9cbb7e6c544e16eae18045665e137fefbc0a3c5547129239fef49d8"} +{"schema_version":1,"benchmark_version":"v1","run_id":"1054878a-b27b-46e5-b9a6-8d6bc3ebec8a","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":2,"started_at":"2026-07-23T11:42:29.049Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8a3148a19d493f9d95e740965080d759189237c39909aa19fc69be443262784f","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T11:42:52.943Z","source_record_sha256":"f5f2b88718734478c4181208300c19098d856c1f1afb48d42e03d3fb0a5cd8d1"} +{"schema_version":1,"benchmark_version":"v1","run_id":"dc98fe27-231c-4a95-af86-12400d405e65","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":3,"started_at":"2026-07-23T11:42:52.943Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8a3148a19d493f9d95e740965080d759189237c39909aa19fc69be443262784f","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T11:43:15.920Z","source_record_sha256":"e4aa45b8a0f7fd32d6cdce7cc5fee63c43885c339568d76e9e20cac4cf1aba5e"} +{"schema_version":1,"benchmark_version":"v1","run_id":"940ed547-9ddc-419c-b0a1-1064f987be0e","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":1,"started_at":"2026-07-23T11:43:15.920Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8581564b8aa970da4b2fbcef387a2e4418ca70298d1c0b2107b4aa7bd16388f7","selection_grounded":true},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"selection_grounded":true,"required_parameters":["season"]},"parameters":{"league":39,"season":2025},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:43:36.976Z","source_record_sha256":"b8ccab960b3ab136b7cadf256b89a4f8da5e090121aaed5d16a0bce35d20ffb3"} +{"schema_version":1,"benchmark_version":"v1","run_id":"8ec93298-1f2c-42ef-ac71-af37df93b741","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":2,"started_at":"2026-07-23T11:43:36.976Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8581564b8aa970da4b2fbcef387a2e4418ca70298d1c0b2107b4aa7bd16388f7","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.98a153bd"},"inspection":{"selection_grounded":true,"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:43:59.877Z","source_record_sha256":"a2a5cdc1c047821de45fbfc48d2686353905e6720ad936e30a8218f2c4f00928"} +{"schema_version":1,"benchmark_version":"v1","run_id":"5f9a762b-7f86-4f7a-a89b-b0abd1e889fc","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":3,"started_at":"2026-07-23T11:43:59.877Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8581564b8aa970da4b2fbcef387a2e4418ca70298d1c0b2107b4aa7bd16388f7","selection_grounded":true},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"selection_grounded":true,"required_parameters":["season"]},"parameters":{"league":39,"season":2025},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:44:24.921Z","source_record_sha256":"68550a9afdb928de6f5eae92226ddf277280452e88acf1a7e4031c08b3894d3a"} diff --git a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.summary.json b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.summary.json new file mode 100644 index 0000000..3e83934 --- /dev/null +++ b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.summary.json @@ -0,0 +1,1979 @@ +{ + "schema_version": 1, + "methodology": "discover-call-v1", + "generated_at": "2026-07-23T11:45:48.815Z", + "models": [ + { + "model": "gpt-5.6-sol", + "lane": "configured-model", + "tasks": 18, + "trials_per_task": 3, + "runs": 54, + "completed": 51, + "executed": 51, + "selection_grounded_rate": 1, + "inspection_grounded_rate": 1, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0.8889, + "call_success_rate": 0.8824, + "result_nonempty_rate": 0.8824, + "workflow_success_rate": 0.7778, + "workflow_success_task_cluster_bootstrap_95": [ + 0.5556, + 0.9444 + ], + "failures_by_stage": { + "adapter": 3 + }, + "failures_by_reason": { + "tool_use_rejected": 3 + } + } + ], + "records": [ + { + "run_id": "371aaf4e-0615-4710-88dc-5c67e8f30685", + "task_id": "weather-london", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "6c8d58e4-e962-49b6-8f06-1fb22482cf5f", + "task_id": "weather-london", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "0e761080-a3c5-4e82-82e2-cb93f97e962f", + "task_id": "weather-london", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "f36b9c33-7310-420d-bc8f-26963eb5b722", + "task_id": "stock-aapl", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "478620bd-257e-4f3f-9faf-e5695f06fe8d", + "task_id": "stock-aapl", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "0a54cf62-5bd6-4137-bc8b-0a1e687b5b1b", + "task_id": "stock-aapl", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "b93e6ec7-aa16-4fa5-b3f9-00e9d729a2f3", + "task_id": "exchange-usd-eur", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "db14c6bf-4780-47db-9af9-e7d8aa2988ff", + "task_id": "exchange-usd-eur", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "886e6db9-fa73-4f8e-9bee-1df89f6cd2ca", + "task_id": "exchange-usd-eur", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "58e58f43-855f-4ebf-a00e-451235a45f93", + "task_id": "news-ai", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "512dc6cf-6366-4973-b4aa-fea5ddad95ff", + "task_id": "news-ai", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "42fd8322-85bc-4cd3-bff3-99382b9ee47d", + "task_id": "news-ai", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "45e748b3-bd65-4f13-a226-9305abcb16ac", + "task_id": "papers-climate", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "44d32825-78ff-4068-a63c-42c43b528904", + "task_id": "papers-climate", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "2a9c4066-3f0f-4bc9-ace4-c89d9914f323", + "task_id": "papers-climate", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "973bb4c3-fd62-4528-910b-d4e9d9f13335", + "task_id": "timezone-tokyo", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "result_nonempty": true, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "c4cdd1fd-1b64-4aab-b0f1-bf410779e959", + "task_id": "timezone-tokyo", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "result_nonempty": true, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "3df5595f-fcd1-4403-ac3b-27d50069f69b", + "task_id": "timezone-tokyo", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "result_nonempty": true, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "62beba38-b80d-42a2-8bfa-d628349af67e", + "task_id": "geocode-address", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "78e43571-51cc-40a9-8b57-8f9c6b890794", + "task_id": "geocode-address", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "67272715-b8f0-4ad6-8671-39c16bfe9f04", + "task_id": "geocode-address", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "d4ddae0d-2f59-4ecc-ab94-06727ec58cca", + "task_id": "ip-geolocation", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": false, + "result_nonempty": false, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "d18e99eb-cc46-4095-a8c5-8372f149e061", + "task_id": "ip-geolocation", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": false, + "result_nonempty": false, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "5568fd3b-32a1-4c2a-8d22-b4c22d69dd9d", + "task_id": "ip-geolocation", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": false, + "result_nonempty": false, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "80454152-d4cc-484e-8492-33d5fb8f06b9", + "task_id": "crypto-price", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "a703667c-b85b-4479-81f4-ea1af3f68891", + "task_id": "crypto-price", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "49211d01-07a0-43e3-82c3-b7fe5a567e04", + "task_id": "crypto-price", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "8dd15e90-d3e9-4ca3-8a64-dd163696948e", + "task_id": "web-search", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "9e62f261-75fb-4f20-9413-d7d048b239a5", + "task_id": "web-search", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "618d54d7-28e3-41ea-8012-b8c054079799", + "task_id": "web-search", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "d3692a53-51a6-4970-ace5-06f8517985ae", + "task_id": "air-quality", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "26276fa6-e245-49b3-855b-60a4ea25b85b", + "task_id": "air-quality", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "98acd9e1-dda2-4d51-84ad-6019ffcb21b3", + "task_id": "air-quality", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "58c60360-c513-442d-b08e-f3c729fc5542", + "task_id": "market-news", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "a23aaa73-481a-4b61-b883-c8280ddcc8e0", + "task_id": "market-news", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "57461645-9d30-4864-8a68-952d6d99a112", + "task_id": "market-news", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "062ac004-ca1d-4938-a2ee-f56389cc5d8e", + "task_id": "domain-intel", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": false, + "executed": false, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": null, + "result_nonempty": null, + "workflow_success": false, + "error_stage": "adapter", + "error_reason": "tool_use_rejected", + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "ea6ee15f-1a67-457d-bf48-e7045eaff0f6", + "task_id": "domain-intel", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": false, + "executed": false, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": null, + "result_nonempty": null, + "workflow_success": false, + "error_stage": "adapter", + "error_reason": "tool_use_rejected", + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "13e9d4a9-c237-4cfe-8b66-2b4bc6727157", + "task_id": "domain-intel", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": false, + "executed": false, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": null, + "result_nonempty": null, + "workflow_success": false, + "error_stage": "adapter", + "error_reason": "tool_use_rejected", + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "46fa3b68-38ad-4728-9252-32e08dd52ee7", + "task_id": "image-search", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "19c84ab6-eab7-4ed1-b536-20d6db38801c", + "task_id": "image-search", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "d2438aee-241f-4be1-8c92-0a21375a39cd", + "task_id": "image-search", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "e8832d33-357f-4f68-925a-d4381142678b", + "task_id": "market-holidays", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "e87ea408-9168-46df-acbd-a61a85f7ca9e", + "task_id": "market-holidays", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "e0ac9896-d511-4ba3-90e8-21f01cfde6bc", + "task_id": "market-holidays", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "5b03766d-4b69-4b3a-acb7-ad89437a239a", + "task_id": "earthquake-catalog", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "62fa2743-433e-4a20-887c-74feafdda446", + "task_id": "earthquake-catalog", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "90981cc4-2bb6-457f-b8eb-950da1d98923", + "task_id": "earthquake-catalog", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "e63213b2-c4c0-4f6c-9d7a-a761069870e2", + "task_id": "company-profile", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": false, + "result_nonempty": false, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "1054878a-b27b-46e5-b9a6-8d6bc3ebec8a", + "task_id": "company-profile", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": false, + "result_nonempty": false, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "dc98fe27-231c-4a95-af86-12400d405e65", + "task_id": "company-profile", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": false, + "result_nonempty": false, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "940ed547-9ddc-419c-b0a1-1064f987be0e", + "task_id": "sports-standings", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "8ec93298-1f2c-42ef-ac71-af37df93b741", + "task_id": "sports-standings", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "5f9a762b-7f86-4f7a-a89b-b0abd1e889fc", + "task_id": "sports-standings", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + } + ] +} diff --git a/benchmarks/discover-call/results/2026-07-23-reference-v1-v4.runs.jsonl b/benchmarks/discover-call/results/2026-07-23-reference-v1-v4.runs.jsonl new file mode 100644 index 0000000..b31cc81 --- /dev/null +++ b/benchmarks/discover-call/results/2026-07-23-reference-v1-v4.runs.jsonl @@ -0,0 +1,54 @@ +{"schema_version":1,"benchmark_version":"v1","run_id":"47cb7805-14a0-4e3d-9467-eada2d3d5aaf","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":1,"started_at":"2026-07-23T11:48:57.201Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"60a9778d19cb8b02430702c056caa75f4e19deac6783e4f41e81045a80bb5f80","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:01.956Z","source_record_sha256":"bb3eccb640d33192f56ffdcd31e28cbb6808e52d855bdb7d98950e6badebff90"} +{"schema_version":1,"benchmark_version":"v1","run_id":"cae0eb3b-e6de-4378-9aaa-7d8f52e5d199","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":2,"started_at":"2026-07-23T11:49:01.956Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"60a9778d19cb8b02430702c056caa75f4e19deac6783e4f41e81045a80bb5f80","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:06.391Z","source_record_sha256":"c7c17c944dadb3959a97004cf2a43106cdf94599d5a268b4ae42607eb7134db6"} +{"schema_version":1,"benchmark_version":"v1","run_id":"2f81c94d-6247-47eb-abb3-23b5a66053be","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":3,"started_at":"2026-07-23T11:49:06.391Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"60a9778d19cb8b02430702c056caa75f4e19deac6783e4f41e81045a80bb5f80","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:09.865Z","source_record_sha256":"1a0f663ae9ddbe3530a06e497cb34a4bee8a0aae462ff40d9d2c8eee412bc102"} +{"schema_version":1,"benchmark_version":"v1","run_id":"a6fd0659-6976-499d-90e2-4db0e7d51719","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-23T11:49:09.865Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9f5212bec5a91748d3d60cb1901272c20496f288ccbbbb3d826f3920e3d66592","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:13.050Z","source_record_sha256":"8972f96aa8c5b2bf909568698fe1344d8a94ec200a021c241c9a2ba66a013250"} +{"schema_version":1,"benchmark_version":"v1","run_id":"47ba4b70-b8fe-4549-99c8-6c3e644e684b","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-23T11:49:13.050Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9f5212bec5a91748d3d60cb1901272c20496f288ccbbbb3d826f3920e3d66592","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:16.054Z","source_record_sha256":"36c277092f214e6254183f8843225aa6d2c44f7d0a96160fec638013892c5a9d"} +{"schema_version":1,"benchmark_version":"v1","run_id":"b9c6a6f7-dbf2-49a3-b37f-076232390cd9","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-23T11:49:16.055Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9f5212bec5a91748d3d60cb1901272c20496f288ccbbbb3d826f3920e3d66592","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:19.027Z","source_record_sha256":"0714760ecc4029e7b6d3d2fb1d8570a223380760705bf7e0d3e03b68373a6a30"} +{"schema_version":1,"benchmark_version":"v1","run_id":"eaccc622-c533-45d9-a3ad-fb8a6061a563","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-23T11:49:19.027Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"05301a2d5f1dcd68a46a128056393fce873a9c954f50a904b20d44e0da086501","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:22.152Z","source_record_sha256":"b22dfc0edc9fcec92bd0c137a12f7a5d3dfdac26c5e6b227e45a584e1aed5e5e"} +{"schema_version":1,"benchmark_version":"v1","run_id":"84f3dd2a-9da0-4510-8f41-bd13698f84f0","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-23T11:49:22.152Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"05301a2d5f1dcd68a46a128056393fce873a9c954f50a904b20d44e0da086501","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:24.995Z","source_record_sha256":"9dbea2b37f2c81c3c65d061a0959a4367cdd20faef7fa1dbfe9854ee23d8c6d8"} +{"schema_version":1,"benchmark_version":"v1","run_id":"1b9a3943-a23b-4ee1-a637-e55204be18ea","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-23T11:49:24.995Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"05301a2d5f1dcd68a46a128056393fce873a9c954f50a904b20d44e0da086501","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:28.646Z","source_record_sha256":"d9fcd63fb991a169e7effa261baf8111331948a9e2e808cb0fa460ad338c2cd8"} +{"schema_version":1,"benchmark_version":"v1","run_id":"e59a22e8-5b81-4233-97f9-204dc9d37cd8","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":1,"started_at":"2026-07-23T11:49:28.646Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c4bbd90fb5d55233f0f6f0ef99607a3f2607d18fc477747e948347693409157a","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"artificial intelligence","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:31.575Z","source_record_sha256":"c564a710dfa464fe7a61301356f3661fe4a97897e15ed38fca59f8c68a59cffb"} +{"schema_version":1,"benchmark_version":"v1","run_id":"53f4c366-ae9d-4b54-9837-f29f855998b5","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":2,"started_at":"2026-07-23T11:49:31.575Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c4bbd90fb5d55233f0f6f0ef99607a3f2607d18fc477747e948347693409157a","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"artificial intelligence","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:34.512Z","source_record_sha256":"efd288148efa0f03827fc210788bdd569559a8e24ae29af457f6c322e87693a9"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ba35f756-d608-4e36-846e-da6c997c012b","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":3,"started_at":"2026-07-23T11:49:34.512Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c4bbd90fb5d55233f0f6f0ef99607a3f2607d18fc477747e948347693409157a","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"artificial intelligence","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:37.192Z","source_record_sha256":"1b4176d0ea0fbf9d04d7e61c271b9a8b9c8203a95fd1177f83fc28399caf85ac"} +{"schema_version":1,"benchmark_version":"v1","run_id":"4f315c20-69d3-4932-b9fe-9ac53abb26b3","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T11:49:37.192Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:41.059Z","source_record_sha256":"9f789afec504d169e789dba41d995393630dde1699d7f443486e5bfd7c363ff7"} +{"schema_version":1,"benchmark_version":"v1","run_id":"994aa1d9-989a-4cbb-bb65-0a6748264e2c","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T11:49:41.059Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:45.414Z","source_record_sha256":"843c439d6403310a6b963c590a9486f153dd92c6433ee058a8586a5da3acb57a"} +{"schema_version":1,"benchmark_version":"v1","run_id":"1e562721-df0e-401f-afb0-97173f7723bb","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T11:49:45.414Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:49.536Z","source_record_sha256":"3c03fe59e1241ad2a33f7b06102a6a57b4717775752fdda2877901831cb423f4"} +{"schema_version":1,"benchmark_version":"v1","run_id":"dabf80ee-36bc-405a-94cd-674a3a026f48","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T11:49:49.536Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"f4605d6b4c45644fc9cec341c593f461cdef28806d63c58187e65c1311fd88b9","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id","message":"Adapter did not return a non-empty tool_id"},"finished_at":"2026-07-23T11:49:51.286Z","source_record_sha256":"ca3514c8b130287b1fdeda4d4059b03e09dbf2dbb5e7b7c92a629ab4868e5cef"} +{"schema_version":1,"benchmark_version":"v1","run_id":"cc6c272d-971c-45da-b3f0-0a980620a770","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T11:49:51.286Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"f4605d6b4c45644fc9cec341c593f461cdef28806d63c58187e65c1311fd88b9","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id","message":"Adapter did not return a non-empty tool_id"},"finished_at":"2026-07-23T11:49:53.649Z","source_record_sha256":"508f234db9002ea4f1a4f542cf03a5ceecc68ef5b2220f3a1aac0afbf7f735d0"} +{"schema_version":1,"benchmark_version":"v1","run_id":"93cd2a79-0d68-4912-8b80-281ff8b5afcf","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T11:49:53.649Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"f4605d6b4c45644fc9cec341c593f461cdef28806d63c58187e65c1311fd88b9","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id","message":"Adapter did not return a non-empty tool_id"},"finished_at":"2026-07-23T11:49:55.417Z","source_record_sha256":"06831aa8ae3762571f2cff2bf7cfb286373e6af28ed4fcf7792799f26a66b91c"} +{"schema_version":1,"benchmark_version":"v1","run_id":"fe761783-51bd-469f-a93d-e70b02c5aca4","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T11:49:55.417Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"70aa2fe386fa3968550e5521c201c217ff7c531c47da064ac80c7028098b0fc1","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:58.783Z","source_record_sha256":"c4b1f95c00f2162eb2ae0ea1a1c18d32bfff43c1aa66c0e090ef29801ca59674"} +{"schema_version":1,"benchmark_version":"v1","run_id":"e066fabb-8e6a-43f1-8de1-118b9b2685d6","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T11:49:58.783Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"70aa2fe386fa3968550e5521c201c217ff7c531c47da064ac80c7028098b0fc1","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:02.312Z","source_record_sha256":"8e8f70b308a50ad0586d20cdae1b4600f8e023f8c311df4a60bcfdc364ee9f8b"} +{"schema_version":1,"benchmark_version":"v1","run_id":"38a5dda0-7976-47cf-8360-8de4b6616f38","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T11:50:02.312Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"70aa2fe386fa3968550e5521c201c217ff7c531c47da064ac80c7028098b0fc1","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:05.390Z","source_record_sha256":"80804429ff13575a752328940d66e93c675d37de2fb9a20abe083ac6a063f5f2"} +{"schema_version":1,"benchmark_version":"v1","run_id":"a9efa250-3048-4843-9625-16b3719e367c","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-23T11:50:05.390Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e945d876b1c1f057c618eb121c5016982e2a640663a00218afdabf96a2f871c5","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true,"required_parameters":["ip"]},"parameters":{"ip":"8.8.8.8"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:08.634Z","source_record_sha256":"a655366007019129793bc3b859fd6ddbd995cb75a67e7df8e0f9e204aaa115f8"} +{"schema_version":1,"benchmark_version":"v1","run_id":"f78bf98f-2a48-4d41-a0be-6fee18aa0e6d","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-23T11:50:08.634Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e945d876b1c1f057c618eb121c5016982e2a640663a00218afdabf96a2f871c5","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true,"required_parameters":["ip"]},"parameters":{"ip":"8.8.8.8"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:11.669Z","source_record_sha256":"5794c2964429894a7ff28e15719e302d5c1b00006e0d46a48a0091aa0373d249"} +{"schema_version":1,"benchmark_version":"v1","run_id":"de52544a-c3ce-4ba2-9a30-136ff600bd46","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-23T11:50:11.669Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e945d876b1c1f057c618eb121c5016982e2a640663a00218afdabf96a2f871c5","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true,"required_parameters":["ip"]},"parameters":{"ip":"8.8.8.8"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:15.049Z","source_record_sha256":"3caa2c5028a16b205dc50eaa713fd02faacbce50e4b6e927d2e406716654467e"} +{"schema_version":1,"benchmark_version":"v1","run_id":"8b9d5b99-2532-4c71-bcbe-86fe01addec9","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":1,"started_at":"2026-07-23T11:50:15.049Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d6fafa5dcc5b80de51ca0ad0cd3e975827e3281c0aa5dbb07e57ede5fb7c75d6","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:18.517Z","source_record_sha256":"b0bd21d4827c242b5d7ac49306b3836434d584b594ccdd31d69aa98a8b5a2eaf"} +{"schema_version":1,"benchmark_version":"v1","run_id":"5750acdd-7fe8-42b7-8055-4e0b493a3563","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":2,"started_at":"2026-07-23T11:50:18.517Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d6fafa5dcc5b80de51ca0ad0cd3e975827e3281c0aa5dbb07e57ede5fb7c75d6","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:21.713Z","source_record_sha256":"9f5b324ec84f9a90f260e24eac25dfd714ca0342a5b446135e26d1d7ae90d649"} +{"schema_version":1,"benchmark_version":"v1","run_id":"9a5d4dd7-0b65-4fcb-8c0c-9236ad8e4385","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":3,"started_at":"2026-07-23T11:50:21.713Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d6fafa5dcc5b80de51ca0ad0cd3e975827e3281c0aa5dbb07e57ede5fb7c75d6","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:24.746Z","source_record_sha256":"f095e22ade3257eea4e57f61ce50ad30bc703b39b7f3d40938e727c2da3a9864"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ba98ef8c-c4a6-4451-9ddf-eeaf399350e6","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":1,"started_at":"2026-07-23T11:50:24.746Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c09c84639eec63b361d1f798b2ce033c380f12dcacaec1d15585f303caca3f52","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:28.102Z","source_record_sha256":"d2b118b09a4620806f4f7c95b0af97295280a9283ac1aa0047b285031a081d5f"} +{"schema_version":1,"benchmark_version":"v1","run_id":"89d87d76-b9b2-43a9-8e97-9b9e585abd2f","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":2,"started_at":"2026-07-23T11:50:28.102Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c09c84639eec63b361d1f798b2ce033c380f12dcacaec1d15585f303caca3f52","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:30.318Z","source_record_sha256":"44ef22f9bc5dc52ef9433772a2fcf042186d2b0728cfdece9f765562e95a23e3"} +{"schema_version":1,"benchmark_version":"v1","run_id":"84e9641c-d425-455f-9c5f-78af187aa8a0","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":3,"started_at":"2026-07-23T11:50:30.318Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c09c84639eec63b361d1f798b2ce033c380f12dcacaec1d15585f303caca3f52","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:32.939Z","source_record_sha256":"388a156cfce0edd62ce1aea09c593fb1cd1bbe633286fe3e401233b3751a5bd5"} +{"schema_version":1,"benchmark_version":"v1","run_id":"bbc436b0-0423-438b-8c99-09c149b108c1","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":1,"started_at":"2026-07-23T11:50:32.939Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:35.293Z","source_record_sha256":"9d9d4b3c45840e35b730cdaf7e0dc764f5687407a14530008be2b0ce5739c3ca"} +{"schema_version":1,"benchmark_version":"v1","run_id":"6e8d097d-729b-49ff-9ff8-26d269b4c392","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":2,"started_at":"2026-07-23T11:50:35.293Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:37.516Z","source_record_sha256":"ea41ada6eef7178da3d7a67c61d73eeb45ed8012c9394e269ed2f9df4488abb9"} +{"schema_version":1,"benchmark_version":"v1","run_id":"36d8ce65-a35b-4f18-bce3-2130cb6bb18f","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":3,"started_at":"2026-07-23T11:50:37.516Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:39.898Z","source_record_sha256":"1023b919340a08acb80c944f97c62089c481d96b1701532db9e0a9a2c9531db5"} +{"schema_version":1,"benchmark_version":"v1","run_id":"2d63ffe2-9364-4a38-9143-310d2253f292","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":1,"started_at":"2026-07-23T11:50:39.898Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"bfe110e90706c2afbad72202d763dfe7ecc090a0acdd71d13ee5cd939020de3d","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:43.417Z","source_record_sha256":"345fecac7ee3c2f461a1883ab78429bf9c16530e648df0de26062da55ca1f3d6"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ac9707ba-8ec7-4621-9bd3-f1924b19fc39","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":2,"started_at":"2026-07-23T11:50:43.417Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"bfe110e90706c2afbad72202d763dfe7ecc090a0acdd71d13ee5cd939020de3d","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:46.844Z","source_record_sha256":"412e4f494e67085a1b12d2a2f75ffda2f56119cbb5720e46acaf2d5e0d3704a9"} +{"schema_version":1,"benchmark_version":"v1","run_id":"5d5face2-a17f-4ebd-955e-f74779ff753b","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":3,"started_at":"2026-07-23T11:50:46.844Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"bfe110e90706c2afbad72202d763dfe7ecc090a0acdd71d13ee5cd939020de3d","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:50.621Z","source_record_sha256":"ab2435871876cb232c3f0ed08d7e54dccf7d2eb3d0ff732927fa2aca3f7225f5"} +{"schema_version":1,"benchmark_version":"v1","run_id":"571bc24e-6bfe-45c0-a24c-751d93c62aa5","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T11:50:50.621Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7c6eca1d49ca307d5d3c65dbae2d15a95977f722f7378d9d78ed7592e69a7e52","selection_grounded":true},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"selection_grounded":true,"required_parameters":["domain"]},"parameters":{"domain":"example.com"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:53.813Z","source_record_sha256":"7cf298e0f4bacd47caf7364fd7332bc0d9229efab746c328741892de23d0d398"} +{"schema_version":1,"benchmark_version":"v1","run_id":"7333c00c-ad5d-4355-9885-60619358b073","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T11:50:53.813Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7c6eca1d49ca307d5d3c65dbae2d15a95977f722f7378d9d78ed7592e69a7e52","selection_grounded":true},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"selection_grounded":true,"required_parameters":["domain"]},"parameters":{"domain":"example.com"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:56.683Z","source_record_sha256":"672e5cc17bf7fe7faf572e699f6b57e82403654fa42dd09ad94f28cb498cb521"} +{"schema_version":1,"benchmark_version":"v1","run_id":"305235ce-03f2-4ad3-a771-f88f78fdd385","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T11:50:56.683Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7c6eca1d49ca307d5d3c65dbae2d15a95977f722f7378d9d78ed7592e69a7e52","selection_grounded":true},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"selection_grounded":true,"required_parameters":["domain"]},"parameters":{"domain":"example.com"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:59.655Z","source_record_sha256":"164657ab106df8fac53297f629233beeb6eb774b0dd7f3eb33c7222660922809"} +{"schema_version":1,"benchmark_version":"v1","run_id":"bbd74432-588a-4d16-bada-e4719e2939aa","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":1,"started_at":"2026-07-23T11:50:59.655Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f226b4b6b337cb8cc47181f72cca9d36b6927f00b68525ffbabb1c97bff58d26","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:03.539Z","source_record_sha256":"1e9330a654eb3c466ed208b1ec3c9fbdad93b81a1c8b580e72da65774f9086c1"} +{"schema_version":1,"benchmark_version":"v1","run_id":"180041f2-2807-4ebe-b2b0-6fff70caad34","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":2,"started_at":"2026-07-23T11:51:03.539Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f226b4b6b337cb8cc47181f72cca9d36b6927f00b68525ffbabb1c97bff58d26","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:06.997Z","source_record_sha256":"2c8cbf34315ebea332cee35646f4553dd14347eef4e2f0c8e682586b69e0d799"} +{"schema_version":1,"benchmark_version":"v1","run_id":"09697d0a-d5dc-45c9-a58e-0f83265ad2a7","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":3,"started_at":"2026-07-23T11:51:06.997Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f226b4b6b337cb8cc47181f72cca9d36b6927f00b68525ffbabb1c97bff58d26","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:10.362Z","source_record_sha256":"87788e797974bcd217f59d742b914ce6feb80a617c7aa485bce22926d61f67b9"} +{"schema_version":1,"benchmark_version":"v1","run_id":"b2313e18-6665-425e-8282-d5c0dffc7269","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":1,"started_at":"2026-07-23T11:51:10.362Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6e380480779ccd9d93e77a96eac5da3e446beaaad9ad28b51fa3ba6c0de70b7b","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:13.250Z","source_record_sha256":"f179c532a1434f7d4b084dce5f38e3bf9af627b201bcbaccca5546a98bb673a3"} +{"schema_version":1,"benchmark_version":"v1","run_id":"0604aab2-f4dd-4ec5-80e2-07f0a46394a3","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":2,"started_at":"2026-07-23T11:51:13.250Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6e380480779ccd9d93e77a96eac5da3e446beaaad9ad28b51fa3ba6c0de70b7b","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:16.159Z","source_record_sha256":"8e86270c79ecc8f891910b1d651587290a81536167854a173d2195c5ac48eb37"} +{"schema_version":1,"benchmark_version":"v1","run_id":"73935f3c-23be-4b2a-9114-5a911d01c2ca","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":3,"started_at":"2026-07-23T11:51:16.159Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6e380480779ccd9d93e77a96eac5da3e446beaaad9ad28b51fa3ba6c0de70b7b","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:18.898Z","source_record_sha256":"a5fb91158fbef2e1d09903059b5e697e9d0c96d38bb04fd7ad70b38ea73716e4"} +{"schema_version":1,"benchmark_version":"v1","run_id":"dad96c5e-3b67-40e4-8871-02d941c0f743","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-23T11:51:18.898Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:25.589Z","source_record_sha256":"49c08423482e2e2ff25d7ab791708b746e64610cd2d92e4f634753abbe3b872b"} +{"schema_version":1,"benchmark_version":"v1","run_id":"3fc5fb2c-f4ca-4362-9936-1758bde06182","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-23T11:51:25.589Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:32.866Z","source_record_sha256":"de289675bf86659570273300f6c458c005c184eee2ccfd418bde189ef3c7b4e5"} +{"schema_version":1,"benchmark_version":"v1","run_id":"698dc8bb-6506-4b16-a949-ce9468d44b63","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T11:51:32.866Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:41.337Z","source_record_sha256":"e1168aa5bc62229b06e59c9e1062b95a3087434e90e9e344c107da4552c3e66d"} +{"schema_version":1,"benchmark_version":"v1","run_id":"9514047d-0a9a-4f1b-a14a-141f7676f466","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":1,"started_at":"2026-07-23T11:51:41.337Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e15912f4cbddea8b650e6fb9eecb8e09d60f2dd18d61bd0f1df0989c4a9f0cac","selection_grounded":true},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:45.787Z","source_record_sha256":"3d97bd5527879d8a88299cbf26255350fb2fcc3db92750ec339f613377f60049"} +{"schema_version":1,"benchmark_version":"v1","run_id":"bcce8d9a-66cc-40d3-b72b-13d1b8b306df","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":2,"started_at":"2026-07-23T11:51:45.787Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e15912f4cbddea8b650e6fb9eecb8e09d60f2dd18d61bd0f1df0989c4a9f0cac","selection_grounded":true},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:49.856Z","source_record_sha256":"b12a8f59abbe680b852bfabe6ca10816598effd87d19acf457132baffe58e4c6"} +{"schema_version":1,"benchmark_version":"v1","run_id":"da6af207-0055-474a-baa8-5286967891e1","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":3,"started_at":"2026-07-23T11:51:49.856Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e15912f4cbddea8b650e6fb9eecb8e09d60f2dd18d61bd0f1df0989c4a9f0cac","selection_grounded":true},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:53.639Z","source_record_sha256":"ad82e114ec205a72a5ea96cfa3072470b6685e9237a832d134b535fc294eb66e"} +{"schema_version":1,"benchmark_version":"v1","run_id":"8cf8bcdd-bb75-4572-91db-44b6f075668b","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":1,"started_at":"2026-07-23T11:51:53.639Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"61aa7a6fac8a6b83b4e3502ad86f65d037077df0a9a9f50440a747046d39f06c","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"selection_grounded":true,"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:57.466Z","source_record_sha256":"f7966fd1f2a1dd3290af8a93df96ba4a6418e20a6254d7a4e1a936f463142dd2"} +{"schema_version":1,"benchmark_version":"v1","run_id":"9d55a677-d6ec-4061-b647-977bcb370dfc","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":2,"started_at":"2026-07-23T11:51:57.466Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"61aa7a6fac8a6b83b4e3502ad86f65d037077df0a9a9f50440a747046d39f06c","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"selection_grounded":true,"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:52:00.460Z","source_record_sha256":"b5fb410551a6642c3618ab6281dc8a97a3914fd7ce4d3ff7380da2e65427009f"} +{"schema_version":1,"benchmark_version":"v1","run_id":"dd1b7396-4eaa-4c46-af88-db113b37c68f","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":3,"started_at":"2026-07-23T11:52:00.460Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"61aa7a6fac8a6b83b4e3502ad86f65d037077df0a9a9f50440a747046d39f06c","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"selection_grounded":true,"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:52:03.442Z","source_record_sha256":"a16c58332263a82ffd97a879f731261d8ccbf4e3a18437b0e9883ea6e6ddb407"} diff --git a/benchmarks/discover-call/results/2026-07-23-reference-v1-v4.summary.json b/benchmarks/discover-call/results/2026-07-23-reference-v1-v4.summary.json new file mode 100644 index 0000000..723194b --- /dev/null +++ b/benchmarks/discover-call/results/2026-07-23-reference-v1-v4.summary.json @@ -0,0 +1,1979 @@ +{ + "schema_version": 1, + "methodology": "discover-call-v1", + "generated_at": "2026-07-23T11:52:09.856Z", + "models": [ + { + "model": "reference-v1", + "lane": "reference", + "tasks": 18, + "trials_per_task": 3, + "runs": 54, + "completed": 51, + "executed": 51, + "selection_grounded_rate": 0.9444, + "inspection_grounded_rate": 0.9444, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0.9444, + "call_success_rate": 1, + "result_nonempty_rate": 1, + "workflow_success_rate": 0.9444, + "workflow_success_task_cluster_bootstrap_95": [ + 0.8333, + 1 + ], + "failures_by_stage": { + "select": 3 + }, + "failures_by_reason": { + "missing_tool_id": 3 + } + } + ], + "records": [ + { + "run_id": "47cb7805-14a0-4e3d-9467-eada2d3d5aaf", + "task_id": "weather-london", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "cae0eb3b-e6de-4378-9aaa-7d8f52e5d199", + "task_id": "weather-london", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "2f81c94d-6247-47eb-abb3-23b5a66053be", + "task_id": "weather-london", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "a6fd0659-6976-499d-90e2-4db0e7d51719", + "task_id": "stock-aapl", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "47ba4b70-b8fe-4549-99c8-6c3e644e684b", + "task_id": "stock-aapl", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "b9c6a6f7-dbf2-49a3-b37f-076232390cd9", + "task_id": "stock-aapl", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "eaccc622-c533-45d9-a3ad-fb8a6061a563", + "task_id": "exchange-usd-eur", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "84f3dd2a-9da0-4510-8f41-bd13698f84f0", + "task_id": "exchange-usd-eur", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "1b9a3943-a23b-4ee1-a637-e55204be18ea", + "task_id": "exchange-usd-eur", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "e59a22e8-5b81-4233-97f9-204dc9d37cd8", + "task_id": "news-ai", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "53f4c366-ae9d-4b54-9837-f29f855998b5", + "task_id": "news-ai", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "ba35f756-d608-4e36-846e-da6c997c012b", + "task_id": "news-ai", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "4f315c20-69d3-4932-b9fe-9ac53abb26b3", + "task_id": "papers-climate", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "994aa1d9-989a-4cbb-bb65-0a6748264e2c", + "task_id": "papers-climate", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "1e562721-df0e-401f-afb0-97173f7723bb", + "task_id": "papers-climate", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "dabf80ee-36bc-405a-94cd-674a3a026f48", + "task_id": "timezone-tokyo", + "model": "reference-v1", + "trial": 1, + "completed": false, + "executed": false, + "selection_grounded": false, + "inspection_grounded": false, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": null, + "result_nonempty": null, + "workflow_success": false, + "error_stage": "select", + "error_reason": "missing_tool_id", + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "cc6c272d-971c-45da-b3f0-0a980620a770", + "task_id": "timezone-tokyo", + "model": "reference-v1", + "trial": 2, + "completed": false, + "executed": false, + "selection_grounded": false, + "inspection_grounded": false, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": null, + "result_nonempty": null, + "workflow_success": false, + "error_stage": "select", + "error_reason": "missing_tool_id", + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "93cd2a79-0d68-4912-8b80-281ff8b5afcf", + "task_id": "timezone-tokyo", + "model": "reference-v1", + "trial": 3, + "completed": false, + "executed": false, + "selection_grounded": false, + "inspection_grounded": false, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": null, + "result_nonempty": null, + "workflow_success": false, + "error_stage": "select", + "error_reason": "missing_tool_id", + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "fe761783-51bd-469f-a93d-e70b02c5aca4", + "task_id": "geocode-address", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "e066fabb-8e6a-43f1-8de1-118b9b2685d6", + "task_id": "geocode-address", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "38a5dda0-7976-47cf-8360-8de4b6616f38", + "task_id": "geocode-address", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "a9efa250-3048-4843-9625-16b3719e367c", + "task_id": "ip-geolocation", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "f78bf98f-2a48-4d41-a0be-6fee18aa0e6d", + "task_id": "ip-geolocation", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "de52544a-c3ce-4ba2-9a30-136ff600bd46", + "task_id": "ip-geolocation", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "8b9d5b99-2532-4c71-bcbe-86fe01addec9", + "task_id": "crypto-price", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "5750acdd-7fe8-42b7-8055-4e0b493a3563", + "task_id": "crypto-price", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "9a5d4dd7-0b65-4fcb-8c0c-9236ad8e4385", + "task_id": "crypto-price", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "ba98ef8c-c4a6-4451-9ddf-eeaf399350e6", + "task_id": "web-search", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "89d87d76-b9b2-43a9-8e97-9b9e585abd2f", + "task_id": "web-search", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "84e9641c-d425-455f-9c5f-78af187aa8a0", + "task_id": "web-search", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "bbc436b0-0423-438b-8c99-09c149b108c1", + "task_id": "air-quality", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "6e8d097d-729b-49ff-9ff8-26d269b4c392", + "task_id": "air-quality", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "36d8ce65-a35b-4f18-bce3-2130cb6bb18f", + "task_id": "air-quality", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "2d63ffe2-9364-4a38-9143-310d2253f292", + "task_id": "market-news", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "ac9707ba-8ec7-4621-9bd3-f1924b19fc39", + "task_id": "market-news", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "5d5face2-a17f-4ebd-955e-f74779ff753b", + "task_id": "market-news", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "571bc24e-6bfe-45c0-a24c-751d93c62aa5", + "task_id": "domain-intel", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "7333c00c-ad5d-4355-9885-60619358b073", + "task_id": "domain-intel", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "305235ce-03f2-4ad3-a771-f88f78fdd385", + "task_id": "domain-intel", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "bbd74432-588a-4d16-bada-e4719e2939aa", + "task_id": "image-search", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "180041f2-2807-4ebe-b2b0-6fff70caad34", + "task_id": "image-search", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "09697d0a-d5dc-45c9-a58e-0f83265ad2a7", + "task_id": "image-search", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "b2313e18-6665-425e-8282-d5c0dffc7269", + "task_id": "market-holidays", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "0604aab2-f4dd-4ec5-80e2-07f0a46394a3", + "task_id": "market-holidays", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "73935f3c-23be-4b2a-9114-5a911d01c2ca", + "task_id": "market-holidays", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "dad96c5e-3b67-40e4-8871-02d941c0f743", + "task_id": "earthquake-catalog", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "3fc5fb2c-f4ca-4362-9936-1758bde06182", + "task_id": "earthquake-catalog", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "698dc8bb-6506-4b16-a949-ce9468d44b63", + "task_id": "earthquake-catalog", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "9514047d-0a9a-4f1b-a14a-141f7676f466", + "task_id": "company-profile", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "bcce8d9a-66cc-40d3-b72b-13d1b8b306df", + "task_id": "company-profile", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "da6af207-0055-474a-baa8-5286967891e1", + "task_id": "company-profile", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "8cf8bcdd-bb75-4572-91db-44b6f075668b", + "task_id": "sports-standings", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "9d55a677-d6ec-4061-b647-977bcb370dfc", + "task_id": "sports-standings", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "dd1b7396-4eaa-4c46-af88-db113b37c68f", + "task_id": "sports-standings", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1", + "toolkit_revision": "deeda797e50a0dae649bf91d7503d09ec497b324", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-22.1", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + } + ] +} diff --git a/benchmarks/discover-call/results/README.md b/benchmarks/discover-call/results/README.md index cd1fe3c..db03992 100644 --- a/benchmarks/discover-call/results/README.md +++ b/benchmarks/discover-call/results/README.md @@ -1,5 +1,68 @@ # Published benchmark results +## 2026-07-23 — v4 public quality baseline + +The v4 baseline compares the curated reference route and the configured model +over immutable `tasks/v4.jsonl`: 18 tasks, three trials per task, Top 10 +discovery, and real calls. All 54 trials in each lane are retained. + +| Metric | Curated reference route | `gpt-5.6-sol` configured model | +| --- | ---: | ---: | +| Runs | 54 | 54 | +| Completed and executed | 51 / 54 | 51 / 54 | +| Selection grounded | 94.44% | 100% | +| Inspection grounded | 94.44% | 100% | +| Required-parameter accuracy | 100% | 100% | +| Constraint accuracy | 94.44% | 88.89% | +| Call success among attempted calls | 100% (51 / 51) | 88.24% (45 / 51) | +| Non-empty result among attempted calls | 100% (51 / 51) | 88.24% (45 / 51) | +| Strict workflow success | 94.44% (51 / 54) | 77.78% (42 / 54) | +| Workflow success, 95% task-cluster bootstrap | 83.33%–100% | 55.56%–94.44% | + +The v4 strict benchmark gap is **16.66 percentage points**. This is an +end-to-end benchmark difference, not a pure routing effect: the two sequential +lanes observed different live Top-10 catalog snapshots, as recorded by their +different catalog-observation digests. + +Shared metadata: + +- task set: `tasks/v4.jsonl` +- task-set SHA-256: + `afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed` +- source revision: + `deeda797e50a0dae649bf91d7503d09ec497b324` +- API base URL: `https://qveris.ai/api/v1` +- observed API revision: `2026-07-22.1` +- catalog revision: `unreported` +- discovery limit: `10` +- runtime: Node.js `v24.2.0`, macOS arm64 +- publication policy: `public-artifact-v1` + +Lane metadata: + +- curated reference route: model `reference-v1`, model revision + `deterministic-reference-v1`, catalog-observation SHA-256 + `7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1` +- configured model: model `gpt-5.6-sol`, provider model revision `unreported`, + reasoning effort `medium`, Codex CLI `0.144.1`, + catalog-observation SHA-256 + `e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054` + +The reference route's three failures are the expected `timezone-tokyo` +coverage misses; its 51 calls all succeeded and returned structurally non-empty +results. The configured model has 12 strict failures: three Tokyo constraint +misses, three IP lookup call failures, three company-profile call failures, and +three domain-intelligence `tool_use_rejected` adapter failures. All three +Bitcoin trials pass in v4, confirming that the task-versioned `id=1` mapping +removed the known v3 scoring false negative. + +Artifacts: + +- [`2026-07-23-reference-v1-v4.runs.jsonl`](2026-07-23-reference-v1-v4.runs.jsonl) +- [`2026-07-23-reference-v1-v4.summary.json`](2026-07-23-reference-v1-v4.summary.json) +- [`2026-07-23-gpt-5.6-sol-configured-v4.runs.jsonl`](2026-07-23-gpt-5.6-sol-configured-v4.runs.jsonl) +- [`2026-07-23-gpt-5.6-sol-configured-v4.summary.json`](2026-07-23-gpt-5.6-sol-configured-v4.summary.json) + ## 2026-07-23 — v3 diagnostic comparison This diagnostic run compares the curated reference route and a configured model diff --git a/benchmarks/discover-call/src/publication.mjs b/benchmarks/discover-call/src/publication.mjs index 63c555b..2e3210c 100644 --- a/benchmarks/discover-call/src/publication.mjs +++ b/benchmarks/discover-call/src/publication.mjs @@ -26,9 +26,7 @@ export function sanitizePublicRecord( catalogObservationSha256 = record.metadata?.catalog_observation_sha256 ?? 'unreported', ) { const selectedToolId = record.selection?.tool_id ?? null; - if (selectedToolId && !approvedToolIds.has(selectedToolId)) { - throw new Error(`Selected tool is not approved for public artifacts: ${selectedToolId}`); - } + const selectedToolApproved = selectedToolId ? approvedToolIds.has(selectedToolId) : false; const resultToolIds = array(record.discovery?.result_tool_ids); const returnedToolIds = array(record.inspection?.returned_tool_ids); @@ -59,7 +57,12 @@ export function sanitizePublicRecord( snapshot_sha256: sha256(JSON.stringify(resultToolIds)), selection_grounded: selectionGrounded, }, - selection: { tool_id: selectedToolId }, + selection: { + tool_id: selectedToolApproved ? selectedToolId : null, + ...(selectedToolId && !selectedToolApproved + ? { tool_id_sha256: sha256(selectedToolId) } + : {}), + }, inspection: { selection_grounded: inspectionGrounded, required_parameters: array(record.inspection?.required_parameters), @@ -100,6 +103,9 @@ export function validatePolicy(policy) { ) { throw new Error('Publication policy legacy_lane_rewrites must map strings to strings'); } + if (policy.unapproved_selected_tool_handling !== 'hash') { + throw new Error('Publication policy must hash unapproved selected tools'); + } } export function validatePublicRecords(records, policy) { @@ -115,6 +121,14 @@ export function validatePublicRecords(records, policy) { if (selectedToolId && !approvedToolIds.has(selectedToolId)) { throw new Error(`Public artifact contains an unapproved selected tool: ${selectedToolId}`); } + if ( + record.selection?.tool_id_sha256 !== undefined && + (typeof record.selection.tool_id_sha256 !== 'string' || + record.selection.tool_id_sha256.length !== 64 || + selectedToolId) + ) { + throw new Error('Public artifact selected tool digest is invalid'); + } if ( !Number.isInteger(record.discovery?.result_count) || typeof record.discovery?.snapshot_sha256 !== 'string' || diff --git a/benchmarks/discover-call/test/publication.test.mjs b/benchmarks/discover-call/test/publication.test.mjs index 9969aa6..c865dbe 100644 --- a/benchmarks/discover-call/test/publication.test.mjs +++ b/benchmarks/discover-call/test/publication.test.mjs @@ -8,6 +8,7 @@ const policy = { policy_id: 'test-public-v1', catalog_visibility_default: 'private', publish_parameters: true, + unapproved_selected_tool_handling: 'hash', legacy_lane_rewrites: { 'pinned-model': 'configured-model' }, approved_selected_tool_ids: ['weather.tool'], forbidden_fields: [ @@ -65,18 +66,17 @@ test('publishes grounded attestations and hashes instead of operational identifi assert.doesNotThrow(() => validatePublicRecords([record], policy)); }); -test('refuses to publish an unapproved selected tool', () => { - assert.throws( - () => - sanitizePublicRecords( - [ - { - selection: { tool_id: 'private.tool' }, - discovery: { result_tool_ids: ['private.tool'] }, - }, - ], - policy, - ), - /not approved/, +test('hashes an unapproved selected tool instead of expanding the public catalog', () => { + const [record] = sanitizePublicRecords( + [ + { + selection: { tool_id: 'private.tool' }, + discovery: { result_tool_ids: ['private.tool'] }, + }, + ], + policy, ); + assert.equal(record.selection.tool_id, null); + assert.equal(record.selection.tool_id_sha256.length, 64); + assert.equal(record.discovery.selection_grounded, true); }); diff --git a/docs/en-US/discover-call-benchmark.md b/docs/en-US/discover-call-benchmark.md index e197640..14785f1 100644 --- a/docs/en-US/discover-call-benchmark.md +++ b/docs/en-US/discover-call-benchmark.md @@ -1,85 +1,93 @@ # Discover → Call accuracy benchmark -QVeris evaluates the complete agent workflow instead of publishing an -unverifiable adjective or scoring only search relevance. The public harness is -in [`benchmarks/discover-call`](../../benchmarks/discover-call/README.md). +QVeris evaluates the complete agent workflow instead of scoring search +relevance alone. The public harness is in +[`benchmarks/discover-call`](../../benchmarks/discover-call/README.md). -This benchmark is scoped to the contract level: deterministic, per-release -scoring of the public discover → inspect → call workflow, reproducible by -anyone with an API key. Domain-level evaluation of long-horizon professional -tasks with judged scoring is a separate instrument and is intentionally not -part of this harness. +This benchmark stays at the contract level: it measures the public discover → +inspect → call workflow with deterministic scoring and real executions. +Long-horizon, judged domain evaluations are a separate instrument. ## Methodology -For every task and model trial, the harness runs `discover`, asks the model to -select a returned capability, runs `inspect`, asks the model to construct -parameters from the current schema, and then performs a real `call` when -execution is enabled. - -The scorer publishes grounded selection, grounded inspection, required-parameter -accuracy, task-constraint accuracy, call success, and strict end-to-end workflow -success. A successful call must also return a non-empty result to satisfy the -result-validity and strict-workflow gates. Workflow success requires every -component to pass and therefore cannot be reported from dry runs. The aggregate -includes a 95% Wilson interval and safe failure counts by stage and reason. -Transient `429` and `503` responses are retried first; exhausted API failures -remain in the denominator and are reported separately by failure stage. - -The task set uses semantic parameter aliases rather than a single fixed tool ID. -This avoids penalizing a model for selecting a different capability that fulfills -the same task while still requiring its choice to come from the actual discovery -response. Model adapters receive canonical messages and a response schema but -never receive the scorer's ground-truth constraints. - -The v3 task set adds explicit handling for combined parameters such as -`symbol=USD/EUR` and opt-in URL decoding, without changing the historical v2 -scoring contract. It also supports three complementary comparison lanes: - -- the deterministic Oracle lane measures the current platform ceiling by - selecting a fixed valid candidate only when it appears in the query's Top 10; -- the pinned-model lane measures changes over time with an immutable model and - adapter configuration; -- the current-model lane measures the currently recommended model. - -The difference between Oracle and model strict-workflow success is the routing -gap. Component metrics and failure reasons identify whether the remaining loss -comes from discovery coverage, model routing, parameter construction, -execution, or result validity. - -## Reproducibility requirements - -Published results must retain failures, use at least three trials per task, name -an immutable model version and adapter revision, record the toolkit commit, and -include raw JSONL records plus the generated summary. Credentials and raw -provider error bodies must never be published. +For every task and trial, the harness runs `discover`, asks the adapter to +select a returned capability, runs `inspect`, asks the adapter to construct +parameters from the current schema, and performs a real `call`. + +The scorer reports grounded selection and inspection, required-parameter +accuracy, task-constraint accuracy, call success, structural result +non-emptiness, and strict end-to-end workflow success. Non-empty does not mean +semantically correct. Dry runs never count as workflow success. The 95% +interval uses task-cluster bootstrap resampling so repeated trials of one task +are not treated as independent task draws. + +Task sets use semantic parameter aliases rather than one fixed tool ID. Model +adapters receive canonical messages and a response schema, but never the +scorer's ground-truth constraints. + +The comparison lanes are: + +- `reference`: a curated reference route that uses a fixed candidate only when + it appears in the observed Top 10. It represents those candidates, not every + possible platform route. +- `configured-model`: a recorded model, CLI, reasoning, adapter, and task-set + configuration when the provider does not expose a verifiable immutable model + revision. +- `pinned-model`: reserved for a provider model revision that can be verified + as immutable; the runner requires `--model-revision`. +- `current-model`: the currently recommended model under the same task + contract. + +The difference between reference and model strict-workflow success is the +**strict benchmark gap**. It is not automatically a pure routing effect: +sequential lanes can observe different live catalog snapshots. Component +metrics, failure reasons, API revision, and catalog-observation digests must be +considered with it. + +## Reproducibility and publication + +Published results retain every failed trial and use at least three trials per +task. They record the model identifier and provider revision (or +`unreported`), adapter and toolkit revisions, task-set digest, runtime, API +revision, catalog revision when reported, catalog-observation digest, endpoint, +and discovery limit. + +Only sanitized JSONL is committed. Public artifacts omit execution, search, and +connection identifiers and the ordered discovery catalog. Approved selected +tool IDs may remain visible; other selected tools are represented only by a +digest. See the +[publication policy](../../benchmarks/discover-call/PUBLICATION_POLICY.md). ## Published results -The v3 comparison was run on 2026-07-23 with three trials per task and real -execution. The deterministic Oracle measures the current fixed-query platform -ceiling; the pinned model uses `gpt-5.6-sol`, medium reasoning effort, and Codex -CLI 0.144.1. +The official v4 baseline was run on 2026-07-23 over 18 immutable tasks, three +trials each, with real calls. -| Metric | Oracle | Pinned model | +| Metric | Curated reference route | `gpt-5.6-sol` configured model | | --- | ---: | ---: | -| Completed and executed | 51 / 54 | 52 / 54 | -| Constraint accuracy | 94.44% | 83.33% | -| Call and non-empty-result success | 100% (51 / 51) | 88.46% (46 / 52) | -| Strict workflow success | 94.44% (51 / 54) | 72.22% (39 / 54) | -| Workflow success, 95% Wilson interval | 84.89%–98.09% | 59.11%–82.38% | - -The strict routing gap is 22.22 percentage points. The Oracle's only failures -were the three Tokyo-timezone trials, where the fixed query returned no Top 10 -capability accepting a city or coordinates. The pinned model had the same three -semantic misses, six selected-tool execution failures, two safely classified -disabled-tool attempts, and additional strict constraint misses. Three -successful Bitcoin calls used CoinMarketCap's numeric `id=1`; because the v3 -constraint expects a value containing `BTC`, they remain known identifier-mapping -false negatives rather than being reclassified after the run. - -See the [result notes, immutable revisions, raw JSONL, and generated -summary](../../benchmarks/discover-call/results/README.md). The synthetic scorer -fixture remains test-only and is not a product-performance claim. The original -v2 result remains preserved there as a historical baseline under its original -scoring contract. +| Completed and executed | 51 / 54 | 51 / 54 | +| Constraint accuracy | 94.44% | 88.89% | +| Call and non-empty-result success | 100% (51 / 51) | 88.24% (45 / 51) | +| Strict workflow success | 94.44% (51 / 54) | 77.78% (42 / 54) | +| Workflow success, 95% task-cluster bootstrap | 83.33%–100% | 55.56%–94.44% | + +The strict benchmark gap is 16.66 percentage points. The reference route's +three failures are Tokyo-timezone coverage misses. The configured model's 12 +strict failures are three Tokyo constraint misses, three IP lookup call +failures, three company-profile call failures, and three safely classified +`tool_use_rejected` adapter failures. + +The configured lane used `gpt-5.6-sol`, medium reasoning, and Codex CLI 0.144.1. +Its provider model revision is `unreported`, so this is not presented as a +pinned model snapshot. Both lanes observed API revision `2026-07-22.1`; the API +did not report a catalog revision, and the separate catalog-observation digests +differ. + +The earlier v3 run is retained only as a diagnostic baseline. Its three +successful Bitcoin calls used provider-specific `id=1`, which v3 incorrectly +scored as constraint failures. Immutable `tasks/v4.jsonl` explicitly recognizes +that mapping, and all three Bitcoin trials pass in v4. + +See the [result notes, revisions, sanitized JSONL, and generated +summaries](../../benchmarks/discover-call/results/README.md). The synthetic +scorer fixture remains test-only and is not a product-performance claim. diff --git a/docs/zh-CN/discover-call-benchmark.md b/docs/zh-CN/discover-call-benchmark.md index 1302840..4484ac5 100644 --- a/docs/zh-CN/discover-call-benchmark.md +++ b/docs/zh-CN/discover-call-benchmark.md @@ -1,63 +1,70 @@ # Discover → Call 准确率评测 -QVeris 评测完整的 Agent 工作流，而不是只使用无法验证的形容词，或只对搜索相关性打分。公开评测工具位于 +QVeris 评测完整的 Agent 工作流，而不只对搜索相关性打分。公开评测工具位于 [`benchmarks/discover-call`](../../benchmarks/discover-call/README.md)。 -本评测的范围限定在契约层：对公开 discover → inspect → call 工作流做确定性打分，每次发版可复跑， -任何持有 API Key 的人都能复现。长周期专业任务、judged 打分的领域级评测是另一套仪器，刻意不纳入 -本工具的范围。 +本评测限定在契约层：对公开 discover → inspect → call 工作流进行确定性打分和真实执行。长周期、 +依赖人工判断的领域级评测属于另一套评测体系。 ## 方法 -每个任务和模型 trial 都按同一流程执行：调用 `discover`，让模型从实际返回结果中选择能力，调用 -`inspect`，让模型根据当前参数 schema 构造参数，并在启用执行时完成真实 `call`。 +每个任务和 trial 都按同一流程运行：调用 `discover`，让 adapter 从实际返回结果中选择能力，调用 +`inspect`，让 adapter 根据当前 schema 构造参数，最后执行真实 `call`。 -评分分别报告选择是否来自 discover、inspect 是否确认同一能力、必填参数完整率、任务约束准确率、 -call 成功率、结果有效率，以及严格的端到端工作流成功率。成功的 call 还必须返回非空结果，才能 -通过结果有效性与严格工作流门槛。工作流成功要求全部分项通过，因此不能用 dry run 生成。汇总 -同时给出 95% Wilson 区间，以及经过安全归类的失败阶段和原因计数。 -`429` 和 `503` 瞬时响应会先按客户端语义重试；重试耗尽的 API 失败仍保留在分母中，并按失败阶段 -单独报告。 +评分分别报告选择与检查是否有真实返回依据、必填参数完整率、任务约束准确率、call 成功率、结果结构 +是否非空，以及严格的端到端工作流成功率。“结果非空”不代表语义正确。Dry run 不计为工作流成功。 +95% 区间使用 task-cluster bootstrap，以任务为重采样单位，避免把同一任务的重复 trial 当作独立任务。 -任务集使用语义参数别名，而不是把唯一 tool ID 当作标准答案。这样不会把能够完成同一任务的其他能力 -误判为失败，同时仍要求模型的选择来自真实 discovery 响应。Model adapter 只会收到固定 messages -与 response schema，不会得到 scorer 使用的 ground-truth constraints。 +任务集使用语义参数别名，而不是把唯一 tool ID 当作标准答案。模型 adapter 只接收固定 messages 和 +response schema，不会得到 scorer 使用的 ground-truth constraints。 -v3 任务集在不改变历史 v2 评分契约的前提下，明确支持 `symbol=USD/EUR` 这类组合参数，以及由任务 -显式启用的 URL 解码，并提供三条互补的对比基线： +对比 lane 的含义如下： -- 确定性的 Oracle lane 只会在固定查询 Top 10 中出现预设有效候选时选择它，用于衡量当前平台上限； -- pinned-model lane 固定不可变模型和 adapter 配置，用于纵向观察版本变化； -- current-model lane 使用当前推荐模型，用于观察当前能力。 +- `reference`：curated reference route。仅当预设候选出现在本次 Top 10 中时才使用；它只代表这些 + 精选候选，不代表平台所有可能路径。 +- `configured-model`：当 provider 没有可核验的不可变模型 revision 时，记录模型、CLI、reasoning、 + adapter 和任务集配置。 +- `pinned-model`：仅用于可核验的不可变 provider 模型 revision；runner 强制要求 + `--model-revision`。 +- `current-model`：在同一任务契约下使用当前推荐模型。 -Oracle 与模型的严格工作流成功率之差就是 routing gap。还需要结合各分项指标和失败原因，判断损失 -来自 discovery 覆盖、模型路由、参数构造、执行，还是结果有效性。 +reference 与模型的严格工作流成功率之差称为 **strict benchmark gap**。它不自动等于纯路由差异： +顺序执行的 lane 可能观察到不同的实时目录快照，因此必须同时查看分项指标、失败原因、API revision +和目录观察摘要。 -## 可复现要求 +## 可复现与公开策略 -公开结果必须保留失败样本，每个任务至少执行三个 trial，注明不可变的模型版本与 adapter revision， -记录 toolkit commit，并同时提交原始 JSONL 与生成的汇总文件。API Key、访问令牌和原始 provider -错误正文不得进入公开结果。 +公开结果保留所有失败 trial，每个任务至少运行三次，并记录模型标识与 provider revision（或 +`unreported`）、adapter 与 toolkit revision、任务集摘要、runtime、API revision、服务端有返回时 +的 catalog revision、catalog observation 摘要、endpoint 和 discovery limit。 + +仓库只提交脱敏后的 JSONL。公开 artifact 不包含 execution、search、connection 标识，也不包含完整 +有序 discovery 目录。已批准的 selected tool ID 可以保留；其他 selected tool 只保留摘要。详见 +[公开策略](../../benchmarks/discover-call/PUBLICATION_POLICY.md)。 ## 已发布结果 -v3 对比基线于 2026-07-23 运行，每个任务执行三个 trial，并启用真实执行。确定性的 Oracle -用于衡量固定查询下的当前平台上限；pinned model 使用 `gpt-5.6-sol`、medium reasoning effort -和 Codex CLI 0.144.1。 +正式 v4 基线于 2026-07-23 运行：18 个不可变任务，每个任务三个 trial，并启用真实调用。 -| 指标 | Oracle | Pinned model | +| 指标 | Curated reference route | `gpt-5.6-sol` configured model | | --- | ---: | ---: | -| 完成参数化并执行 | 51 / 54 | 52 / 54 | -| 任务约束准确率 | 94.44% | 83.33% | -| call 成功且结果非空 | 100%（51 / 51） | 88.46%（46 / 52） | -| 严格工作流成功率 | 94.44%（51 / 54） | 72.22%（39 / 54） | -| 工作流成功率 95% Wilson 区间 | 84.89%–98.09% | 59.11%–82.38% | - -严格 routing gap 为 22.22 个百分点。Oracle 的三个失败全部来自东京时区任务：固定查询的 Top 10 -中没有能够接收城市或坐标的能力。Pinned model 同样出现三个时区语义未命中，另有六次所选工具 -执行失败、两次经过安全分类的禁用工具尝试，以及其他严格约束未命中。三次 Bitcoin 调用通过 -CoinMarketCap 的数字标识 `id=1` 成功返回非空结果，但 v3 约束要求参数值包含 `BTC`，因此仍作为 -已知的标识映射假阴性保留，未在运行后静默改分。 - -完整的[结果说明、不可变 revision、原始 JSONL 与生成汇总](../../benchmarks/discover-call/results/README.md) -均已公开。原始 v2 结果按其原评分契约保留为历史基线。scorer fixture 仍只用于测试，不代表产品性能。 +| 完成参数化并执行 | 51 / 54 | 51 / 54 | +| 任务约束准确率 | 94.44% | 88.89% | +| call 成功且结果非空 | 100%（51 / 51） | 88.24%（45 / 51） | +| 严格工作流成功率 | 94.44%（51 / 54） | 77.78%（42 / 54） | +| 工作流成功率 95% task-cluster bootstrap | 83.33%–100% | 55.56%–94.44% | + +strict benchmark gap 为 16.66 个百分点。reference 的三次失败均为东京时区目录覆盖缺口。configured +model 的 12 次严格失败包括：三次东京约束未命中、三次 IP lookup 调用失败、三次 company profile +调用失败，以及三次安全归类的 `tool_use_rejected` adapter 失败。 + +configured lane 使用 `gpt-5.6-sol`、medium reasoning 和 Codex CLI 0.144.1。provider 模型 +revision 为 `unreported`，因此不会称为 pinned model snapshot。两个 lane 都观察到 API revision +`2026-07-22.1`；API 未报告 catalog revision，且两次独立运行的 catalog observation 摘要不同。 + +较早的 v3 只保留为 diagnostic baseline。其三次 Bitcoin 调用通过 provider 特定的 `id=1` 成功返回， +但 v3 错误地将其计为约束失败。不可变 `tasks/v4.jsonl` 已明确识别该映射，v4 的三次 Bitcoin trial +全部通过。 + +完整的[结果说明、revision、脱敏 JSONL 与生成汇总](../../benchmarks/discover-call/results/README.md) +均已公开。scorer fixture 仍只用于测试，不代表产品性能。 From f0d091bff6f2ecf1dcc7750c20d004f9c3281203 Mon Sep 17 00:00:00 2001 From: Linfang Wang Date: Thu, 23 Jul 2026 20:19:54 +0800 Subject: [PATCH 10/19] fix(benchmark): harden public baseline invariants --- .../discover-call/PUBLICATION_POLICY.md | 4 +- benchmarks/discover-call/README.md | 4 +- .../discover-call/fixtures/sample-runs.jsonl | 12 +- .../discover-call/publication-policy.json | 9 +- ...07-23-gpt-5.6-sol-configured-v4.runs.jsonl | 6 +- ...-23-gpt-5.6-sol-configured-v4.summary.json | 57 ++++- .../2026-07-23-gpt-5.6-sol-v2.runs.jsonl | 8 +- .../2026-07-23-gpt-5.6-sol-v2.summary.json | 224 +++++++++++------- .../2026-07-23-gpt-5.6-sol-v3.runs.jsonl | 4 +- .../2026-07-23-gpt-5.6-sol-v3.summary.json | 57 ++++- .../2026-07-23-oracle-v1-v3.runs.jsonl | 6 +- .../2026-07-23-oracle-v1-v3.summary.json | 57 ++++- .../2026-07-23-reference-v1-v4.runs.jsonl | 6 +- .../2026-07-23-reference-v1-v4.summary.json | 57 ++++- benchmarks/discover-call/results/README.md | 15 +- benchmarks/discover-call/src/publication.mjs | 116 +++++++-- benchmarks/discover-call/src/run.mjs | 46 +++- benchmarks/discover-call/src/scoring.mjs | 35 ++- benchmarks/discover-call/src/validate.mjs | 14 +- .../discover-call/test/adapter.test.mjs | 45 +++- .../discover-call/test/publication.test.mjs | 92 ++++++- .../discover-call/test/scoring.test.mjs | 32 ++- 22 files changed, 743 insertions(+), 163 deletions(-) diff --git a/benchmarks/discover-call/PUBLICATION_POLICY.md b/benchmarks/discover-call/PUBLICATION_POLICY.md index 0749124..dfce567 100644 --- a/benchmarks/discover-call/PUBLICATION_POLICY.md +++ b/benchmarks/discover-call/PUBLICATION_POLICY.md @@ -38,7 +38,9 @@ The publication command enforces `publication-policy.json`. It replaces the ordered discovery list with its count and SHA-256 digest and preserves grounded selection as an attestation. An approved selected tool keeps its ID; an unapproved selected tool is represented only by a SHA-256 digest, so a benchmark -run cannot silently expand the public catalog. +run cannot silently expand the public catalog. Metadata is projected through an +explicit allowlist, errors retain only normalized stage/reason codes, and +credential-shaped parameter keys are rejected case-insensitively. Provider authorization and namespace ownership must be confirmed before adding a tool ID to the public allowlist. Removing an approval requires regenerating all diff --git a/benchmarks/discover-call/README.md b/benchmarks/discover-call/README.md index 0133438..454d001 100644 --- a/benchmarks/discover-call/README.md +++ b/benchmarks/discover-call/README.md @@ -47,7 +47,9 @@ workflow-success denominator. The summary includes a deterministic 95% task-cluster bootstrap interval for workflow success. Tasks, rather than individual trials, are resampled so three trials of one task are not treated as three independent task draws. Dry runs -never count as workflow success. +never count as workflow success. A successful legacy call with no recorded +result-content observation is reported as unknown; it is never inferred to have +a non-empty result or included in a strict workflow point estimate. ## Task sets diff --git a/benchmarks/discover-call/fixtures/sample-runs.jsonl b/benchmarks/discover-call/fixtures/sample-runs.jsonl index 975c832..42015fc 100644 --- a/benchmarks/discover-call/fixtures/sample-runs.jsonl +++ b/benchmarks/discover-call/fixtures/sample-runs.jsonl @@ -1,6 +1,6 @@ -{"schema_version":1,"benchmark_version":"v1","run_id":"fixture-1","model":"fixture-model","task_id":"weather-london","trial":1,"status":"completed","discovery":{"result_tool_ids":["weather.forecast"]},"selection":{"tool_id":"weather.forecast"},"inspection":{"returned_tool_ids":["weather.forecast"],"required_parameters":["city"]},"parameters":{"city":"London"},"call":{"attempted":true,"success":true}} -{"schema_version":1,"benchmark_version":"v1","run_id":"fixture-2","model":"fixture-model","task_id":"stock-aapl","trial":1,"status":"completed","discovery":{"result_tool_ids":["market.quote"]},"selection":{"tool_id":"market.quote"},"inspection":{"returned_tool_ids":["market.quote"],"required_parameters":["symbol"]},"parameters":{"symbol":"AAPL"},"call":{"attempted":true,"success":true}} -{"schema_version":1,"benchmark_version":"v1","run_id":"fixture-3","model":"fixture-model","task_id":"exchange-usd-eur","trial":1,"status":"completed","discovery":{"result_tool_ids":["currency.convert"]},"selection":{"tool_id":"currency.convert"},"inspection":{"returned_tool_ids":["currency.convert"],"required_parameters":["base_currency","quote_currency"]},"parameters":{"base_currency":"USD","quote_currency":"EUR"},"call":{"attempted":true,"success":true}} -{"schema_version":1,"benchmark_version":"v1","run_id":"fixture-4","model":"fixture-model","task_id":"news-ai","trial":1,"status":"completed","discovery":{"result_tool_ids":["news.search"]},"selection":{"tool_id":"news.search"},"inspection":{"returned_tool_ids":["news.search"],"required_parameters":["query"]},"parameters":{"query":"artificial intelligence news"},"call":{"attempted":true,"success":true}} -{"schema_version":1,"benchmark_version":"v1","run_id":"fixture-5","model":"fixture-model","task_id":"papers-climate","trial":1,"status":"completed","discovery":{"result_tool_ids":["papers.search"]},"selection":{"tool_id":"papers.search"},"inspection":{"returned_tool_ids":["papers.search"],"required_parameters":["query"]},"parameters":{"query":"climate risk papers"},"call":{"attempted":true,"success":true}} -{"schema_version":1,"benchmark_version":"v1","run_id":"fixture-6","model":"fixture-model","task_id":"timezone-tokyo","trial":1,"status":"completed","discovery":{"result_tool_ids":["timezone.lookup"]},"selection":{"tool_id":"timezone.lookup"},"inspection":{"returned_tool_ids":["timezone.lookup"],"required_parameters":["city"]},"parameters":{"city":"Tokyo"},"call":{"attempted":true,"success":true}} +{"schema_version":1,"benchmark_version":"v1","run_id":"fixture-1","model":"fixture-model","task_id":"weather-london","trial":1,"status":"completed","discovery":{"result_tool_ids":["weather.forecast"]},"selection":{"tool_id":"weather.forecast"},"inspection":{"returned_tool_ids":["weather.forecast"],"required_parameters":["city"]},"parameters":{"city":"London"},"call":{"attempted":true,"success":true,"result_nonempty":true}} +{"schema_version":1,"benchmark_version":"v1","run_id":"fixture-2","model":"fixture-model","task_id":"stock-aapl","trial":1,"status":"completed","discovery":{"result_tool_ids":["market.quote"]},"selection":{"tool_id":"market.quote"},"inspection":{"returned_tool_ids":["market.quote"],"required_parameters":["symbol"]},"parameters":{"symbol":"AAPL"},"call":{"attempted":true,"success":true,"result_nonempty":true}} +{"schema_version":1,"benchmark_version":"v1","run_id":"fixture-3","model":"fixture-model","task_id":"exchange-usd-eur","trial":1,"status":"completed","discovery":{"result_tool_ids":["currency.convert"]},"selection":{"tool_id":"currency.convert"},"inspection":{"returned_tool_ids":["currency.convert"],"required_parameters":["base_currency","quote_currency"]},"parameters":{"base_currency":"USD","quote_currency":"EUR"},"call":{"attempted":true,"success":true,"result_nonempty":true}} +{"schema_version":1,"benchmark_version":"v1","run_id":"fixture-4","model":"fixture-model","task_id":"news-ai","trial":1,"status":"completed","discovery":{"result_tool_ids":["news.search"]},"selection":{"tool_id":"news.search"},"inspection":{"returned_tool_ids":["news.search"],"required_parameters":["query"]},"parameters":{"query":"artificial intelligence news"},"call":{"attempted":true,"success":true,"result_nonempty":true}} +{"schema_version":1,"benchmark_version":"v1","run_id":"fixture-5","model":"fixture-model","task_id":"papers-climate","trial":1,"status":"completed","discovery":{"result_tool_ids":["papers.search"]},"selection":{"tool_id":"papers.search"},"inspection":{"returned_tool_ids":["papers.search"],"required_parameters":["query"]},"parameters":{"query":"climate risk papers"},"call":{"attempted":true,"success":true,"result_nonempty":true}} +{"schema_version":1,"benchmark_version":"v1","run_id":"fixture-6","model":"fixture-model","task_id":"timezone-tokyo","trial":1,"status":"completed","discovery":{"result_tool_ids":["timezone.lookup"]},"selection":{"tool_id":"timezone.lookup"},"inspection":{"returned_tool_ids":["timezone.lookup"],"required_parameters":["city"]},"parameters":{"city":"Tokyo"},"call":{"attempted":true,"success":true,"result_nonempty":true}} diff --git a/benchmarks/discover-call/publication-policy.json b/benchmarks/discover-call/publication-policy.json index ba6c7f3..16b6dfc 100644 --- a/benchmarks/discover-call/publication-policy.json +++ b/benchmarks/discover-call/publication-policy.json @@ -43,6 +43,13 @@ "connection_id", "remaining_credits", "result_tool_ids", - "returned_tool_ids" + "returned_tool_ids", + "api_key", + "access_token", + "refresh_token", + "oauth_token", + "authorization", + "client_secret", + "password" ] } diff --git a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.runs.jsonl b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.runs.jsonl index f7410e2..34a2bfe 100644 --- a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.runs.jsonl +++ b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.runs.jsonl @@ -34,9 +34,9 @@ {"schema_version":1,"benchmark_version":"v1","run_id":"58c60360-c513-442d-b08e-f3c729fc5542","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":1,"started_at":"2026-07-23T11:35:00.421Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1fa372fa0e73a010285d8cae90eb1dc1b04ca995386b24d01d04998e09b2f09f","selection_grounded":true},"selection":{"tool_id":null,"tool_id_sha256":"aa938efb4e16c513e85cee2e773d507bc478cb8543b68333ab921785516c7a2e"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:35:22.330Z","source_record_sha256":"59258c6c137545703d2b6ee4fd34db1f67b130aac2b9ffcc15ff326bce2b74a9"} {"schema_version":1,"benchmark_version":"v1","run_id":"a23aaa73-481a-4b61-b883-c8280ddcc8e0","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":2,"started_at":"2026-07-23T11:35:22.330Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1fa372fa0e73a010285d8cae90eb1dc1b04ca995386b24d01d04998e09b2f09f","selection_grounded":true},"selection":{"tool_id":null,"tool_id_sha256":"aa938efb4e16c513e85cee2e773d507bc478cb8543b68333ab921785516c7a2e"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:35:43.366Z","source_record_sha256":"af69472b0970e418b33cb16e5e055559d872ad62dca122c8f09021ea0425e118"} {"schema_version":1,"benchmark_version":"v1","run_id":"57461645-9d30-4864-8a68-952d6d99a112","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":3,"started_at":"2026-07-23T11:35:43.366Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1fa372fa0e73a010285d8cae90eb1dc1b04ca995386b24d01d04998e09b2f09f","selection_grounded":true},"selection":{"tool_id":null,"tool_id_sha256":"aa938efb4e16c513e85cee2e773d507bc478cb8543b68333ab921785516c7a2e"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:36:08.708Z","source_record_sha256":"e54b101e094e8c07304b3e62a0d7f0c31ec354414df19fa337ad640f12d3977a"} -{"schema_version":1,"benchmark_version":"v1","run_id":"062ac004-ca1d-4938-a2ee-f56389cc5d8e","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T11:36:08.708Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"bcbd727391a8751121de802694245a7b97b9063d15176e9d3077129ca89c1f23","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected","message":"Adapter invocation failed"},"finished_at":"2026-07-23T11:36:48.187Z","source_record_sha256":"e1cc5dbe6957ce04839841829c7e6cb828c4a5c86ed5523f3f74021f6c56e65f"} -{"schema_version":1,"benchmark_version":"v1","run_id":"ea6ee15f-1a67-457d-bf48-e7045eaff0f6","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T11:36:48.187Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"bcbd727391a8751121de802694245a7b97b9063d15176e9d3077129ca89c1f23","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected","message":"Adapter invocation failed"},"finished_at":"2026-07-23T11:37:37.695Z","source_record_sha256":"d4cf42d286d5f51b1f00d85fdf776f606ff182d2187b06d000af45d5357557b9"} -{"schema_version":1,"benchmark_version":"v1","run_id":"13e9d4a9-c237-4cfe-8b66-2b4bc6727157","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T11:37:37.695Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"bcbd727391a8751121de802694245a7b97b9063d15176e9d3077129ca89c1f23","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected","message":"Adapter invocation failed"},"finished_at":"2026-07-23T11:38:18.107Z","source_record_sha256":"928570b21b09c2fd63f8ae7ee1faf5f9b3b0b66ce7de966f60ec6f95c0d98eb6"} +{"schema_version":1,"benchmark_version":"v1","run_id":"062ac004-ca1d-4938-a2ee-f56389cc5d8e","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T11:36:08.708Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"bcbd727391a8751121de802694245a7b97b9063d15176e9d3077129ca89c1f23","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected"},"finished_at":"2026-07-23T11:36:48.187Z","source_record_sha256":"e1cc5dbe6957ce04839841829c7e6cb828c4a5c86ed5523f3f74021f6c56e65f"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ea6ee15f-1a67-457d-bf48-e7045eaff0f6","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T11:36:48.187Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"bcbd727391a8751121de802694245a7b97b9063d15176e9d3077129ca89c1f23","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected"},"finished_at":"2026-07-23T11:37:37.695Z","source_record_sha256":"d4cf42d286d5f51b1f00d85fdf776f606ff182d2187b06d000af45d5357557b9"} +{"schema_version":1,"benchmark_version":"v1","run_id":"13e9d4a9-c237-4cfe-8b66-2b4bc6727157","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T11:37:37.695Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"bcbd727391a8751121de802694245a7b97b9063d15176e9d3077129ca89c1f23","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected"},"finished_at":"2026-07-23T11:38:18.107Z","source_record_sha256":"928570b21b09c2fd63f8ae7ee1faf5f9b3b0b66ce7de966f60ec6f95c0d98eb6"} {"schema_version":1,"benchmark_version":"v1","run_id":"46fa3b68-38ad-4728-9252-32e08dd52ee7","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":1,"started_at":"2026-07-23T11:38:18.107Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"406f82f0151e2aab38d28e5369d6e3b6b8f1e5a40414bd6ea7cafc3eb9f40259","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":10,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:38:46.770Z","source_record_sha256":"b0691ac1115ede5d59b2c49e6c7ad87c6c440e8ad911982666f36d32ffa00b5e"} {"schema_version":1,"benchmark_version":"v1","run_id":"19c84ab6-eab7-4ed1-b536-20d6db38801c","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":2,"started_at":"2026-07-23T11:38:46.770Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"406f82f0151e2aab38d28e5369d6e3b6b8f1e5a40414bd6ea7cafc3eb9f40259","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:39:13.722Z","source_record_sha256":"3326539c1c392a2b9f51ea323d1c669cbaa2134e4a71361b2a194a1bca2932d4"} {"schema_version":1,"benchmark_version":"v1","run_id":"d2438aee-241f-4be1-8c92-0a21375a39cd","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":3,"started_at":"2026-07-23T11:39:13.722Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"406f82f0151e2aab38d28e5369d6e3b6b8f1e5a40414bd6ea7cafc3eb9f40259","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:39:36.630Z","source_record_sha256":"57f8f030306de487ac1e7f3de74fbca58e026038c7612b4bf239b44e747a5322"} diff --git a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.summary.json b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.summary.json index 3e83934..a741e3e 100644 --- a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.summary.json +++ b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.summary.json @@ -1,7 +1,7 @@ { "schema_version": 1, "methodology": "discover-call-v1", - "generated_at": "2026-07-23T11:45:48.815Z", + "generated_at": "2026-07-23T12:14:22.053Z", "models": [ { "model": "gpt-5.6-sol", @@ -17,6 +17,7 @@ "constraint_accuracy": 0.8889, "call_success_rate": 0.8824, "result_nonempty_rate": 0.8824, + "pre_result_gate_workflow_success_rate": 0.7778, "workflow_success_rate": 0.7778, "workflow_success_task_cluster_bootstrap_95": [ 0.5556, @@ -44,6 +45,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -80,6 +82,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -116,6 +119,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -152,6 +156,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -188,6 +193,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -224,6 +230,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -260,6 +267,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -296,6 +304,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -332,6 +341,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -368,6 +378,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -404,6 +415,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -440,6 +452,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -476,6 +489,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -512,6 +526,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -548,6 +563,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -584,6 +600,7 @@ "constraint_accuracy": 0, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -620,6 +637,7 @@ "constraint_accuracy": 0, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -656,6 +674,7 @@ "constraint_accuracy": 0, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -692,6 +711,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -728,6 +748,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -764,6 +785,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -800,6 +822,7 @@ "constraint_accuracy": 1, "call_success": false, "result_nonempty": false, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -836,6 +859,7 @@ "constraint_accuracy": 1, "call_success": false, "result_nonempty": false, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -872,6 +896,7 @@ "constraint_accuracy": 1, "call_success": false, "result_nonempty": false, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -908,6 +933,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -944,6 +970,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -980,6 +1007,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1016,6 +1044,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1052,6 +1081,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1088,6 +1118,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1124,6 +1155,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1160,6 +1192,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1196,6 +1229,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1232,6 +1266,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1268,6 +1303,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1304,6 +1340,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1340,6 +1377,7 @@ "constraint_accuracy": 0, "call_success": null, "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": "adapter", "error_reason": "tool_use_rejected", @@ -1376,6 +1414,7 @@ "constraint_accuracy": 0, "call_success": null, "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": "adapter", "error_reason": "tool_use_rejected", @@ -1412,6 +1451,7 @@ "constraint_accuracy": 0, "call_success": null, "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": "adapter", "error_reason": "tool_use_rejected", @@ -1448,6 +1488,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1484,6 +1525,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1520,6 +1562,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1556,6 +1599,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1592,6 +1636,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1628,6 +1673,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1664,6 +1710,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1700,6 +1747,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1736,6 +1784,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1772,6 +1821,7 @@ "constraint_accuracy": 1, "call_success": false, "result_nonempty": false, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -1808,6 +1858,7 @@ "constraint_accuracy": 1, "call_success": false, "result_nonempty": false, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -1844,6 +1895,7 @@ "constraint_accuracy": 1, "call_success": false, "result_nonempty": false, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -1880,6 +1932,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1916,6 +1969,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1952,6 +2006,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, diff --git a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.runs.jsonl b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.runs.jsonl index a3a1fe3..9804606 100644 --- a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.runs.jsonl +++ b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.runs.jsonl @@ -34,9 +34,9 @@ {"schema_version":1,"benchmark_version":"v1","run_id":"70a47304-ae9e-4cb5-8331-5df6a8e83a43","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":1,"started_at":"2026-07-23T06:44:12.213Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e5da17c29aa94866e3abc93b7ae7169f66e151ae69893d952e89fcee78460d21","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:44:39.351Z","source_record_sha256":"3f8d52a2dd4bbf58e9f9be36f1e434b694eb757da69d8159f0243888a598eb4b"} {"schema_version":1,"benchmark_version":"v1","run_id":"73355630-bc92-4e0a-a1c8-4f25f895a949","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":2,"started_at":"2026-07-23T06:44:39.351Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e5da17c29aa94866e3abc93b7ae7169f66e151ae69893d952e89fcee78460d21","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:45:14.542Z","source_record_sha256":"3a19bbc3b8533c11da052cc57dac3f7c2d2f5f04a555da0668605d0369dd1b6e"} {"schema_version":1,"benchmark_version":"v1","run_id":"f05cea11-e23c-463a-98f1-e5c0b4f51e88","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":3,"started_at":"2026-07-23T06:45:14.542Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e5da17c29aa94866e3abc93b7ae7169f66e151ae69893d952e89fcee78460d21","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:45:40.767Z","source_record_sha256":"b1f8a12fd2910883acab983ca94f2d489b62427c0daa9759d3ed850c590cde0d"} -{"schema_version":1,"benchmark_version":"v1","run_id":"ae3e60aa-8ee9-4650-b3e2-6d906a42da8b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T06:45:40.767Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","message":"Adapter invocation failed"},"finished_at":"2026-07-23T06:46:30.257Z","source_record_sha256":"a662ab99b03038d4da9d8e89149c4d3180ec80a1ddba685c896089c14c660114"} -{"schema_version":1,"benchmark_version":"v1","run_id":"cc1eebe6-e68b-45ba-8f32-827f98baa240","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T06:46:30.257Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","message":"Adapter invocation failed"},"finished_at":"2026-07-23T06:47:52.318Z","source_record_sha256":"e8e64e193750c200d9940be5a99c552cad0ecb01e495121124aa07804da7b2d0"} -{"schema_version":1,"benchmark_version":"v1","run_id":"19382342-d937-46dc-ba4a-c71139081673","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T06:47:52.318Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","message":"Adapter invocation failed"},"finished_at":"2026-07-23T06:48:55.681Z","source_record_sha256":"487eeb948773c4df6e60d2333f6e54cfa05a7e143dbd87ea81953b8cfcf02f6f"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ae3e60aa-8ee9-4650-b3e2-6d906a42da8b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T06:45:40.767Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":null},"finished_at":"2026-07-23T06:46:30.257Z","source_record_sha256":"a662ab99b03038d4da9d8e89149c4d3180ec80a1ddba685c896089c14c660114"} +{"schema_version":1,"benchmark_version":"v1","run_id":"cc1eebe6-e68b-45ba-8f32-827f98baa240","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T06:46:30.257Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":null},"finished_at":"2026-07-23T06:47:52.318Z","source_record_sha256":"e8e64e193750c200d9940be5a99c552cad0ecb01e495121124aa07804da7b2d0"} +{"schema_version":1,"benchmark_version":"v1","run_id":"19382342-d937-46dc-ba4a-c71139081673","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T06:47:52.318Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":null},"finished_at":"2026-07-23T06:48:55.681Z","source_record_sha256":"487eeb948773c4df6e60d2333f6e54cfa05a7e143dbd87ea81953b8cfcf02f6f"} {"schema_version":1,"benchmark_version":"v1","run_id":"48e281c3-3a7a-4327-9911-427072bc6fcd","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":1,"started_at":"2026-07-23T06:48:55.681Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"47a10fc51eaffa7d634cfeb673e3977424960edb526cf36296eab022d673e074","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:49:28.588Z","source_record_sha256":"8429545504e4b2eef7082c3c1fce7c561bd1be1de082be34fd96e2c5149690c5"} {"schema_version":1,"benchmark_version":"v1","run_id":"d23e1fc9-03a5-4c76-8761-22c9de0e6231","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":2,"started_at":"2026-07-23T06:49:28.588Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"47a10fc51eaffa7d634cfeb673e3977424960edb526cf36296eab022d673e074","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":10,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:49:55.653Z","source_record_sha256":"7164bd4f310c9dda3a27e9ff046018d44184c359ec7c7f0db3880e16476eee65"} {"schema_version":1,"benchmark_version":"v1","run_id":"262b0d23-181c-4b1c-8241-8cfd530dbb96","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":3,"started_at":"2026-07-23T06:49:55.653Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"47a10fc51eaffa7d634cfeb673e3977424960edb526cf36296eab022d673e074","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:50:22.831Z","source_record_sha256":"8620cabf967667758d4370970f8e6d82026af991bf28e2b2dfbf7fc602d9c6b1"} @@ -45,7 +45,7 @@ {"schema_version":1,"benchmark_version":"v1","run_id":"0ab6d79a-6510-4b11-966c-7063b6646f73","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":3,"started_at":"2026-07-23T06:51:30.279Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5d74987403b7f6009ae5a6da3cdcfac27dc35aa8f8dc5c333e65725c97f0cd0a","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:51:58.663Z","source_record_sha256":"5c935b04a6e98677b01766c543ce5eea027cdf4b2f27d6f12677d78d1bbfa0a7"} {"schema_version":1,"benchmark_version":"v1","run_id":"245b91ee-988d-474f-8fd1-82da7c62183a","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-23T06:51:58.663Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:52:30.880Z","source_record_sha256":"d86e68f4637cad728737fb4b0e4283ba4766a64304df4ed5b0843bb5ffc8185a"} {"schema_version":1,"benchmark_version":"v1","run_id":"dc38691f-d8f6-40f8-9e77-9288c33a7e68","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-23T06:52:30.881Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:53:04.476Z","source_record_sha256":"c97ae4740814eb914f3859c0d16a16dec1298fa3ff5caf4ae386299734840da7"} -{"schema_version":1,"benchmark_version":"v1","run_id":"c1624700-5aba-4cfc-8a68-acabf3de024a","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T06:53:04.476Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","message":"Adapter invocation failed"},"finished_at":"2026-07-23T06:55:18.009Z","source_record_sha256":"d567ee1f534ced827b85bb21fc9c7c8b41325ea27130743f8e95d0396532f4bd"} +{"schema_version":1,"benchmark_version":"v1","run_id":"c1624700-5aba-4cfc-8a68-acabf3de024a","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T06:53:04.476Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":null},"finished_at":"2026-07-23T06:55:18.009Z","source_record_sha256":"d567ee1f534ced827b85bb21fc9c7c8b41325ea27130743f8e95d0396532f4bd"} {"schema_version":1,"benchmark_version":"v1","run_id":"0e485876-22f8-4265-91e0-51990c3644e9","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":1,"started_at":"2026-07-23T06:55:18.009Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fa45aa3f97145ca8594d0a5ff1f0fba0f1abb533dfb4cfabbf84a07e4a76fba8","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false},"finished_at":"2026-07-23T06:55:45.466Z","source_record_sha256":"e8d8d305c2052be04c186b5fb864515ed6c6066c552f64a7df60e6b7aec061a6"} {"schema_version":1,"benchmark_version":"v1","run_id":"ea693fd1-b596-40b1-bd16-eec213eecb12","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":2,"started_at":"2026-07-23T06:55:45.466Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fa45aa3f97145ca8594d0a5ff1f0fba0f1abb533dfb4cfabbf84a07e4a76fba8","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false},"finished_at":"2026-07-23T06:56:19.074Z","source_record_sha256":"0ebe2c1c3adbf481010ee766b90c35e838b2c50bfd738c866806459e727f03b5"} {"schema_version":1,"benchmark_version":"v1","run_id":"abf65b68-8d35-453f-89b6-d30447a432e7","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":3,"started_at":"2026-07-23T06:56:19.074Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fa45aa3f97145ca8594d0a5ff1f0fba0f1abb533dfb4cfabbf84a07e4a76fba8","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false},"finished_at":"2026-07-23T06:56:53.486Z","source_record_sha256":"83d6dc9000481394c87614ff3a0dff0fcf553b0b50f7e2f7b556a5f986c5db06"} diff --git a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.summary.json b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.summary.json index ade5af6..e6f0ef9 100644 --- a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.summary.json +++ b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.summary.json @@ -1,7 +1,7 @@ { "schema_version": 1, "methodology": "discover-call-v1", - "generated_at": "2026-07-23T11:10:34.155Z", + "generated_at": "2026-07-23T12:14:22.012Z", "models": [ { "model": "gpt-5.6-sol", @@ -16,12 +16,10 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0.7593, "call_success_rate": 0.88, - "result_nonempty_rate": 0.88, - "workflow_success_rate": 0.6481, - "workflow_success_task_cluster_bootstrap_95": [ - 0.4259, - 0.8519 - ], + "result_nonempty_rate": null, + "pre_result_gate_workflow_success_rate": 0.6481, + "workflow_success_rate": null, + "workflow_success_task_cluster_bootstrap_95": null, "failures_by_stage": { "adapter": 4 }, @@ -41,8 +39,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -77,8 +76,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -113,8 +113,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -149,8 +150,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -185,8 +187,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -221,8 +224,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -257,7 +261,8 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, - "result_nonempty": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -293,7 +298,8 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, - "result_nonempty": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -329,7 +335,8 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, - "result_nonempty": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -365,7 +372,8 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, - "result_nonempty": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -401,7 +409,8 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, - "result_nonempty": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -437,7 +446,8 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, - "result_nonempty": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -473,8 +483,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -509,8 +520,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -545,8 +557,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -581,7 +594,8 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, - "result_nonempty": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -617,7 +631,8 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, - "result_nonempty": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -653,7 +668,8 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 0, "call_success": true, - "result_nonempty": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -689,8 +705,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -725,8 +742,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -761,8 +779,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -798,6 +817,7 @@ "constraint_accuracy": 1, "call_success": false, "result_nonempty": false, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -834,6 +854,7 @@ "constraint_accuracy": 1, "call_success": false, "result_nonempty": false, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -870,6 +891,7 @@ "constraint_accuracy": 1, "call_success": false, "result_nonempty": false, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -905,8 +927,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -941,8 +964,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -977,8 +1001,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -1013,8 +1038,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -1049,8 +1075,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -1085,8 +1112,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -1121,8 +1149,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -1157,8 +1186,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -1193,8 +1223,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -1229,8 +1260,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -1265,8 +1297,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -1301,8 +1334,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -1338,6 +1372,7 @@ "constraint_accuracy": 0, "call_success": null, "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": "adapter", "error_reason": null, @@ -1374,6 +1409,7 @@ "constraint_accuracy": 0, "call_success": null, "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": "adapter", "error_reason": null, @@ -1410,6 +1446,7 @@ "constraint_accuracy": 0, "call_success": null, "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": "adapter", "error_reason": null, @@ -1445,8 +1482,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -1481,8 +1519,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -1517,8 +1556,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -1553,8 +1593,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -1589,8 +1630,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -1625,8 +1667,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -1661,8 +1704,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -1697,8 +1741,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -1734,6 +1779,7 @@ "constraint_accuracy": 0, "call_success": null, "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": "adapter", "error_reason": null, @@ -1770,6 +1816,7 @@ "constraint_accuracy": 1, "call_success": false, "result_nonempty": false, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -1806,6 +1853,7 @@ "constraint_accuracy": 1, "call_success": false, "result_nonempty": false, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -1842,6 +1890,7 @@ "constraint_accuracy": 1, "call_success": false, "result_nonempty": false, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -1877,8 +1926,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -1913,8 +1963,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { @@ -1949,8 +2000,9 @@ "required_parameter_accuracy": 1, "constraint_accuracy": 1, "call_success": true, - "result_nonempty": true, - "workflow_success": true, + "result_nonempty": null, + "pre_result_gate_workflow_success": true, + "workflow_success": null, "error_stage": null, "error_reason": null, "metadata": { diff --git a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.runs.jsonl b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.runs.jsonl index 7c27d0b..730278e 100644 --- a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.runs.jsonl +++ b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.runs.jsonl @@ -34,9 +34,9 @@ {"schema_version":1,"benchmark_version":"v1","run_id":"0e125c12-7a7b-4c87-9e64-54a8be79f774","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":1,"started_at":"2026-07-23T09:57:36.396Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8d10f08954c40da564662fec151d75440ec8391aefdf74cfc8e2c32a1ee26dbb","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:58:04.308Z","source_record_sha256":"40c87aa07d78b9b835112c13584563870d4b64eb7a23230de8085d7a344f1851"} {"schema_version":1,"benchmark_version":"v1","run_id":"7a2ca54b-b346-4343-b6fb-54a264c5b778","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":2,"started_at":"2026-07-23T09:58:04.308Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8d10f08954c40da564662fec151d75440ec8391aefdf74cfc8e2c32a1ee26dbb","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:58:29.894Z","source_record_sha256":"b0b04c429f3abe332a69253f6d6e831d1702da29097ab1285883cf4945269647"} {"schema_version":1,"benchmark_version":"v1","run_id":"e98dc7fd-0ed4-4487-a753-c6c197254603","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":3,"started_at":"2026-07-23T09:58:29.894Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8d10f08954c40da564662fec151d75440ec8391aefdf74cfc8e2c32a1ee26dbb","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:58:58.074Z","source_record_sha256":"4dc549878c034c8440e01e24b20ced68fd25617caef32d35519f5ee2750ec0d3"} -{"schema_version":1,"benchmark_version":"v1","run_id":"a22d10aa-0f23-4f76-b2a2-692576b4b2a2","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T09:58:58.074Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected","message":"Adapter invocation failed"},"finished_at":"2026-07-23T09:59:40.997Z","source_record_sha256":"8e0f1a8dd393b2e3e3f8480ede541b2effe403ed92acfa0e2fab9f4269ed9246"} +{"schema_version":1,"benchmark_version":"v1","run_id":"a22d10aa-0f23-4f76-b2a2-692576b4b2a2","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T09:58:58.074Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected"},"finished_at":"2026-07-23T09:59:40.997Z","source_record_sha256":"8e0f1a8dd393b2e3e3f8480ede541b2effe403ed92acfa0e2fab9f4269ed9246"} {"schema_version":1,"benchmark_version":"v1","run_id":"97361151-f794-478a-8b90-e8e544c484f7","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T09:59:40.997Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:00:15.027Z","source_record_sha256":"f9b49fcdc5f1e3d00fdb7185a218da04e37fc6e60da566fc3aa2e9073ab191a1"} -{"schema_version":1,"benchmark_version":"v1","run_id":"e877975e-2fff-4ec9-a487-0bddba738e4f","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T10:00:15.027Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected","message":"Adapter invocation failed"},"finished_at":"2026-07-23T10:00:55.710Z","source_record_sha256":"d6bd4b821637a43316f544a37fc975c0b59797e77cd24ef7fae03280ddaa3721"} +{"schema_version":1,"benchmark_version":"v1","run_id":"e877975e-2fff-4ec9-a487-0bddba738e4f","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T10:00:15.027Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected"},"finished_at":"2026-07-23T10:00:55.710Z","source_record_sha256":"d6bd4b821637a43316f544a37fc975c0b59797e77cd24ef7fae03280ddaa3721"} {"schema_version":1,"benchmark_version":"v1","run_id":"d9fe796d-cd3d-4d0c-8bb3-0cd1fc7e128d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":1,"started_at":"2026-07-23T10:00:55.710Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"406f82f0151e2aab38d28e5369d6e3b6b8f1e5a40414bd6ea7cafc3eb9f40259","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:01:21.601Z","source_record_sha256":"61bcf3bc4829f5e23fb25aa989ab9e25e8bdc1b9066d6e8e7dae7ec7d3fe39a4"} {"schema_version":1,"benchmark_version":"v1","run_id":"e3d0d1fd-e32c-4701-b87a-7d2907aae773","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":2,"started_at":"2026-07-23T10:01:21.601Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"406f82f0151e2aab38d28e5369d6e3b6b8f1e5a40414bd6ea7cafc3eb9f40259","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:01:46.789Z","source_record_sha256":"0c98fe2b9fab21c84a6018f2c17577e36c6f1db959d8959006e8a27a1d509c9d"} {"schema_version":1,"benchmark_version":"v1","run_id":"109d0ded-433f-4ef6-a7ed-f8cf19b897be","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":3,"started_at":"2026-07-23T10:01:46.789Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"413e70c9b613711904d82925006e89c88e507640e6650646084db3a76f64b8ce","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:02:17.125Z","source_record_sha256":"25f730ed07cea7449205d61271acef2544d40eae7a33b9a76bc2ed2d6eb53a7d"} diff --git a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.summary.json b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.summary.json index a4a61cd..8f3e7ea 100644 --- a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.summary.json +++ b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.summary.json @@ -1,7 +1,7 @@ { "schema_version": 1, "methodology": "discover-call-v1", - "generated_at": "2026-07-23T11:10:34.270Z", + "generated_at": "2026-07-23T12:14:22.020Z", "models": [ { "model": "gpt-5.6-sol", @@ -17,6 +17,7 @@ "constraint_accuracy": 0.8333, "call_success_rate": 0.8846, "result_nonempty_rate": 0.8846, + "pre_result_gate_workflow_success_rate": 0.7222, "workflow_success_rate": 0.7222, "workflow_success_task_cluster_bootstrap_95": [ 0.5, @@ -44,6 +45,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -80,6 +82,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -116,6 +119,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -152,6 +156,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -188,6 +193,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -224,6 +230,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -260,6 +267,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -296,6 +304,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -332,6 +341,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -368,6 +378,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -404,6 +415,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -440,6 +452,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -476,6 +489,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -512,6 +526,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -548,6 +563,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -584,6 +600,7 @@ "constraint_accuracy": 0, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -620,6 +637,7 @@ "constraint_accuracy": 0, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -656,6 +674,7 @@ "constraint_accuracy": 0, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -692,6 +711,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -728,6 +748,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -764,6 +785,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -800,6 +822,7 @@ "constraint_accuracy": 1, "call_success": false, "result_nonempty": false, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -836,6 +859,7 @@ "constraint_accuracy": 1, "call_success": false, "result_nonempty": false, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -872,6 +896,7 @@ "constraint_accuracy": 1, "call_success": false, "result_nonempty": false, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -908,6 +933,7 @@ "constraint_accuracy": 0, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -944,6 +970,7 @@ "constraint_accuracy": 0, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -980,6 +1007,7 @@ "constraint_accuracy": 0, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -1016,6 +1044,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1052,6 +1081,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1088,6 +1118,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1124,6 +1155,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1160,6 +1192,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1196,6 +1229,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1232,6 +1266,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1268,6 +1303,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1304,6 +1340,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1340,6 +1377,7 @@ "constraint_accuracy": 0, "call_success": null, "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": "adapter", "error_reason": "tool_use_rejected", @@ -1376,6 +1414,7 @@ "constraint_accuracy": 0, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -1412,6 +1451,7 @@ "constraint_accuracy": 0, "call_success": null, "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": "adapter", "error_reason": "tool_use_rejected", @@ -1448,6 +1488,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1484,6 +1525,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1520,6 +1562,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1556,6 +1599,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1592,6 +1636,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1628,6 +1673,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1664,6 +1710,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1700,6 +1747,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1736,6 +1784,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1772,6 +1821,7 @@ "constraint_accuracy": 1, "call_success": false, "result_nonempty": false, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -1808,6 +1858,7 @@ "constraint_accuracy": 1, "call_success": false, "result_nonempty": false, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -1844,6 +1895,7 @@ "constraint_accuracy": 1, "call_success": false, "result_nonempty": false, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": null, "error_reason": null, @@ -1880,6 +1932,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1916,6 +1969,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1952,6 +2006,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, diff --git a/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.runs.jsonl b/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.runs.jsonl index 209c806..0f518d2 100644 --- a/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.runs.jsonl +++ b/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.runs.jsonl @@ -13,9 +13,9 @@ {"schema_version":1,"benchmark_version":"v1","run_id":"7ddaaf21-9eb5-4cbf-9e9e-ad0cf2b4da30","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T07:53:26.275Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:30.277Z","source_record_sha256":"7f876ab3626ab04941230d16c52d7ad76831f5c7eca25e27ced0795f330b21b5"} {"schema_version":1,"benchmark_version":"v1","run_id":"aa7da099-31bf-4f14-94b5-04b148b81bde","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T07:53:30.278Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:34.089Z","source_record_sha256":"14497b8df3945d1204c9f850aff21659fed945adf62f0282570a7fd03b000b8d"} {"schema_version":1,"benchmark_version":"v1","run_id":"b42f18f4-c0fe-443f-9d07-cb4870e88af6","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T07:53:34.089Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:37.784Z","source_record_sha256":"6671f9b91d3b83f7ad972c4764083b7aeedfbcc2e2f0483c454d8d2b9d90e2b0"} -{"schema_version":1,"benchmark_version":"v1","run_id":"ab798371-3cf3-4945-87c6-4240b8b9e36f","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T07:53:37.784Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"97d9f951a9f193786150828de15857112d2439f3832ea0b92efcd5108ada92af","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id","message":"Adapter did not return a non-empty tool_id"},"finished_at":"2026-07-23T07:53:39.282Z","source_record_sha256":"334143d7288f3d7bd9e0e1542e391875145a4f8d296ac3deab16320d4de545ee"} -{"schema_version":1,"benchmark_version":"v1","run_id":"3879f2c1-7e03-4819-b2b4-4dc353c9b6b6","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T07:53:39.282Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"97d9f951a9f193786150828de15857112d2439f3832ea0b92efcd5108ada92af","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id","message":"Adapter did not return a non-empty tool_id"},"finished_at":"2026-07-23T07:53:41.087Z","source_record_sha256":"664918ea59c543cacbe4d218fc05fbd56995b603c5981355eb040dd1b64da7af"} -{"schema_version":1,"benchmark_version":"v1","run_id":"cb7160d0-c6ea-4e8b-a60e-bffc3e6747df","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T07:53:41.087Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"97d9f951a9f193786150828de15857112d2439f3832ea0b92efcd5108ada92af","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id","message":"Adapter did not return a non-empty tool_id"},"finished_at":"2026-07-23T07:53:42.486Z","source_record_sha256":"f020400865f1d1eee9552674edd440cdb237b26d9dbe01217fd1704109c7bc5c"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ab798371-3cf3-4945-87c6-4240b8b9e36f","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T07:53:37.784Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"97d9f951a9f193786150828de15857112d2439f3832ea0b92efcd5108ada92af","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id"},"finished_at":"2026-07-23T07:53:39.282Z","source_record_sha256":"334143d7288f3d7bd9e0e1542e391875145a4f8d296ac3deab16320d4de545ee"} +{"schema_version":1,"benchmark_version":"v1","run_id":"3879f2c1-7e03-4819-b2b4-4dc353c9b6b6","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T07:53:39.282Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"97d9f951a9f193786150828de15857112d2439f3832ea0b92efcd5108ada92af","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id"},"finished_at":"2026-07-23T07:53:41.087Z","source_record_sha256":"664918ea59c543cacbe4d218fc05fbd56995b603c5981355eb040dd1b64da7af"} +{"schema_version":1,"benchmark_version":"v1","run_id":"cb7160d0-c6ea-4e8b-a60e-bffc3e6747df","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T07:53:41.087Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"97d9f951a9f193786150828de15857112d2439f3832ea0b92efcd5108ada92af","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id"},"finished_at":"2026-07-23T07:53:42.486Z","source_record_sha256":"f020400865f1d1eee9552674edd440cdb237b26d9dbe01217fd1704109c7bc5c"} {"schema_version":1,"benchmark_version":"v1","run_id":"c004662d-e17c-441a-8439-d18a4a1988c5","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T07:53:42.486Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f3a89453c750d6ddfee82c21bfd021c18cbefd53654879c10f93d3b6200c7d13","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:46.409Z","source_record_sha256":"81cc941960ad0e0e7d1ca50a6703be7197c26fa3190096771441100819fb438b"} {"schema_version":1,"benchmark_version":"v1","run_id":"aa9437b4-4a68-4467-9195-e421a33d242d","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T07:53:46.409Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f3a89453c750d6ddfee82c21bfd021c18cbefd53654879c10f93d3b6200c7d13","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:51.677Z","source_record_sha256":"98c2f8f11295550da5dc09caaeb1cbba0e058a4e0672402acb9052882f0162ce"} {"schema_version":1,"benchmark_version":"v1","run_id":"c3632840-0ea8-46f3-9c0b-d721c14211ed","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T07:53:51.677Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f3a89453c750d6ddfee82c21bfd021c18cbefd53654879c10f93d3b6200c7d13","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:55.919Z","source_record_sha256":"07899db418e2d3ff161a1468af71d0d8d9f0d5ecb0e3aaeda74c8501b5c858a3"} diff --git a/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.summary.json b/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.summary.json index 4d72a09..b191d8b 100644 --- a/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.summary.json +++ b/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.summary.json @@ -1,7 +1,7 @@ { "schema_version": 1, "methodology": "discover-call-v1", - "generated_at": "2026-07-23T11:10:34.385Z", + "generated_at": "2026-07-23T12:14:22.036Z", "models": [ { "model": "oracle-v1", @@ -17,6 +17,7 @@ "constraint_accuracy": 0.9444, "call_success_rate": 1, "result_nonempty_rate": 1, + "pre_result_gate_workflow_success_rate": 0.9444, "workflow_success_rate": 0.9444, "workflow_success_task_cluster_bootstrap_95": [ 0.8333, @@ -44,6 +45,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -80,6 +82,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -116,6 +119,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -152,6 +156,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -188,6 +193,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -224,6 +230,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -260,6 +267,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -296,6 +304,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -332,6 +341,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -368,6 +378,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -404,6 +415,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -440,6 +452,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -476,6 +489,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -512,6 +526,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -548,6 +563,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -584,6 +600,7 @@ "constraint_accuracy": 0, "call_success": null, "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": "select", "error_reason": "missing_tool_id", @@ -620,6 +637,7 @@ "constraint_accuracy": 0, "call_success": null, "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": "select", "error_reason": "missing_tool_id", @@ -656,6 +674,7 @@ "constraint_accuracy": 0, "call_success": null, "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": "select", "error_reason": "missing_tool_id", @@ -692,6 +711,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -728,6 +748,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -764,6 +785,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -800,6 +822,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -836,6 +859,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -872,6 +896,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -908,6 +933,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -944,6 +970,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -980,6 +1007,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1016,6 +1044,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1052,6 +1081,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1088,6 +1118,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1124,6 +1155,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1160,6 +1192,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1196,6 +1229,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1232,6 +1266,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1268,6 +1303,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1304,6 +1340,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1340,6 +1377,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1376,6 +1414,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1412,6 +1451,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1448,6 +1488,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1484,6 +1525,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1520,6 +1562,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1556,6 +1599,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1592,6 +1636,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1628,6 +1673,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1664,6 +1710,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1700,6 +1747,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1736,6 +1784,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1772,6 +1821,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1808,6 +1858,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1844,6 +1895,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1880,6 +1932,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1916,6 +1969,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1952,6 +2006,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, diff --git a/benchmarks/discover-call/results/2026-07-23-reference-v1-v4.runs.jsonl b/benchmarks/discover-call/results/2026-07-23-reference-v1-v4.runs.jsonl index b31cc81..208067e 100644 --- a/benchmarks/discover-call/results/2026-07-23-reference-v1-v4.runs.jsonl +++ b/benchmarks/discover-call/results/2026-07-23-reference-v1-v4.runs.jsonl @@ -13,9 +13,9 @@ {"schema_version":1,"benchmark_version":"v1","run_id":"4f315c20-69d3-4932-b9fe-9ac53abb26b3","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T11:49:37.192Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:41.059Z","source_record_sha256":"9f789afec504d169e789dba41d995393630dde1699d7f443486e5bfd7c363ff7"} {"schema_version":1,"benchmark_version":"v1","run_id":"994aa1d9-989a-4cbb-bb65-0a6748264e2c","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T11:49:41.059Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:45.414Z","source_record_sha256":"843c439d6403310a6b963c590a9486f153dd92c6433ee058a8586a5da3acb57a"} {"schema_version":1,"benchmark_version":"v1","run_id":"1e562721-df0e-401f-afb0-97173f7723bb","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T11:49:45.414Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:49.536Z","source_record_sha256":"3c03fe59e1241ad2a33f7b06102a6a57b4717775752fdda2877901831cb423f4"} -{"schema_version":1,"benchmark_version":"v1","run_id":"dabf80ee-36bc-405a-94cd-674a3a026f48","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T11:49:49.536Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"f4605d6b4c45644fc9cec341c593f461cdef28806d63c58187e65c1311fd88b9","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id","message":"Adapter did not return a non-empty tool_id"},"finished_at":"2026-07-23T11:49:51.286Z","source_record_sha256":"ca3514c8b130287b1fdeda4d4059b03e09dbf2dbb5e7b7c92a629ab4868e5cef"} -{"schema_version":1,"benchmark_version":"v1","run_id":"cc6c272d-971c-45da-b3f0-0a980620a770","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T11:49:51.286Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"f4605d6b4c45644fc9cec341c593f461cdef28806d63c58187e65c1311fd88b9","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id","message":"Adapter did not return a non-empty tool_id"},"finished_at":"2026-07-23T11:49:53.649Z","source_record_sha256":"508f234db9002ea4f1a4f542cf03a5ceecc68ef5b2220f3a1aac0afbf7f735d0"} -{"schema_version":1,"benchmark_version":"v1","run_id":"93cd2a79-0d68-4912-8b80-281ff8b5afcf","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T11:49:53.649Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"f4605d6b4c45644fc9cec341c593f461cdef28806d63c58187e65c1311fd88b9","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id","message":"Adapter did not return a non-empty tool_id"},"finished_at":"2026-07-23T11:49:55.417Z","source_record_sha256":"06831aa8ae3762571f2cff2bf7cfb286373e6af28ed4fcf7792799f26a66b91c"} +{"schema_version":1,"benchmark_version":"v1","run_id":"dabf80ee-36bc-405a-94cd-674a3a026f48","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T11:49:49.536Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"f4605d6b4c45644fc9cec341c593f461cdef28806d63c58187e65c1311fd88b9","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id"},"finished_at":"2026-07-23T11:49:51.286Z","source_record_sha256":"ca3514c8b130287b1fdeda4d4059b03e09dbf2dbb5e7b7c92a629ab4868e5cef"} +{"schema_version":1,"benchmark_version":"v1","run_id":"cc6c272d-971c-45da-b3f0-0a980620a770","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T11:49:51.286Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"f4605d6b4c45644fc9cec341c593f461cdef28806d63c58187e65c1311fd88b9","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id"},"finished_at":"2026-07-23T11:49:53.649Z","source_record_sha256":"508f234db9002ea4f1a4f542cf03a5ceecc68ef5b2220f3a1aac0afbf7f735d0"} +{"schema_version":1,"benchmark_version":"v1","run_id":"93cd2a79-0d68-4912-8b80-281ff8b5afcf","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T11:49:53.649Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"f4605d6b4c45644fc9cec341c593f461cdef28806d63c58187e65c1311fd88b9","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id"},"finished_at":"2026-07-23T11:49:55.417Z","source_record_sha256":"06831aa8ae3762571f2cff2bf7cfb286373e6af28ed4fcf7792799f26a66b91c"} {"schema_version":1,"benchmark_version":"v1","run_id":"fe761783-51bd-469f-a93d-e70b02c5aca4","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T11:49:55.417Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"70aa2fe386fa3968550e5521c201c217ff7c531c47da064ac80c7028098b0fc1","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:58.783Z","source_record_sha256":"c4b1f95c00f2162eb2ae0ea1a1c18d32bfff43c1aa66c0e090ef29801ca59674"} {"schema_version":1,"benchmark_version":"v1","run_id":"e066fabb-8e6a-43f1-8de1-118b9b2685d6","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T11:49:58.783Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"70aa2fe386fa3968550e5521c201c217ff7c531c47da064ac80c7028098b0fc1","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:02.312Z","source_record_sha256":"8e8f70b308a50ad0586d20cdae1b4600f8e023f8c311df4a60bcfdc364ee9f8b"} {"schema_version":1,"benchmark_version":"v1","run_id":"38a5dda0-7976-47cf-8360-8de4b6616f38","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T11:50:02.312Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"70aa2fe386fa3968550e5521c201c217ff7c531c47da064ac80c7028098b0fc1","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:05.390Z","source_record_sha256":"80804429ff13575a752328940d66e93c675d37de2fb9a20abe083ac6a063f5f2"} diff --git a/benchmarks/discover-call/results/2026-07-23-reference-v1-v4.summary.json b/benchmarks/discover-call/results/2026-07-23-reference-v1-v4.summary.json index 723194b..9ec48e1 100644 --- a/benchmarks/discover-call/results/2026-07-23-reference-v1-v4.summary.json +++ b/benchmarks/discover-call/results/2026-07-23-reference-v1-v4.summary.json @@ -1,7 +1,7 @@ { "schema_version": 1, "methodology": "discover-call-v1", - "generated_at": "2026-07-23T11:52:09.856Z", + "generated_at": "2026-07-23T12:14:22.009Z", "models": [ { "model": "reference-v1", @@ -17,6 +17,7 @@ "constraint_accuracy": 0.9444, "call_success_rate": 1, "result_nonempty_rate": 1, + "pre_result_gate_workflow_success_rate": 0.9444, "workflow_success_rate": 0.9444, "workflow_success_task_cluster_bootstrap_95": [ 0.8333, @@ -44,6 +45,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -80,6 +82,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -116,6 +119,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -152,6 +156,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -188,6 +193,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -224,6 +230,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -260,6 +267,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -296,6 +304,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -332,6 +341,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -368,6 +378,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -404,6 +415,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -440,6 +452,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -476,6 +489,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -512,6 +526,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -548,6 +563,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -584,6 +600,7 @@ "constraint_accuracy": 0, "call_success": null, "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": "select", "error_reason": "missing_tool_id", @@ -620,6 +637,7 @@ "constraint_accuracy": 0, "call_success": null, "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": "select", "error_reason": "missing_tool_id", @@ -656,6 +674,7 @@ "constraint_accuracy": 0, "call_success": null, "result_nonempty": null, + "pre_result_gate_workflow_success": false, "workflow_success": false, "error_stage": "select", "error_reason": "missing_tool_id", @@ -692,6 +711,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -728,6 +748,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -764,6 +785,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -800,6 +822,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -836,6 +859,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -872,6 +896,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -908,6 +933,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -944,6 +970,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -980,6 +1007,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1016,6 +1044,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1052,6 +1081,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1088,6 +1118,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1124,6 +1155,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1160,6 +1192,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1196,6 +1229,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1232,6 +1266,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1268,6 +1303,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1304,6 +1340,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1340,6 +1377,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1376,6 +1414,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1412,6 +1451,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1448,6 +1488,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1484,6 +1525,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1520,6 +1562,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1556,6 +1599,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1592,6 +1636,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1628,6 +1673,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1664,6 +1710,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1700,6 +1747,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1736,6 +1784,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1772,6 +1821,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1808,6 +1858,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1844,6 +1895,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1880,6 +1932,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1916,6 +1969,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, @@ -1952,6 +2006,7 @@ "constraint_accuracy": 1, "call_success": true, "result_nonempty": true, + "pre_result_gate_workflow_success": true, "workflow_success": true, "error_stage": null, "error_reason": null, diff --git a/benchmarks/discover-call/results/README.md b/benchmarks/discover-call/results/README.md index db03992..797dce9 100644 --- a/benchmarks/discover-call/results/README.md +++ b/benchmarks/discover-call/results/README.md @@ -163,8 +163,10 @@ No failed trial was removed or selectively rerun. | Required-parameter accuracy | 100% | | Constraint accuracy | 75.93% | | Call success among attempted calls | 88.00% (44 / 50) | -| Strict workflow success | 64.81% (35 / 54) | -| Workflow success, 95% task-cluster bootstrap | 42.59%–85.19% | +| Historical pre-result-gate workflow success | 64.81% (35 / 54) | +| Result non-empty | Unreported | +| Current strict workflow success | Not reportable | +| Current strict workflow interval | Not reportable | Reproducibility metadata: @@ -190,9 +192,12 @@ Artifacts: ### Interpretation The component metrics have different denominators. Call success is 44 of the 50 -attempted calls; strict workflow success is 35 of all 54 trials and additionally -requires grounded selection and inspection, complete required parameters, and -all task constraints. +attempted calls. The original v2 scorer counted 35 of all 54 trials after +grounded selection and inspection, complete required parameters, task +constraints, and call success. That historical result predates result-content +observation, so the 44 successful calls cannot be assumed to have non-empty +results. The current scorer reports the pre-result-gate rate separately and +leaves strict workflow success and its interval unavailable. The four adapter-stage failures were retained: all three domain-intelligence trials selected a catalog capability whose inspected contract exposed no usable diff --git a/benchmarks/discover-call/src/publication.mjs b/benchmarks/discover-call/src/publication.mjs index 2e3210c..467bfff 100644 --- a/benchmarks/discover-call/src/publication.mjs +++ b/benchmarks/discover-call/src/publication.mjs @@ -1,5 +1,37 @@ import { createHash } from 'node:crypto'; +const PUBLIC_METADATA_FIELDS = new Set([ + 'lane', + 'model_revision', + 'adapter_revision', + 'toolkit_revision', + 'task_set_sha256', + 'api_base_url', + 'api_revision', + 'catalog_revision', + 'catalog_observation_sha256', + 'runtime', + 'discovery_limit', + 'execute', + 'publication_policy', +]); + +const REQUIRED_FORBIDDEN_FIELDS = [ + 'execution_id', + 'search_id', + 'connection_id', + 'remaining_credits', + 'result_tool_ids', + 'returned_tool_ids', + 'api_key', + 'access_token', + 'refresh_token', + 'oauth_token', + 'authorization', + 'client_secret', + 'password', +]; + export function sanitizePublicRecords(records, policy) { validatePolicy(policy); const approvedToolIds = new Set(policy.approved_selected_tool_ids); @@ -69,11 +101,11 @@ export function sanitizePublicRecord( }, parameters: policy.publish_parameters === true ? record.parameters : null, call, - ...(record.error ? { error: record.error } : {}), + ...(record.error ? { error: publicError(record.error) } : {}), finished_at: record.finished_at, source_record_sha256: sha256(JSON.stringify(record)), }; - assertNoForbiddenFields(sanitized, new Set(policy.forbidden_fields)); + assertNoForbiddenFields(sanitized, forbiddenFieldSet(policy)); return sanitized; } @@ -93,8 +125,17 @@ export function validatePolicy(policy) { ) { throw new Error('Publication policy needs approved_selected_tool_ids'); } - if (!Array.isArray(policy.forbidden_fields) || !policy.forbidden_fields.includes('execution_id')) { - throw new Error('Publication policy must forbid execution_id'); + if (!Array.isArray(policy.forbidden_fields)) { + throw new Error('Publication policy needs forbidden_fields'); + } + const forbidden = forbiddenFieldSet(policy); + const missingForbidden = REQUIRED_FORBIDDEN_FIELDS.filter( + (field) => !forbidden.has(normalizeFieldName(field)), + ); + if (missingForbidden.length > 0) { + throw new Error( + `Publication policy must forbid protected fields: ${missingForbidden.join(', ')}`, + ); } if ( policy.legacy_lane_rewrites !== undefined && @@ -111,43 +152,54 @@ export function validatePolicy(policy) { export function validatePublicRecords(records, policy) { validatePolicy(policy); const approvedToolIds = new Set(policy.approved_selected_tool_ids); - const forbidden = new Set(policy.forbidden_fields); + const forbidden = forbiddenFieldSet(policy); for (const record of records) { assertNoForbiddenFields(record, forbidden); if (record.metadata?.publication_policy !== policy.policy_id) { throw new Error(`Public artifact is missing publication policy ${policy.policy_id}`); } + for (const field of Object.keys(record.metadata ?? {})) { + if (!PUBLIC_METADATA_FIELDS.has(field)) { + throw new Error(`Public artifact contains unsupported metadata: ${field}`); + } + } const selectedToolId = record.selection?.tool_id; if (selectedToolId && !approvedToolIds.has(selectedToolId)) { throw new Error(`Public artifact contains an unapproved selected tool: ${selectedToolId}`); } if ( record.selection?.tool_id_sha256 !== undefined && - (typeof record.selection.tool_id_sha256 !== 'string' || - record.selection.tool_id_sha256.length !== 64 || - selectedToolId) + (!isSha256(record.selection.tool_id_sha256) || selectedToolId) ) { throw new Error('Public artifact selected tool digest is invalid'); } if ( !Number.isInteger(record.discovery?.result_count) || - typeof record.discovery?.snapshot_sha256 !== 'string' || - record.discovery.snapshot_sha256.length !== 64 + !isSha256(record.discovery?.snapshot_sha256) ) { throw new Error('Public artifact discovery needs a count and SHA-256 snapshot'); } - if (typeof record.source_record_sha256 !== 'string' || record.source_record_sha256.length !== 64) { + if (!isSha256(record.source_record_sha256)) { throw new Error('Public artifact needs source_record_sha256'); } + if (record.error) { + const errorFields = Object.keys(record.error); + if (errorFields.some((field) => !['stage', 'reason_code'].includes(field))) { + throw new Error('Public artifact error contains unsupported fields'); + } + } } } function publicMetadata(metadata = {}, policy, catalogObservationSha256) { const lane = policy.legacy_lane_rewrites?.[metadata.lane] ?? metadata.lane ?? 'model'; return { - ...metadata, lane, model_revision: metadata.model_revision ?? 'unreported', + adapter_revision: metadata.adapter_revision ?? 'unreported', + toolkit_revision: metadata.toolkit_revision ?? 'unreported', + task_set_sha256: metadata.task_set_sha256 ?? 'unreported', + api_base_url: metadata.api_base_url ?? 'unreported', api_revision: metadata.api_revision ?? 'unreported', catalog_revision: metadata.catalog_revision ?? 'unreported', catalog_observation_sha256: catalogObservationSha256, @@ -156,17 +208,18 @@ function publicMetadata(metadata = {}, policy, catalogObservationSha256) { platform: metadata.runtime?.platform ?? 'unreported', arch: metadata.runtime?.arch ?? 'unreported', }, + discovery_limit: metadata.discovery_limit ?? 'unreported', + execute: metadata.execute === true, publication_policy: policy.policy_id, }; } function commonMetadataValue(records, field) { - const values = new Set( - records - .map((record) => record.metadata?.[field]) - .filter((value) => typeof value === 'string' && value && value !== 'pending'), - ); - return values.size === 1 ? [...values][0] : null; + if (records.length === 0) return null; + const values = records.map((record) => record.metadata?.[field]); + if (values.some((value) => !isSha256(value))) return null; + const unique = new Set(values); + return unique.size === 1 ? values[0] : null; } function assertNoForbiddenFields(value, forbidden, path = '$') { @@ -176,15 +229,40 @@ function assertNoForbiddenFields(value, forbidden, path = '$') { } if (!value || typeof value !== 'object') return; for (const [key, child] of Object.entries(value)) { - if (forbidden.has(key)) throw new Error(`Forbidden public artifact field at ${path}.${key}`); + if (forbidden.has(normalizeFieldName(key))) { + throw new Error(`Forbidden public artifact field at ${path}.${key}`); + } assertNoForbiddenFields(child, forbidden, `${path}.${key}`); } } +function publicError(error) { + return { + stage: safeCode(error?.stage), + reason_code: safeCode(error?.reason_code), + }; +} + +function safeCode(value) { + return typeof value === 'string' && /^[a-z][a-z0-9_]{1,63}$/.test(value) ? value : null; +} + +function normalizeFieldName(value) { + return value.toLowerCase().replace(/[^a-z0-9]/g, ''); +} + +function forbiddenFieldSet(policy) { + return new Set(policy.forbidden_fields.map(normalizeFieldName)); +} + function sha256(value) { return createHash('sha256').update(value).digest('hex'); } +function isSha256(value) { + return typeof value === 'string' && /^[a-f0-9]{64}$/.test(value); +} + function array(value) { return Array.isArray(value) ? value : []; } diff --git a/benchmarks/discover-call/src/run.mjs b/benchmarks/discover-call/src/run.mjs index fc4136a..be31b79 100644 --- a/benchmarks/discover-call/src/run.mjs +++ b/benchmarks/discover-call/src/run.mjs @@ -70,8 +70,16 @@ export async function main(argv = process.argv.slice(2)) { process.stdout.write(`Wrote ${records.length} benchmark records to ${output}\n`); } -export function createProcessAdapter({ command, args = [], timeoutMs = 120_000 }) { +export function createProcessAdapter({ + command, + args = [], + timeoutMs = 120_000, + forceKillAfterMs = 1_000, +}) { if (typeof command !== 'string' || !command) throw new Error('--adapter is required'); + if (!Number.isInteger(forceKillAfterMs) || forceKillAfterMs < 1 || forceKillAfterMs > 10_000) { + throw new Error('forceKillAfterMs must be an integer from 1 to 10000'); + } const adapterEnv = { ...process.env }; delete adapterEnv.QVERIS_API_KEY; return (payload) => @@ -86,33 +94,51 @@ export function createProcessAdapter({ command, args = [], timeoutMs = 120_000 } let stderr = ''; let stdoutExceeded = false; let stderrExceeded = false; - const timer = setTimeout(() => { + let settled = false; + let forceKillTimer; + const cleanup = () => { + clearTimeout(timer); + if (forceKillTimer) clearTimeout(forceKillTimer); + }; + const rejectOnce = (error) => { + if (settled) return; + settled = true; + reject(error); + }; + const terminate = (error) => { + if (settled) return; + clearTimeout(timer); child.kill('SIGTERM'); - reject(adapterError('Adapter timed out', 'timeout')); + forceKillTimer = setTimeout(() => child.kill('SIGKILL'), forceKillAfterMs); + forceKillTimer.unref(); + rejectOnce(error); + }; + const timer = setTimeout(() => { + terminate(adapterError('Adapter timed out', 'timeout')); }, timeoutMs); child.stdout.on('data', (chunk) => { stdout += chunk; if (stdout.length > 1_000_000 && !stdoutExceeded) { stdoutExceeded = true; - child.kill('SIGTERM'); + terminate(adapterError('Adapter output exceeded the limit', 'stdout_limit')); } }); child.stderr.on('data', (chunk) => { stderr += chunk; if (stderr.length > 100_000 && !stderrExceeded) { stderrExceeded = true; - child.kill('SIGTERM'); + terminate(adapterError('Adapter error output exceeded the limit', 'stderr_limit')); } }); child.on('error', () => { - clearTimeout(timer); - reject(adapterError('Adapter could not be started', 'start_failed')); + cleanup(); + rejectOnce(adapterError('Adapter could not be started', 'start_failed')); }); child.on('close', (code) => { - clearTimeout(timer); - if (stdoutExceeded) return reject(adapterError('Adapter output exceeded the limit', 'stdout_limit')); - if (stderrExceeded) return reject(adapterError('Adapter error output exceeded the limit', 'stderr_limit')); + cleanup(); + if (settled) return; + settled = true; if (code !== 0) { return reject( adapterError('Adapter exited unsuccessfully', adapterReasonFromStderr(stderr) || 'process_exit'), diff --git a/benchmarks/discover-call/src/scoring.mjs b/benchmarks/discover-call/src/scoring.mjs index 4c4843c..2adef01 100644 --- a/benchmarks/discover-call/src/scoring.mjs +++ b/benchmarks/discover-call/src/scoring.mjs @@ -103,18 +103,22 @@ export function scoreRecord(task, record) { const callSuccess = executed ? record.call?.success === true : null; const resultNonempty = executed && callSuccess - ? record.call?.result_nonempty ?? record.call?.result_valid ?? true + ? booleanOrNull(record.call?.result_nonempty ?? record.call?.result_valid) : executed ? false : null; - const workflowSuccess = + const preResultGateWorkflowSuccess = completed && selectionGrounded && inspectionGrounded && requiredParameterAccuracy === 1 && constraintAccuracy === 1 && - callSuccess === true && - resultNonempty === true; + callSuccess === true; + const workflowSuccess = preResultGateWorkflowSuccess + ? resultNonempty === null + ? null + : resultNonempty === true + : false; return { run_id: record.run_id, @@ -129,6 +133,7 @@ export function scoreRecord(task, record) { constraint_accuracy: round(constraintAccuracy), call_success: callSuccess, result_nonempty: resultNonempty, + pre_result_gate_workflow_success: preResultGateWorkflowSuccess, workflow_success: workflowSuccess, error_stage: record.error?.stage ?? null, error_reason: record.error?.reason_code ?? null, @@ -138,8 +143,10 @@ export function scoreRecord(task, record) { function aggregateModel(model, records) { const executed = records.filter((record) => record.executed); - const workflowWins = records.filter((record) => record.workflow_success).length; - const interval = taskClusterBootstrapInterval(records); + const resultEvidenceComplete = executed.every((record) => record.result_nonempty !== null); + const workflowEvidenceComplete = records.every((record) => record.workflow_success !== null); + const workflowWins = records.filter((record) => record.workflow_success === true).length; + const interval = workflowEvidenceComplete ? taskClusterBootstrapInterval(records) : null; const taskCount = new Set(records.map((record) => record.task_id)).size; const failuresByStage = {}; const failuresByReason = {}; @@ -162,9 +169,15 @@ function aggregateModel(model, records) { required_parameter_accuracy: round(mean(records.map((record) => record.required_parameter_accuracy))), constraint_accuracy: round(mean(records.map((record) => record.constraint_accuracy))), call_success_rate: executed.length ? round(mean(executed.map((record) => record.call_success))) : null, - result_nonempty_rate: executed.length ? round(mean(executed.map((record) => record.result_nonempty))) : null, - workflow_success_rate: round(workflowWins / records.length), - workflow_success_task_cluster_bootstrap_95: interval.map(round), + result_nonempty_rate: + executed.length && resultEvidenceComplete + ? round(mean(executed.map((record) => record.result_nonempty))) + : null, + pre_result_gate_workflow_success_rate: round( + mean(records.map((record) => record.pre_result_gate_workflow_success)), + ), + workflow_success_rate: workflowEvidenceComplete ? round(workflowWins / records.length) : null, + workflow_success_task_cluster_bootstrap_95: interval?.map(round) ?? null, failures_by_stage: failuresByStage, failures_by_reason: failuresByReason, }; @@ -268,3 +281,7 @@ function array(value) { function plainObject(value) { return value !== null && typeof value === 'object' && !Array.isArray(value); } + +function booleanOrNull(value) { + return typeof value === 'boolean' ? value : null; +} diff --git a/benchmarks/discover-call/src/validate.mjs b/benchmarks/discover-call/src/validate.mjs index 5144b6c..c03fa62 100644 --- a/benchmarks/discover-call/src/validate.mjs +++ b/benchmarks/discover-call/src/validate.mjs @@ -1,6 +1,7 @@ import { readFile, readdir } from 'node:fs/promises'; import { resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { isDeepStrictEqual } from 'node:util'; import { validateTask } from './harness.mjs'; import { readJsonLines } from './io.mjs'; @@ -43,9 +44,20 @@ for (const file of resultFiles) { const tasks = await readJsonLines(resolve(root, `tasks/v${version}.jsonl`)); const records = await readJsonLines(resolve(root, 'results', file)); validatePublicRecords(records, policy); - scoreRecords(tasks, records); + const generatedSummary = scoreRecords(tasks, records); + const summaryFile = file.replace(/\.runs\.jsonl$/, '.summary.json'); + const publishedSummary = JSON.parse(await readFile(resolve(root, 'results', summaryFile), 'utf8')); + if (!isDeepStrictEqual(comparableSummary(generatedSummary), comparableSummary(publishedSummary))) { + throw new Error(`${summaryFile}: summary does not match its public run records`); + } } process.stdout.write( `Validated ${counts.join(', ')}, ${fixture.length} fixture records, and ${resultFiles.length} public result files\n`, ); + +function comparableSummary(summary) { + if (!summary || typeof summary !== 'object' || Array.isArray(summary)) return summary; + const { generated_at: _generatedAt, ...comparable } = summary; + return comparable; +} diff --git a/benchmarks/discover-call/test/adapter.test.mjs b/benchmarks/discover-call/test/adapter.test.mjs index 596c8da..fc4b10d 100644 --- a/benchmarks/discover-call/test/adapter.test.mjs +++ b/benchmarks/discover-call/test/adapter.test.mjs @@ -1,6 +1,8 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { resolve } from 'node:path'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { createProcessAdapter, main } from '../src/run.mjs'; @@ -92,6 +94,28 @@ test('process adapter classifies timeouts', async () => { }); }); +test('process adapter force-kills a child that ignores SIGTERM', async () => { + const directory = await mkdtemp(join(tmpdir(), 'qveris-adapter-test-')); + const pidPath = join(directory, 'pid'); + try { + const invoke = createProcessAdapter({ + command: process.execPath, + args: [ + '-e', + "require('node:fs').writeFileSync(process.argv[1], String(process.pid)); process.on('SIGTERM', () => {}); process.stdin.resume(); setInterval(() => {}, 1000)", + pidPath, + ], + timeoutMs: 50, + forceKillAfterMs: 20, + }); + await assert.rejects(invoke({ stage: 'select' }), /timed out/); + const pid = Number(await readFile(pidPath, 'utf8')); + assert.equal(await waitForProcessExit(pid), true); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + test('reference adapter selects only configured candidates present in discovery', async () => { const invoke = createProcessAdapter({ command: process.execPath, @@ -260,3 +284,22 @@ test('Codex adapter extracts the final structured message and rejects tool use', /no structured output/, ); }); + +function processExists(pid) { + try { + process.kill(pid, 0); + return true; + } catch (error) { + if (error?.code === 'ESRCH') return false; + throw error; + } +} + +async function waitForProcessExit(pid, timeoutMs = 1_000) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (!processExists(pid)) return true; + await new Promise((resolvePromise) => setTimeout(resolvePromise, 20)); + } + return !processExists(pid); +} diff --git a/benchmarks/discover-call/test/publication.test.mjs b/benchmarks/discover-call/test/publication.test.mjs index c865dbe..28b4f73 100644 --- a/benchmarks/discover-call/test/publication.test.mjs +++ b/benchmarks/discover-call/test/publication.test.mjs @@ -1,7 +1,11 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { sanitizePublicRecords, validatePublicRecords } from '../src/publication.mjs'; +import { + sanitizePublicRecords, + validatePolicy, + validatePublicRecords, +} from '../src/publication.mjs'; const policy = { schema_version: 1, @@ -18,6 +22,13 @@ const policy = { 'remaining_credits', 'result_tool_ids', 'returned_tool_ids', + 'api_key', + 'access_token', + 'refresh_token', + 'oauth_token', + 'authorization', + 'client_secret', + 'password', ], }; @@ -29,7 +40,11 @@ test('publishes grounded attestations and hashes instead of operational identifi benchmark_version: 'v4', run_id: 'run-1', model: 'model-a', - metadata: { lane: 'configured-model', api_revision: '2026-07-22.1' }, + metadata: { + lane: 'configured-model', + api_revision: '2026-07-22.1', + internal_note: 'must-not-publish', + }, task_id: 'weather', trial: 1, status: 'completed', @@ -59,6 +74,7 @@ test('publishes grounded attestations and hashes instead of operational identifi assert.equal(record.metadata.publication_policy, policy.policy_id); assert.equal(record.metadata.model_revision, 'unreported'); assert.equal(record.metadata.catalog_revision, 'unreported'); + assert.equal('internal_note' in record.metadata, false); assert.equal(record.source_record_sha256.length, 64); for (const forbidden of policy.forbidden_fields) { assert.equal(JSON.stringify(record).includes(`"${forbidden}"`), false); @@ -80,3 +96,75 @@ test('hashes an unapproved selected tool instead of expanding the public catalog assert.equal(record.selection.tool_id_sha256.length, 64); assert.equal(record.discovery.selection_grounded, true); }); + +test('projects safe error fields and rejects credential-shaped parameter keys', () => { + const [record] = sanitizePublicRecords( + [ + { + selection: { tool_id: null }, + error: { + stage: 'adapter', + reason_code: 'tool_use_rejected', + message: 'raw provider body with a secret', + provider_response: 'private', + }, + }, + ], + policy, + ); + assert.deepEqual(record.error, { + stage: 'adapter', + reason_code: 'tool_use_rejected', + }); + assert.throws( + () => + validatePublicRecords( + [ + { + ...record, + metadata: { ...record.metadata, internal_note: 'must-not-publish' }, + }, + ], + policy, + ), + /unsupported metadata/, + ); + assert.throws( + () => + validatePublicRecords( + [ + { + ...record, + error: { ...record.error, message: 'raw provider body' }, + }, + ], + policy, + ), + /unsupported fields/, + ); + + assert.throws( + () => + sanitizePublicRecords( + [ + { + selection: { tool_id: null }, + parameters: { apiKey: 'must-not-publish' }, + }, + ], + policy, + ), + /Forbidden public artifact field/, + ); +}); + +test('publication policy cannot omit a required protected field', () => { + assert.throws( + () => + validatePolicy({ + ...policy, + forbidden_fields: policy.forbidden_fields.filter((field) => field !== 'api_key'), + }), + /must forbid protected fields: api_key/, + ); +}); diff --git a/benchmarks/discover-call/test/scoring.test.mjs b/benchmarks/discover-call/test/scoring.test.mjs index 8d064ed..204727a 100644 --- a/benchmarks/discover-call/test/scoring.test.mjs +++ b/benchmarks/discover-call/test/scoring.test.mjs @@ -20,7 +20,7 @@ test('scores a grounded, complete, successful workflow', () => { selection: { tool_id: 'weather.forecast' }, inspection: { returned_tool_ids: ['weather.forecast'], required_parameters: ['city'] }, parameters: { city: 'london' }, - call: { attempted: true, success: true }, + call: { attempted: true, success: true, result_nonempty: true }, }); assert.equal(result.selection_grounded, true); @@ -129,7 +129,7 @@ test('aggregates per model with parameter and workflow rates', () => { selection: { tool_id: 'weather.forecast' }, inspection: { returned_tool_ids: ['weather.forecast'], required_parameters: ['city'] }, parameters: { city: 'London' }, - call: { attempted: true, success: true }, + call: { attempted: true, success: true, result_nonempty: true }, }, { run_id: 'run-2', @@ -156,6 +156,7 @@ test('aggregates per model with parameter and workflow rates', () => { parameters: summary.models[0].required_parameter_accuracy, constraints: summary.models[0].constraint_accuracy, calls: summary.models[0].call_success_rate, + preResult: summary.models[0].pre_result_gate_workflow_success_rate, workflow: summary.models[0].workflow_success_rate, failures: summary.models[0].failures_by_stage, failureReasons: summary.models[0].failures_by_reason, @@ -168,6 +169,7 @@ test('aggregates per model with parameter and workflow rates', () => { parameters: 0.5, constraints: 0.5, calls: 0.5, + preResult: 0.5, workflow: 0.5, failures: {}, failureReasons: {}, @@ -176,6 +178,32 @@ test('aggregates per model with parameter and workflow rates', () => { assert.equal(summary.models[0].workflow_success_task_cluster_bootstrap_95.length, 2); }); +test('does not infer non-empty results when a successful legacy call lacks evidence', () => { + const record = { + run_id: 'run-unobserved', + task_id: task.id, + model: 'model-a', + trial: 1, + status: 'completed', + discovery: { result_tool_ids: ['weather.forecast'] }, + selection: { tool_id: 'weather.forecast' }, + inspection: { returned_tool_ids: ['weather.forecast'], required_parameters: ['city'] }, + parameters: { city: 'London' }, + call: { attempted: true, success: true }, + }; + + const result = scoreRecord(task, record); + assert.equal(result.result_nonempty, null); + assert.equal(result.pre_result_gate_workflow_success, true); + assert.equal(result.workflow_success, null); + + const summary = scoreRecords([task], [record]); + assert.equal(summary.models[0].result_nonempty_rate, null); + assert.equal(summary.models[0].pre_result_gate_workflow_success_rate, 1); + assert.equal(summary.models[0].workflow_success_rate, null); + assert.equal(summary.models[0].workflow_success_task_cluster_bootstrap_95, null); +}); + test('scores aliases with task-versioned accepted values', () => { const btcTask = { id: 'crypto-price', From cdd92a406c4467ec75631e497b46f1a66ab0748a Mon Sep 17 00:00:00 2001 From: Linfang Wang Date: Thu, 23 Jul 2026 21:28:38 +0800 Subject: [PATCH 11/19] fix(benchmark): enforce public artifact boundaries --- .../discover-call/PUBLICATION_POLICY.md | 12 +- benchmarks/discover-call/README.md | 13 +- .../discover-call/adapters/codex-cli.mjs | 44 +++- .../discover-call/publication-policy.json | 9 +- ...07-23-gpt-5.6-sol-configured-v4.runs.jsonl | 108 ++++---- .../2026-07-23-gpt-5.6-sol-v2.runs.jsonl | 108 ++++---- .../2026-07-23-gpt-5.6-sol-v3.runs.jsonl | 108 ++++---- .../2026-07-23-oracle-v1-v3.runs.jsonl | 108 ++++---- .../2026-07-23-reference-v1-v4.runs.jsonl | 108 ++++---- benchmarks/discover-call/results/README.md | 3 + benchmarks/discover-call/src/publication.mjs | 248 +++++++++++++++++- benchmarks/discover-call/src/publish.mjs | 22 +- benchmarks/discover-call/src/scoring.mjs | 51 +++- benchmarks/discover-call/src/validate.mjs | 13 +- .../discover-call/test/adapter.test.mjs | 18 ++ .../discover-call/test/publication.test.mjs | 190 ++++++++++++-- .../discover-call/test/scoring.test.mjs | 79 ++++++ docs/en-US/discover-call-benchmark.md | 8 +- docs/zh-CN/discover-call-benchmark.md | 6 +- 19 files changed, 901 insertions(+), 355 deletions(-) diff --git a/benchmarks/discover-call/PUBLICATION_POLICY.md b/benchmarks/discover-call/PUBLICATION_POLICY.md index dfce567..37b6186 100644 --- a/benchmarks/discover-call/PUBLICATION_POLICY.md +++ b/benchmarks/discover-call/PUBLICATION_POLICY.md @@ -31,16 +31,20 @@ and account-specific availability remain internal. Public artifacts must not contain API keys, OAuth tokens, connection/account identifiers, `search_id`, `execution_id`, credit balances, raw provider errors, -raw tool results, private prompts, tenant-private tools, or the unfiltered -ordered discovery catalog. +raw tool results, parameter values, private prompts, tenant-private tools, or +the unfiltered ordered discovery catalog. Inspected parameter names are also +omitted because they can disclose the input schema of an unapproved tool. The publication command enforces `publication-policy.json`. It replaces the ordered discovery list with its count and SHA-256 digest and preserves grounded selection as an attestation. An approved selected tool keeps its ID; an unapproved selected tool is represented only by a SHA-256 digest, so a benchmark run cannot silently expand the public catalog. Metadata is projected through an -explicit allowlist, errors retain only normalized stage/reason codes, and -credential-shaped parameter keys are rejected case-insensitively. +explicit allowlist, errors retain only normalized stage/reason codes, and raw +parameter values are replaced by required-parameter and constraint-accuracy +attestations. Validation rejects every unsupported field instead of relying on +a sensitive-key denylist. API base URLs must be explicitly approved by the +policy so internal or staging origins cannot enter public artifacts. Provider authorization and namespace ownership must be confirmed before adding a tool ID to the public allowlist. Removing an approval requires regenerating all diff --git a/benchmarks/discover-call/README.md b/benchmarks/discover-call/README.md index 454d001..6eb8304 100644 --- a/benchmarks/discover-call/README.md +++ b/benchmarks/discover-call/README.md @@ -71,7 +71,8 @@ Task sets are versioned and immutable once referenced by a published result: - `tasks/v4.jsonl` — the current immutable quality-baseline contract. It copies v3 and explicitly accepts `id=1` only for the crypto task's `id` alias. This task-versioned mapping fixes the known v3 scoring false negative without - retroactively changing v3. + retroactively changing v3. Provider-specific `alias_values` use exact + matching even when the human-readable constraint uses `contains`. New tasks land as a new `tasks/vN.jsonl` revision, never by editing an existing file, and must still score deterministically at the contract level. @@ -159,7 +160,10 @@ npm run publish -- \ Keep raw records outside the public repository. Publication removes operational identifiers and the ordered discovery catalog, adds count/digest attestations, -and hashes selected tools that are not explicitly approved. +hashes selected tools that are not explicitly approved, and replaces raw +parameter values with required-parameter and constraint-accuracy attestations. +Inspected parameter names are omitted so a hashed tool cannot leak schema +details. Validate the checked-in task set, fixtures, and scorer: @@ -267,13 +271,16 @@ An official result must include: - at least three trials per task; - `--execute` records for every reported workflow-success denominator; - failures retained in the denominator; no selective reruns or task removal; -- no API keys, access tokens, private prompts, raw provider error bodies, +- no API keys, access tokens, parameter values, private prompts, raw provider error bodies, execution/search/connection identifiers, or unfiltered catalog results. The scorer rejects missing tasks, unequal trial counts, duplicate trials, and non-consecutive trial numbering for every model. It also rejects duplicate run IDs and aggregation across different adapter, toolkit, task-set, endpoint, discovery-limit, or execution settings. +The checked-in-artifact validator also verifies the exact task-set digest, +requires real execution and at least three trials per task, and rejects fields +outside the public artifact schema. The catalog and artifact visibility boundary is defined in [`PUBLICATION_POLICY.md`](PUBLICATION_POLICY.md). Until discovery reports diff --git a/benchmarks/discover-call/adapters/codex-cli.mjs b/benchmarks/discover-call/adapters/codex-cli.mjs index 135e246..046f437 100644 --- a/benchmarks/discover-call/adapters/codex-cli.mjs +++ b/benchmarks/discover-call/adapters/codex-cli.mjs @@ -118,7 +118,16 @@ export async function invokeCodex(payload) { } } -function runCodex(invocation) { +export function runCodex( + invocation, + { outputLimit = 1_000_000, forceKillAfterMs = 1_000 } = {}, +) { + if (!Number.isInteger(outputLimit) || outputLimit < 1) { + throw new Error('outputLimit must be a positive integer'); + } + if (!Number.isInteger(forceKillAfterMs) || forceKillAfterMs < 1) { + throw new Error('forceKillAfterMs must be a positive integer'); + } return new Promise((resolve, reject) => { const env = { ...process.env }; delete env.QVERIS_API_KEY; @@ -128,21 +137,32 @@ function runCodex(invocation) { env, }); let output = ''; - let outputExceeded = false; let forwardedSignal; let forceKillTimer; + let terminationError; + let settled = false; const cleanup = () => { process.off('SIGINT', forwardSigint); process.off('SIGTERM', forwardSigterm); if (forceKillTimer) clearTimeout(forceKillTimer); }; + const rejectOnce = (error) => { + if (settled) return; + settled = true; + reject(error); + }; + const terminate = (error, signal = 'SIGTERM') => { + if (terminationError || settled) return; + terminationError = error; + child.kill(signal); + forceKillTimer = setTimeout(() => child.kill('SIGKILL'), forceKillAfterMs); + forceKillTimer.unref(); + }; const forwardSignal = (signal) => { if (forwardedSignal) return; forwardedSignal = signal; - child.kill(signal); - forceKillTimer = setTimeout(() => child.kill('SIGKILL'), 1_000); - forceKillTimer.unref(); + terminate(adapterFailure('Codex CLI interrupted', 'interrupted'), signal); }; const forwardSigint = () => forwardSignal('SIGINT'); const forwardSigterm = () => forwardSignal('SIGTERM'); @@ -151,26 +171,26 @@ function runCodex(invocation) { child.stdin.on('error', () => {}); child.stdout.on('data', (chunk) => { + if (terminationError) return; output += chunk; - if (output.length > 1_000_000 && !outputExceeded) { - outputExceeded = true; - child.kill('SIGTERM'); + if (output.length > outputLimit) { + terminate(adapterFailure('Codex CLI output exceeded the adapter limit', 'output_limit')); } }); child.stderr.resume(); child.on('error', () => { cleanup(); - reject(adapterFailure('Codex CLI could not be started', 'start_failed')); + rejectOnce(adapterFailure('Codex CLI could not be started', 'start_failed')); }); child.on('close', (code) => { cleanup(); + if (settled) return; if (forwardedSignal) { process.kill(process.pid, forwardedSignal); return; } - if (outputExceeded) { - return reject(adapterFailure('Codex CLI output exceeded the adapter limit', 'output_limit')); - } + if (terminationError) return rejectOnce(terminationError); + settled = true; if (code === 0) resolve(output); else reject(adapterFailure('Codex CLI invocation failed', 'cli_failed')); }); diff --git a/benchmarks/discover-call/publication-policy.json b/benchmarks/discover-call/publication-policy.json index 16b6dfc..9f5eb13 100644 --- a/benchmarks/discover-call/publication-policy.json +++ b/benchmarks/discover-call/publication-policy.json @@ -2,12 +2,15 @@ "schema_version": 1, "policy_id": "public-artifact-v1", "catalog_visibility_default": "private", - "publish_parameters": true, + "publish_parameters": false, "unapproved_selected_tool_handling": "hash", "legacy_lane_rewrites": { "oracle": "reference", "pinned-model": "configured-model" }, + "approved_api_base_urls": [ + "https://qveris.ai/api/v1" + ], "approved_selected_tool_ids": [ "amap_webservice.geocode.geo.retrieve.v3", "api-football.standings.list.v3.76f7703e", @@ -50,6 +53,8 @@ "oauth_token", "authorization", "client_secret", - "password" + "password", + "parameters", + "required_parameters" ] } diff --git a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.runs.jsonl b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.runs.jsonl index 34a2bfe..f5b8045 100644 --- a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.runs.jsonl +++ b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.runs.jsonl @@ -1,54 +1,54 @@ -{"schema_version":1,"benchmark_version":"v1","run_id":"371aaf4e-0615-4710-88dc-5c67e8f30685","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":1,"started_at":"2026-07-23T11:21:32.795Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"244bb1be6e597bd95075f0125dca8e2be2687f8607960916cb7a36ebbb25a5c9","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:21:55.843Z","source_record_sha256":"19b58e231fede0c078aec557ebbf6485480f275bca4fee4439579831fa306ee6"} -{"schema_version":1,"benchmark_version":"v1","run_id":"6c8d58e4-e962-49b6-8f06-1fb22482cf5f","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":2,"started_at":"2026-07-23T11:21:55.843Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"244bb1be6e597bd95075f0125dca8e2be2687f8607960916cb7a36ebbb25a5c9","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:22:19.345Z","source_record_sha256":"c377bc6f28eaa91c5faa5ab0abcfc8b74961cf294aa859842f16a03959b24162"} -{"schema_version":1,"benchmark_version":"v1","run_id":"0e761080-a3c5-4e82-82e2-cb93f97e962f","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":3,"started_at":"2026-07-23T11:22:19.345Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"244bb1be6e597bd95075f0125dca8e2be2687f8607960916cb7a36ebbb25a5c9","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:22:42.136Z","source_record_sha256":"a37ae7222aff18ebba3449bf0a8d50b1e11dc6e92beda09be6e12b73a877e8f2"} -{"schema_version":1,"benchmark_version":"v1","run_id":"f36b9c33-7310-420d-bc8f-26963eb5b722","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-23T11:22:42.136Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7090bc0d2bb54692f07a94a9c33a11631fe09798990dd9238790662776c22977","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:23:05.424Z","source_record_sha256":"a535f4847de398dce635099257343d97919151812fbb3ae2e801e2fbd1f493f1"} -{"schema_version":1,"benchmark_version":"v1","run_id":"478620bd-257e-4f3f-9faf-e5695f06fe8d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-23T11:23:05.424Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7090bc0d2bb54692f07a94a9c33a11631fe09798990dd9238790662776c22977","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON","eod":false,"timezone":"Exchange"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:23:31.485Z","source_record_sha256":"6778249cf3d5816cfaf1ec55f0de21bd325a88963280aea3c4d9902b3dcd77cd"} -{"schema_version":1,"benchmark_version":"v1","run_id":"0a54cf62-5bd6-4137-bc8b-0a1e687b5b1b","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-23T11:23:31.486Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7090bc0d2bb54692f07a94a9c33a11631fe09798990dd9238790662776c22977","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:23:53.420Z","source_record_sha256":"8275132e39abda084cdc23d716b7d032f91a5f28e80f9607dbe4b4d743a2d9bc"} -{"schema_version":1,"benchmark_version":"v1","run_id":"b93e6ec7-aa16-4fa5-b3f9-00e9d729a2f3","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-23T11:23:53.420Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2c68180e93be4ac81c6dbcab29687374ce54f07ddafe04848159bf8cfb882657","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":100,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:24:17.564Z","source_record_sha256":"18bf6bd0c62dfd783cf0de106292da5f50086c41b3d79c5c0430703f09f28624"} -{"schema_version":1,"benchmark_version":"v1","run_id":"db14c6bf-4780-47db-9af9-e7d8aa2988ff","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-23T11:24:17.564Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2c68180e93be4ac81c6dbcab29687374ce54f07ddafe04848159bf8cfb882657","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":100,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:24:41.890Z","source_record_sha256":"0f458900b9044b56fa89b726fdbf76bd9c494e46a236f5eb142b82f15f2e71b2"} -{"schema_version":1,"benchmark_version":"v1","run_id":"886e6db9-fa73-4f8e-9bee-1df89f6cd2ca","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-23T11:24:41.890Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2c68180e93be4ac81c6dbcab29687374ce54f07ddafe04848159bf8cfb882657","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:25:06.537Z","source_record_sha256":"6b42685637dad3c7dda90776bb768e967c3fb3fae6c3feaa40037d8ba69e0b60"} -{"schema_version":1,"benchmark_version":"v1","run_id":"58e58f43-855f-4ebf-a00e-451235a45f93","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":1,"started_at":"2026-07-23T11:25:06.537Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3cd090fbd5820604d10766221f77d1503a3d3271d85af28370d853e9dfbe34a1","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20AI","from":"2026-07-16","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:25:32.429Z","source_record_sha256":"8c07e0c9547070cfe87b68a2a6f1627fab3e453e495c0cce48b928eb7a51e51f"} -{"schema_version":1,"benchmark_version":"v1","run_id":"512dc6cf-6366-4973-b4aa-fea5ddad95ff","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":2,"started_at":"2026-07-23T11:25:32.429Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3cd090fbd5820604d10766221f77d1503a3d3271d85af28370d853e9dfbe34a1","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20AI","from":"2026-07-16","to":"2026-07-23T23:59:59+08:00","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:26:02.529Z","source_record_sha256":"02cf9ccaaa1239778a7e9d8715121b96e9a940d8a3072d4aec14a0d443c7cac3"} -{"schema_version":1,"benchmark_version":"v1","run_id":"42fd8322-85bc-4cd3-bff3-99382b9ee47d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":3,"started_at":"2026-07-23T11:26:02.529Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3cd090fbd5820604d10766221f77d1503a3d3271d85af28370d853e9dfbe34a1","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20OpenAI%20OR%20Anthropic%20OR%20%22machine%20learning%22","from":"2026-07-16","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:26:29.513Z","source_record_sha256":"67ebb682dfbc685b78cdb37f36ca4ba68a4a9677abe4e07f820f0d5d4cbba5cd"} -{"schema_version":1,"benchmark_version":"v1","run_id":"45e748b3-bd65-4f13-a226-9305abcb16ac","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T11:26:29.513Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d2b69a74b7ca36c6145b2f658dff7eae70ca9bdc663ebb85438798ecfdfbd517","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:26:54.396Z","source_record_sha256":"32eb9620da428b7fcd3e84022035dd5143e1ac02219e1144e743406fbff80901"} -{"schema_version":1,"benchmark_version":"v1","run_id":"44d32825-78ff-4068-a63c-42c43b528904","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T11:26:54.396Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d2b69a74b7ca36c6145b2f658dff7eae70ca9bdc663ebb85438798ecfdfbd517","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:27:19.885Z","source_record_sha256":"6c16e9d4850f6481daed57e2925f3fa62e4574552eda0019289d08d7efe7b49a"} -{"schema_version":1,"benchmark_version":"v1","run_id":"2a9c4066-3f0f-4bc9-ace4-c89d9914f323","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T11:27:19.885Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d2b69a74b7ca36c6145b2f658dff7eae70ca9bdc663ebb85438798ecfdfbd517","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:27:42.379Z","source_record_sha256":"83e747801cb18bf90368e5ff3c1797f615b3733a361a472614933fbc95412c59"} -{"schema_version":1,"benchmark_version":"v1","run_id":"973bb4c3-fd62-4528-910b-d4e9d9f13335","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T11:27:42.379Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"cc42608bbca8d2cfc12e11d4ace5f987cf4158fab8f32026c3e26d0e41e070aa","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:28:04.893Z","source_record_sha256":"6b210ca05391e2857e4a5f40d58014fc877f15f7adf3e4bffe231515010052c1"} -{"schema_version":1,"benchmark_version":"v1","run_id":"c4cdd1fd-1b64-4aab-b0f1-bf410779e959","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T11:28:04.894Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"cc42608bbca8d2cfc12e11d4ace5f987cf4158fab8f32026c3e26d0e41e070aa","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:28:34.255Z","source_record_sha256":"ba64b5942d022dde3eb7130aa7b16b1192cd3a0863da475c646f762fd801a91c"} -{"schema_version":1,"benchmark_version":"v1","run_id":"3df5595f-fcd1-4403-ac3b-27d50069f69b","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T11:28:34.255Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"cc42608bbca8d2cfc12e11d4ace5f987cf4158fab8f32026c3e26d0e41e070aa","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:28:58.524Z","source_record_sha256":"2084e7785dea619498038bc7f5b1aa917251af8fccf8ed45b99331fab5a43938"} -{"schema_version":1,"benchmark_version":"v1","run_id":"62beba38-b80d-42a2-8bfa-d628349af67e","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T11:28:58.524Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"ee2a0bbf7535b51d34f66e007ce04a9df26e23ab59529da23e5e644a32833f97","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:29:27.009Z","source_record_sha256":"87a076866c31c19ab3a6ba6aaa82dcd2373b2278dd99dc56c0613d9c13a218ff"} -{"schema_version":1,"benchmark_version":"v1","run_id":"78e43571-51cc-40a9-8b57-8f9c6b890794","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T11:29:27.009Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"ee2a0bbf7535b51d34f66e007ce04a9df26e23ab59529da23e5e644a32833f97","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:29:53.909Z","source_record_sha256":"340312f4dd31b0de6018de3f37ca4eab6f42bc62284d77f730cfc9ef7f15461c"} -{"schema_version":1,"benchmark_version":"v1","run_id":"67272715-b8f0-4ad6-8671-39c16bfe9f04","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T11:29:53.909Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"ee2a0bbf7535b51d34f66e007ce04a9df26e23ab59529da23e5e644a32833f97","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:30:20.862Z","source_record_sha256":"6a50a43d4ab1809ce760dc8735afc6967ce3a2ac55d872637fb5d153bb2f7749"} -{"schema_version":1,"benchmark_version":"v1","run_id":"d4ddae0d-2f59-4ecc-ab94-06727ec58cca","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-23T11:30:20.862Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"223bd05e0fcf12f37867246279bb93c17de568e2fb09922157c1600b83ce9e41","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T11:30:41.307Z","source_record_sha256":"8952e54a57b7bcef0c388f060b6e135b13337f5a62d7411670794fd651ca545e"} -{"schema_version":1,"benchmark_version":"v1","run_id":"d18e99eb-cc46-4095-a8c5-8372f149e061","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-23T11:30:41.307Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"223bd05e0fcf12f37867246279bb93c17de568e2fb09922157c1600b83ce9e41","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T11:31:02.728Z","source_record_sha256":"f629a1ec2d0328840f5be6d19470d8c937f866e296cc2be42ac66f519da5cc89"} -{"schema_version":1,"benchmark_version":"v1","run_id":"5568fd3b-32a1-4c2a-8d22-b4c22d69dd9d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-23T11:31:02.728Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"223bd05e0fcf12f37867246279bb93c17de568e2fb09922157c1600b83ce9e41","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T11:31:32.626Z","source_record_sha256":"24ebbb700feac211b1280355832cf27d5fbfecdac09a112646d1b4ca76fd21ac"} -{"schema_version":1,"benchmark_version":"v1","run_id":"80454152-d4cc-484e-8492-33d5fb8f06b9","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":1,"started_at":"2026-07-23T11:31:32.626Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2099bfa8887678c84902e7ec565c63edeef86ac5127e75b290713f8ff3a4f197","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:31:57.655Z","source_record_sha256":"7cf07a8098fc7de2900eee2c9471d41a5eeb8dec92903448c513bdc0e102d845"} -{"schema_version":1,"benchmark_version":"v1","run_id":"a703667c-b85b-4479-81f4-ea1af3f68891","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":2,"started_at":"2026-07-23T11:31:57.655Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66d9bb9820728c627264f0ade0a1ca2560a70f6bcb8e608cc1ff1dcba98fc27c","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:32:21.294Z","source_record_sha256":"d869eeb25eb2ddb2ba07e3c5abc1d375330ee466b7f89fe5fe9a6c6d1d91b19d"} -{"schema_version":1,"benchmark_version":"v1","run_id":"49211d01-07a0-43e3-82c3-b7fe5a567e04","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":3,"started_at":"2026-07-23T11:32:21.294Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66d9bb9820728c627264f0ade0a1ca2560a70f6bcb8e608cc1ff1dcba98fc27c","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:32:44.251Z","source_record_sha256":"fa084b186970020b9e6a71a1548b8362734847844104781dee93ed3e03002d8f"} -{"schema_version":1,"benchmark_version":"v1","run_id":"8dd15e90-d3e9-4ca3-8a64-dd163696948e","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":1,"started_at":"2026-07-23T11:32:44.251Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5e68e6de1d0883ee6bc55cb00d388d9a48144cac87bf641c9821f40515e7ea32","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:33:10.768Z","source_record_sha256":"d4d3af4888539d89a5f7ee1ac44781bf0493fe29b8c8ed69c54939ece33e1de0"} -{"schema_version":1,"benchmark_version":"v1","run_id":"9e62f261-75fb-4f20-9413-d7d048b239a5","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":2,"started_at":"2026-07-23T11:33:10.768Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5e68e6de1d0883ee6bc55cb00d388d9a48144cac87bf641c9821f40515e7ea32","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:33:35.333Z","source_record_sha256":"9d43f86f5725ac6b93f5b4160928570eb7823f158deb174594bc1666c0239029"} -{"schema_version":1,"benchmark_version":"v1","run_id":"618d54d7-28e3-41ea-8012-b8c054079799","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":3,"started_at":"2026-07-23T11:33:35.333Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5e68e6de1d0883ee6bc55cb00d388d9a48144cac87bf641c9821f40515e7ea32","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:33:57.236Z","source_record_sha256":"4f7bd85dec08098ae63d92fa1790041056e03428f2e763e15fcdbe27facc6d08"} -{"schema_version":1,"benchmark_version":"v1","run_id":"d3692a53-51a6-4970-ace5-06f8517985ae","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":1,"started_at":"2026-07-23T11:33:57.236Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:34:18.423Z","source_record_sha256":"007e7efd2a7ac75ec56044b2cdc4d9b083a58ce9e360a071b6c1fad953748480"} -{"schema_version":1,"benchmark_version":"v1","run_id":"26276fa6-e245-49b3-855b-60a4ea25b85b","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":2,"started_at":"2026-07-23T11:34:18.423Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:34:39.915Z","source_record_sha256":"264d4a0eb360bae5ab942c0fa5fdd5a29134af30bc82caf54e78efbd8bd2c81d"} -{"schema_version":1,"benchmark_version":"v1","run_id":"98acd9e1-dda2-4d51-84ad-6019ffcb21b3","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":3,"started_at":"2026-07-23T11:34:39.915Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:35:00.421Z","source_record_sha256":"fd6d10826ed0789d8a3421f44979c2507485b29181ae8648a512ec8b92b84b82"} -{"schema_version":1,"benchmark_version":"v1","run_id":"58c60360-c513-442d-b08e-f3c729fc5542","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":1,"started_at":"2026-07-23T11:35:00.421Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1fa372fa0e73a010285d8cae90eb1dc1b04ca995386b24d01d04998e09b2f09f","selection_grounded":true},"selection":{"tool_id":null,"tool_id_sha256":"aa938efb4e16c513e85cee2e773d507bc478cb8543b68333ab921785516c7a2e"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:35:22.330Z","source_record_sha256":"59258c6c137545703d2b6ee4fd34db1f67b130aac2b9ffcc15ff326bce2b74a9"} -{"schema_version":1,"benchmark_version":"v1","run_id":"a23aaa73-481a-4b61-b883-c8280ddcc8e0","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":2,"started_at":"2026-07-23T11:35:22.330Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1fa372fa0e73a010285d8cae90eb1dc1b04ca995386b24d01d04998e09b2f09f","selection_grounded":true},"selection":{"tool_id":null,"tool_id_sha256":"aa938efb4e16c513e85cee2e773d507bc478cb8543b68333ab921785516c7a2e"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:35:43.366Z","source_record_sha256":"af69472b0970e418b33cb16e5e055559d872ad62dca122c8f09021ea0425e118"} -{"schema_version":1,"benchmark_version":"v1","run_id":"57461645-9d30-4864-8a68-952d6d99a112","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":3,"started_at":"2026-07-23T11:35:43.366Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1fa372fa0e73a010285d8cae90eb1dc1b04ca995386b24d01d04998e09b2f09f","selection_grounded":true},"selection":{"tool_id":null,"tool_id_sha256":"aa938efb4e16c513e85cee2e773d507bc478cb8543b68333ab921785516c7a2e"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:36:08.708Z","source_record_sha256":"e54b101e094e8c07304b3e62a0d7f0c31ec354414df19fa337ad640f12d3977a"} -{"schema_version":1,"benchmark_version":"v1","run_id":"062ac004-ca1d-4938-a2ee-f56389cc5d8e","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T11:36:08.708Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"bcbd727391a8751121de802694245a7b97b9063d15176e9d3077129ca89c1f23","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected"},"finished_at":"2026-07-23T11:36:48.187Z","source_record_sha256":"e1cc5dbe6957ce04839841829c7e6cb828c4a5c86ed5523f3f74021f6c56e65f"} -{"schema_version":1,"benchmark_version":"v1","run_id":"ea6ee15f-1a67-457d-bf48-e7045eaff0f6","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T11:36:48.187Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"bcbd727391a8751121de802694245a7b97b9063d15176e9d3077129ca89c1f23","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected"},"finished_at":"2026-07-23T11:37:37.695Z","source_record_sha256":"d4cf42d286d5f51b1f00d85fdf776f606ff182d2187b06d000af45d5357557b9"} -{"schema_version":1,"benchmark_version":"v1","run_id":"13e9d4a9-c237-4cfe-8b66-2b4bc6727157","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T11:37:37.695Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"bcbd727391a8751121de802694245a7b97b9063d15176e9d3077129ca89c1f23","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected"},"finished_at":"2026-07-23T11:38:18.107Z","source_record_sha256":"928570b21b09c2fd63f8ae7ee1faf5f9b3b0b66ce7de966f60ec6f95c0d98eb6"} -{"schema_version":1,"benchmark_version":"v1","run_id":"46fa3b68-38ad-4728-9252-32e08dd52ee7","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":1,"started_at":"2026-07-23T11:38:18.107Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"406f82f0151e2aab38d28e5369d6e3b6b8f1e5a40414bd6ea7cafc3eb9f40259","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":10,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:38:46.770Z","source_record_sha256":"b0691ac1115ede5d59b2c49e6c7ad87c6c440e8ad911982666f36d32ffa00b5e"} -{"schema_version":1,"benchmark_version":"v1","run_id":"19c84ab6-eab7-4ed1-b536-20d6db38801c","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":2,"started_at":"2026-07-23T11:38:46.770Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"406f82f0151e2aab38d28e5369d6e3b6b8f1e5a40414bd6ea7cafc3eb9f40259","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:39:13.722Z","source_record_sha256":"3326539c1c392a2b9f51ea323d1c669cbaa2134e4a71361b2a194a1bca2932d4"} -{"schema_version":1,"benchmark_version":"v1","run_id":"d2438aee-241f-4be1-8c92-0a21375a39cd","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":3,"started_at":"2026-07-23T11:39:13.722Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"406f82f0151e2aab38d28e5369d6e3b6b8f1e5a40414bd6ea7cafc3eb9f40259","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:39:36.630Z","source_record_sha256":"57f8f030306de487ac1e7f3de74fbca58e026038c7612b4bf239b44e747a5322"} -{"schema_version":1,"benchmark_version":"v1","run_id":"e8832d33-357f-4f68-925a-d4381142678b","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":1,"started_at":"2026-07-23T11:39:36.631Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2057edc3ba2996ada07d11b091f42383501397f2bd7499d626a300e8022fe784","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:40:01.350Z","source_record_sha256":"8cb15016b1a0c653593e161998b315894277c9cb902fa5c330a3063d1160a0d7"} -{"schema_version":1,"benchmark_version":"v1","run_id":"e87ea408-9168-46df-acbd-a61a85f7ca9e","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":2,"started_at":"2026-07-23T11:40:01.350Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2057edc3ba2996ada07d11b091f42383501397f2bd7499d626a300e8022fe784","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:40:23.845Z","source_record_sha256":"0d878dd10de5848935a490bf4a52317f61c12593c5f73f2aa339a4ec25863952"} -{"schema_version":1,"benchmark_version":"v1","run_id":"e0ac9896-d511-4ba3-90e8-21f01cfde6bc","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":3,"started_at":"2026-07-23T11:40:23.845Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2057edc3ba2996ada07d11b091f42383501397f2bd7499d626a300e8022fe784","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:40:46.507Z","source_record_sha256":"7fbe8c452d77153276d8bc178a8e9ff6998a55ecdfc493ecc0a37a86400ab94f"} -{"schema_version":1,"benchmark_version":"v1","run_id":"5b03766d-4b69-4b3a-acb7-ad89437a239a","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-23T11:40:46.507Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:41:12.868Z","source_record_sha256":"d3033fec8aaaa09d0a46ac19779117f41f0aa04f75a1d20e9cb483295c208615"} -{"schema_version":1,"benchmark_version":"v1","run_id":"62fa2743-433e-4a20-887c-74feafdda446","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-23T11:41:12.868Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:41:40.786Z","source_record_sha256":"d8cbbadffea583a4baab90221b7bc5b3b934c0d2890a78cbd3a85dcc9e2905f2"} -{"schema_version":1,"benchmark_version":"v1","run_id":"90981cc4-2bb6-457f-b8eb-950da1d98923","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T11:41:40.786Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:42:07.441Z","source_record_sha256":"f046d6ef60c603401007987ef63d16bf71e7a3f98e1f70365a07db4c53c65917"} -{"schema_version":1,"benchmark_version":"v1","run_id":"e63213b2-c4c0-4f6c-9d7a-a761069870e2","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":1,"started_at":"2026-07-23T11:42:07.441Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8a3148a19d493f9d95e740965080d759189237c39909aa19fc69be443262784f","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T11:42:29.049Z","source_record_sha256":"32003fe6a9cbb7e6c544e16eae18045665e137fefbc0a3c5547129239fef49d8"} -{"schema_version":1,"benchmark_version":"v1","run_id":"1054878a-b27b-46e5-b9a6-8d6bc3ebec8a","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":2,"started_at":"2026-07-23T11:42:29.049Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8a3148a19d493f9d95e740965080d759189237c39909aa19fc69be443262784f","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T11:42:52.943Z","source_record_sha256":"f5f2b88718734478c4181208300c19098d856c1f1afb48d42e03d3fb0a5cd8d1"} -{"schema_version":1,"benchmark_version":"v1","run_id":"dc98fe27-231c-4a95-af86-12400d405e65","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":3,"started_at":"2026-07-23T11:42:52.943Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8a3148a19d493f9d95e740965080d759189237c39909aa19fc69be443262784f","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T11:43:15.920Z","source_record_sha256":"e4aa45b8a0f7fd32d6cdce7cc5fee63c43885c339568d76e9e20cac4cf1aba5e"} -{"schema_version":1,"benchmark_version":"v1","run_id":"940ed547-9ddc-419c-b0a1-1064f987be0e","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":1,"started_at":"2026-07-23T11:43:15.920Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8581564b8aa970da4b2fbcef387a2e4418ca70298d1c0b2107b4aa7bd16388f7","selection_grounded":true},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"selection_grounded":true,"required_parameters":["season"]},"parameters":{"league":39,"season":2025},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:43:36.976Z","source_record_sha256":"b8ccab960b3ab136b7cadf256b89a4f8da5e090121aaed5d16a0bce35d20ffb3"} -{"schema_version":1,"benchmark_version":"v1","run_id":"8ec93298-1f2c-42ef-ac71-af37df93b741","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":2,"started_at":"2026-07-23T11:43:36.976Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8581564b8aa970da4b2fbcef387a2e4418ca70298d1c0b2107b4aa7bd16388f7","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.98a153bd"},"inspection":{"selection_grounded":true,"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:43:59.877Z","source_record_sha256":"a2a5cdc1c047821de45fbfc48d2686353905e6720ad936e30a8218f2c4f00928"} -{"schema_version":1,"benchmark_version":"v1","run_id":"5f9a762b-7f86-4f7a-a89b-b0abd1e889fc","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":3,"started_at":"2026-07-23T11:43:59.877Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8581564b8aa970da4b2fbcef387a2e4418ca70298d1c0b2107b4aa7bd16388f7","selection_grounded":true},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"selection_grounded":true,"required_parameters":["season"]},"parameters":{"league":39,"season":2025},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:44:24.921Z","source_record_sha256":"68550a9afdb928de6f5eae92226ddf277280452e88acf1a7e4031c08b3894d3a"} +{"schema_version":1,"benchmark_version":"v1","run_id":"371aaf4e-0615-4710-88dc-5c67e8f30685","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":1,"started_at":"2026-07-23T11:21:32.795Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"244bb1be6e597bd95075f0125dca8e2be2687f8607960916cb7a36ebbb25a5c9","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:21:55.843Z","source_record_sha256":"19b58e231fede0c078aec557ebbf6485480f275bca4fee4439579831fa306ee6","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"6c8d58e4-e962-49b6-8f06-1fb22482cf5f","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":2,"started_at":"2026-07-23T11:21:55.843Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"244bb1be6e597bd95075f0125dca8e2be2687f8607960916cb7a36ebbb25a5c9","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:22:19.345Z","source_record_sha256":"c377bc6f28eaa91c5faa5ab0abcfc8b74961cf294aa859842f16a03959b24162","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"0e761080-a3c5-4e82-82e2-cb93f97e962f","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":3,"started_at":"2026-07-23T11:22:19.345Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"244bb1be6e597bd95075f0125dca8e2be2687f8607960916cb7a36ebbb25a5c9","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:22:42.136Z","source_record_sha256":"a37ae7222aff18ebba3449bf0a8d50b1e11dc6e92beda09be6e12b73a877e8f2","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"f36b9c33-7310-420d-bc8f-26963eb5b722","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-23T11:22:42.136Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7090bc0d2bb54692f07a94a9c33a11631fe09798990dd9238790662776c22977","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:23:05.424Z","source_record_sha256":"a535f4847de398dce635099257343d97919151812fbb3ae2e801e2fbd1f493f1","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"478620bd-257e-4f3f-9faf-e5695f06fe8d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-23T11:23:05.424Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7090bc0d2bb54692f07a94a9c33a11631fe09798990dd9238790662776c22977","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:23:31.485Z","source_record_sha256":"6778249cf3d5816cfaf1ec55f0de21bd325a88963280aea3c4d9902b3dcd77cd","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"0a54cf62-5bd6-4137-bc8b-0a1e687b5b1b","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-23T11:23:31.486Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7090bc0d2bb54692f07a94a9c33a11631fe09798990dd9238790662776c22977","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:23:53.420Z","source_record_sha256":"8275132e39abda084cdc23d716b7d032f91a5f28e80f9607dbe4b4d743a2d9bc","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"b93e6ec7-aa16-4fa5-b3f9-00e9d729a2f3","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-23T11:23:53.420Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2c68180e93be4ac81c6dbcab29687374ce54f07ddafe04848159bf8cfb882657","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:24:17.564Z","source_record_sha256":"18bf6bd0c62dfd783cf0de106292da5f50086c41b3d79c5c0430703f09f28624","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"db14c6bf-4780-47db-9af9-e7d8aa2988ff","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-23T11:24:17.564Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2c68180e93be4ac81c6dbcab29687374ce54f07ddafe04848159bf8cfb882657","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:24:41.890Z","source_record_sha256":"0f458900b9044b56fa89b726fdbf76bd9c494e46a236f5eb142b82f15f2e71b2","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"886e6db9-fa73-4f8e-9bee-1df89f6cd2ca","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-23T11:24:41.890Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2c68180e93be4ac81c6dbcab29687374ce54f07ddafe04848159bf8cfb882657","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:25:06.537Z","source_record_sha256":"6b42685637dad3c7dda90776bb768e967c3fb3fae6c3feaa40037d8ba69e0b60","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"58e58f43-855f-4ebf-a00e-451235a45f93","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":1,"started_at":"2026-07-23T11:25:06.537Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3cd090fbd5820604d10766221f77d1503a3d3271d85af28370d853e9dfbe34a1","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:25:32.429Z","source_record_sha256":"8c07e0c9547070cfe87b68a2a6f1627fab3e453e495c0cce48b928eb7a51e51f","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"512dc6cf-6366-4973-b4aa-fea5ddad95ff","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":2,"started_at":"2026-07-23T11:25:32.429Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3cd090fbd5820604d10766221f77d1503a3d3271d85af28370d853e9dfbe34a1","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:26:02.529Z","source_record_sha256":"02cf9ccaaa1239778a7e9d8715121b96e9a940d8a3072d4aec14a0d443c7cac3","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"42fd8322-85bc-4cd3-bff3-99382b9ee47d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":3,"started_at":"2026-07-23T11:26:02.529Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3cd090fbd5820604d10766221f77d1503a3d3271d85af28370d853e9dfbe34a1","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:26:29.513Z","source_record_sha256":"67ebb682dfbc685b78cdb37f36ca4ba68a4a9677abe4e07f820f0d5d4cbba5cd","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"45e748b3-bd65-4f13-a226-9305abcb16ac","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T11:26:29.513Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d2b69a74b7ca36c6145b2f658dff7eae70ca9bdc663ebb85438798ecfdfbd517","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:26:54.396Z","source_record_sha256":"32eb9620da428b7fcd3e84022035dd5143e1ac02219e1144e743406fbff80901","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"44d32825-78ff-4068-a63c-42c43b528904","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T11:26:54.396Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d2b69a74b7ca36c6145b2f658dff7eae70ca9bdc663ebb85438798ecfdfbd517","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:27:19.885Z","source_record_sha256":"6c16e9d4850f6481daed57e2925f3fa62e4574552eda0019289d08d7efe7b49a","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"2a9c4066-3f0f-4bc9-ace4-c89d9914f323","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T11:27:19.885Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d2b69a74b7ca36c6145b2f658dff7eae70ca9bdc663ebb85438798ecfdfbd517","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:27:42.379Z","source_record_sha256":"83e747801cb18bf90368e5ff3c1797f615b3733a361a472614933fbc95412c59","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"973bb4c3-fd62-4528-910b-d4e9d9f13335","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T11:27:42.379Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"cc42608bbca8d2cfc12e11d4ace5f987cf4158fab8f32026c3e26d0e41e070aa","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:28:04.893Z","source_record_sha256":"6b210ca05391e2857e4a5f40d58014fc877f15f7adf3e4bffe231515010052c1","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"c4cdd1fd-1b64-4aab-b0f1-bf410779e959","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T11:28:04.894Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"cc42608bbca8d2cfc12e11d4ace5f987cf4158fab8f32026c3e26d0e41e070aa","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:28:34.255Z","source_record_sha256":"ba64b5942d022dde3eb7130aa7b16b1192cd3a0863da475c646f762fd801a91c","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"3df5595f-fcd1-4403-ac3b-27d50069f69b","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T11:28:34.255Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"cc42608bbca8d2cfc12e11d4ace5f987cf4158fab8f32026c3e26d0e41e070aa","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:28:58.524Z","source_record_sha256":"2084e7785dea619498038bc7f5b1aa917251af8fccf8ed45b99331fab5a43938","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"62beba38-b80d-42a2-8bfa-d628349af67e","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T11:28:58.524Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"ee2a0bbf7535b51d34f66e007ce04a9df26e23ab59529da23e5e644a32833f97","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:29:27.009Z","source_record_sha256":"87a076866c31c19ab3a6ba6aaa82dcd2373b2278dd99dc56c0613d9c13a218ff","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"78e43571-51cc-40a9-8b57-8f9c6b890794","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T11:29:27.009Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"ee2a0bbf7535b51d34f66e007ce04a9df26e23ab59529da23e5e644a32833f97","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:29:53.909Z","source_record_sha256":"340312f4dd31b0de6018de3f37ca4eab6f42bc62284d77f730cfc9ef7f15461c","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"67272715-b8f0-4ad6-8671-39c16bfe9f04","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T11:29:53.909Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"ee2a0bbf7535b51d34f66e007ce04a9df26e23ab59529da23e5e644a32833f97","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:30:20.862Z","source_record_sha256":"6a50a43d4ab1809ce760dc8735afc6967ce3a2ac55d872637fb5d153bb2f7749","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"d4ddae0d-2f59-4ecc-ab94-06727ec58cca","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-23T11:30:20.862Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"223bd05e0fcf12f37867246279bb93c17de568e2fb09922157c1600b83ce9e41","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T11:30:41.307Z","source_record_sha256":"8952e54a57b7bcef0c388f060b6e135b13337f5a62d7411670794fd651ca545e","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"d18e99eb-cc46-4095-a8c5-8372f149e061","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-23T11:30:41.307Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"223bd05e0fcf12f37867246279bb93c17de568e2fb09922157c1600b83ce9e41","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T11:31:02.728Z","source_record_sha256":"f629a1ec2d0328840f5be6d19470d8c937f866e296cc2be42ac66f519da5cc89","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"5568fd3b-32a1-4c2a-8d22-b4c22d69dd9d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-23T11:31:02.728Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"223bd05e0fcf12f37867246279bb93c17de568e2fb09922157c1600b83ce9e41","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T11:31:32.626Z","source_record_sha256":"24ebbb700feac211b1280355832cf27d5fbfecdac09a112646d1b4ca76fd21ac","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"80454152-d4cc-484e-8492-33d5fb8f06b9","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":1,"started_at":"2026-07-23T11:31:32.626Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2099bfa8887678c84902e7ec565c63edeef86ac5127e75b290713f8ff3a4f197","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:31:57.655Z","source_record_sha256":"7cf07a8098fc7de2900eee2c9471d41a5eeb8dec92903448c513bdc0e102d845","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"a703667c-b85b-4479-81f4-ea1af3f68891","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":2,"started_at":"2026-07-23T11:31:57.655Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66d9bb9820728c627264f0ade0a1ca2560a70f6bcb8e608cc1ff1dcba98fc27c","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:32:21.294Z","source_record_sha256":"d869eeb25eb2ddb2ba07e3c5abc1d375330ee466b7f89fe5fe9a6c6d1d91b19d","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"49211d01-07a0-43e3-82c3-b7fe5a567e04","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":3,"started_at":"2026-07-23T11:32:21.294Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66d9bb9820728c627264f0ade0a1ca2560a70f6bcb8e608cc1ff1dcba98fc27c","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:32:44.251Z","source_record_sha256":"fa084b186970020b9e6a71a1548b8362734847844104781dee93ed3e03002d8f","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"8dd15e90-d3e9-4ca3-8a64-dd163696948e","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":1,"started_at":"2026-07-23T11:32:44.251Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5e68e6de1d0883ee6bc55cb00d388d9a48144cac87bf641c9821f40515e7ea32","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:33:10.768Z","source_record_sha256":"d4d3af4888539d89a5f7ee1ac44781bf0493fe29b8c8ed69c54939ece33e1de0","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"9e62f261-75fb-4f20-9413-d7d048b239a5","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":2,"started_at":"2026-07-23T11:33:10.768Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5e68e6de1d0883ee6bc55cb00d388d9a48144cac87bf641c9821f40515e7ea32","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:33:35.333Z","source_record_sha256":"9d43f86f5725ac6b93f5b4160928570eb7823f158deb174594bc1666c0239029","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"618d54d7-28e3-41ea-8012-b8c054079799","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":3,"started_at":"2026-07-23T11:33:35.333Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5e68e6de1d0883ee6bc55cb00d388d9a48144cac87bf641c9821f40515e7ea32","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:33:57.236Z","source_record_sha256":"4f7bd85dec08098ae63d92fa1790041056e03428f2e763e15fcdbe27facc6d08","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"d3692a53-51a6-4970-ace5-06f8517985ae","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":1,"started_at":"2026-07-23T11:33:57.236Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:34:18.423Z","source_record_sha256":"007e7efd2a7ac75ec56044b2cdc4d9b083a58ce9e360a071b6c1fad953748480","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"26276fa6-e245-49b3-855b-60a4ea25b85b","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":2,"started_at":"2026-07-23T11:34:18.423Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:34:39.915Z","source_record_sha256":"264d4a0eb360bae5ab942c0fa5fdd5a29134af30bc82caf54e78efbd8bd2c81d","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"98acd9e1-dda2-4d51-84ad-6019ffcb21b3","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":3,"started_at":"2026-07-23T11:34:39.915Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:35:00.421Z","source_record_sha256":"fd6d10826ed0789d8a3421f44979c2507485b29181ae8648a512ec8b92b84b82","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"58c60360-c513-442d-b08e-f3c729fc5542","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":1,"started_at":"2026-07-23T11:35:00.421Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1fa372fa0e73a010285d8cae90eb1dc1b04ca995386b24d01d04998e09b2f09f","selection_grounded":true},"selection":{"tool_id":null,"tool_id_sha256":"aa938efb4e16c513e85cee2e773d507bc478cb8543b68333ab921785516c7a2e"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:35:22.330Z","source_record_sha256":"59258c6c137545703d2b6ee4fd34db1f67b130aac2b9ffcc15ff326bce2b74a9","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"a23aaa73-481a-4b61-b883-c8280ddcc8e0","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":2,"started_at":"2026-07-23T11:35:22.330Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1fa372fa0e73a010285d8cae90eb1dc1b04ca995386b24d01d04998e09b2f09f","selection_grounded":true},"selection":{"tool_id":null,"tool_id_sha256":"aa938efb4e16c513e85cee2e773d507bc478cb8543b68333ab921785516c7a2e"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:35:43.366Z","source_record_sha256":"af69472b0970e418b33cb16e5e055559d872ad62dca122c8f09021ea0425e118","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"57461645-9d30-4864-8a68-952d6d99a112","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":3,"started_at":"2026-07-23T11:35:43.366Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1fa372fa0e73a010285d8cae90eb1dc1b04ca995386b24d01d04998e09b2f09f","selection_grounded":true},"selection":{"tool_id":null,"tool_id_sha256":"aa938efb4e16c513e85cee2e773d507bc478cb8543b68333ab921785516c7a2e"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:36:08.708Z","source_record_sha256":"e54b101e094e8c07304b3e62a0d7f0c31ec354414df19fa337ad640f12d3977a","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"062ac004-ca1d-4938-a2ee-f56389cc5d8e","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T11:36:08.708Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"bcbd727391a8751121de802694245a7b97b9063d15176e9d3077129ca89c1f23","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true},"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected"},"finished_at":"2026-07-23T11:36:48.187Z","source_record_sha256":"e1cc5dbe6957ce04839841829c7e6cb828c4a5c86ed5523f3f74021f6c56e65f","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"ea6ee15f-1a67-457d-bf48-e7045eaff0f6","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T11:36:48.187Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"bcbd727391a8751121de802694245a7b97b9063d15176e9d3077129ca89c1f23","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true},"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected"},"finished_at":"2026-07-23T11:37:37.695Z","source_record_sha256":"d4cf42d286d5f51b1f00d85fdf776f606ff182d2187b06d000af45d5357557b9","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"13e9d4a9-c237-4cfe-8b66-2b4bc6727157","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T11:37:37.695Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"bcbd727391a8751121de802694245a7b97b9063d15176e9d3077129ca89c1f23","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true},"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected"},"finished_at":"2026-07-23T11:38:18.107Z","source_record_sha256":"928570b21b09c2fd63f8ae7ee1faf5f9b3b0b66ce7de966f60ec6f95c0d98eb6","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"46fa3b68-38ad-4728-9252-32e08dd52ee7","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":1,"started_at":"2026-07-23T11:38:18.107Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"406f82f0151e2aab38d28e5369d6e3b6b8f1e5a40414bd6ea7cafc3eb9f40259","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:38:46.770Z","source_record_sha256":"b0691ac1115ede5d59b2c49e6c7ad87c6c440e8ad911982666f36d32ffa00b5e","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"19c84ab6-eab7-4ed1-b536-20d6db38801c","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":2,"started_at":"2026-07-23T11:38:46.770Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"406f82f0151e2aab38d28e5369d6e3b6b8f1e5a40414bd6ea7cafc3eb9f40259","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:39:13.722Z","source_record_sha256":"3326539c1c392a2b9f51ea323d1c669cbaa2134e4a71361b2a194a1bca2932d4","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"d2438aee-241f-4be1-8c92-0a21375a39cd","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":3,"started_at":"2026-07-23T11:39:13.722Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"406f82f0151e2aab38d28e5369d6e3b6b8f1e5a40414bd6ea7cafc3eb9f40259","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:39:36.630Z","source_record_sha256":"57f8f030306de487ac1e7f3de74fbca58e026038c7612b4bf239b44e747a5322","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"e8832d33-357f-4f68-925a-d4381142678b","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":1,"started_at":"2026-07-23T11:39:36.631Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2057edc3ba2996ada07d11b091f42383501397f2bd7499d626a300e8022fe784","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:40:01.350Z","source_record_sha256":"8cb15016b1a0c653593e161998b315894277c9cb902fa5c330a3063d1160a0d7","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"e87ea408-9168-46df-acbd-a61a85f7ca9e","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":2,"started_at":"2026-07-23T11:40:01.350Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2057edc3ba2996ada07d11b091f42383501397f2bd7499d626a300e8022fe784","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:40:23.845Z","source_record_sha256":"0d878dd10de5848935a490bf4a52317f61c12593c5f73f2aa339a4ec25863952","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"e0ac9896-d511-4ba3-90e8-21f01cfde6bc","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":3,"started_at":"2026-07-23T11:40:23.845Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2057edc3ba2996ada07d11b091f42383501397f2bd7499d626a300e8022fe784","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:40:46.507Z","source_record_sha256":"7fbe8c452d77153276d8bc178a8e9ff6998a55ecdfc493ecc0a37a86400ab94f","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"5b03766d-4b69-4b3a-acb7-ad89437a239a","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-23T11:40:46.507Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:41:12.868Z","source_record_sha256":"d3033fec8aaaa09d0a46ac19779117f41f0aa04f75a1d20e9cb483295c208615","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"62fa2743-433e-4a20-887c-74feafdda446","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-23T11:41:12.868Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:41:40.786Z","source_record_sha256":"d8cbbadffea583a4baab90221b7bc5b3b934c0d2890a78cbd3a85dcc9e2905f2","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"90981cc4-2bb6-457f-b8eb-950da1d98923","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T11:41:40.786Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:42:07.441Z","source_record_sha256":"f046d6ef60c603401007987ef63d16bf71e7a3f98e1f70365a07db4c53c65917","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"e63213b2-c4c0-4f6c-9d7a-a761069870e2","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":1,"started_at":"2026-07-23T11:42:07.441Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8a3148a19d493f9d95e740965080d759189237c39909aa19fc69be443262784f","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T11:42:29.049Z","source_record_sha256":"32003fe6a9cbb7e6c544e16eae18045665e137fefbc0a3c5547129239fef49d8","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"1054878a-b27b-46e5-b9a6-8d6bc3ebec8a","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":2,"started_at":"2026-07-23T11:42:29.049Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8a3148a19d493f9d95e740965080d759189237c39909aa19fc69be443262784f","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T11:42:52.943Z","source_record_sha256":"f5f2b88718734478c4181208300c19098d856c1f1afb48d42e03d3fb0a5cd8d1","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"dc98fe27-231c-4a95-af86-12400d405e65","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":3,"started_at":"2026-07-23T11:42:52.943Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8a3148a19d493f9d95e740965080d759189237c39909aa19fc69be443262784f","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T11:43:15.920Z","source_record_sha256":"e4aa45b8a0f7fd32d6cdce7cc5fee63c43885c339568d76e9e20cac4cf1aba5e","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"940ed547-9ddc-419c-b0a1-1064f987be0e","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":1,"started_at":"2026-07-23T11:43:15.920Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8581564b8aa970da4b2fbcef387a2e4418ca70298d1c0b2107b4aa7bd16388f7","selection_grounded":true},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:43:36.976Z","source_record_sha256":"b8ccab960b3ab136b7cadf256b89a4f8da5e090121aaed5d16a0bce35d20ffb3","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"8ec93298-1f2c-42ef-ac71-af37df93b741","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":2,"started_at":"2026-07-23T11:43:36.976Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8581564b8aa970da4b2fbcef387a2e4418ca70298d1c0b2107b4aa7bd16388f7","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.98a153bd"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:43:59.877Z","source_record_sha256":"a2a5cdc1c047821de45fbfc48d2686353905e6720ad936e30a8218f2c4f00928","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"5f9a762b-7f86-4f7a-a89b-b0abd1e889fc","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/codex-cli-0.144.1/medium","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":3,"started_at":"2026-07-23T11:43:59.877Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8581564b8aa970da4b2fbcef387a2e4418ca70298d1c0b2107b4aa7bd16388f7","selection_grounded":true},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:44:24.921Z","source_record_sha256":"68550a9afdb928de6f5eae92226ddf277280452e88acf1a7e4031c08b3894d3a","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} diff --git a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.runs.jsonl b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.runs.jsonl index 9804606..b48cfba 100644 --- a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.runs.jsonl +++ b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.runs.jsonl @@ -1,54 +1,54 @@ -{"schema_version":1,"benchmark_version":"v1","run_id":"ac88304b-0662-4d2b-bb4b-778a65d4d84d","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":1,"started_at":"2026-07-23T06:26:39.032Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8ff1444540e79a77ed35ac1b5cb0b6de9b5cbfad78aca4b2d6f896484d9fd1d4","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:27:00.228Z","source_record_sha256":"e5922113293b9cbaef44499c19030629c3cd20fb1d48276eb4ef820887c78c9d"} -{"schema_version":1,"benchmark_version":"v1","run_id":"5b911181-4d6c-472a-a2a0-4cb12e433f06","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":2,"started_at":"2026-07-23T06:27:00.228Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8ff1444540e79a77ed35ac1b5cb0b6de9b5cbfad78aca4b2d6f896484d9fd1d4","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:27:23.671Z","source_record_sha256":"daa24fddaeade477213fd0cfdafdaf2a07333f517ebf55f0d5d65dcb0bbb9f6b"} -{"schema_version":1,"benchmark_version":"v1","run_id":"a8268395-01a9-413b-b2af-ba30d2e71f9a","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":3,"started_at":"2026-07-23T06:27:23.671Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8ff1444540e79a77ed35ac1b5cb0b6de9b5cbfad78aca4b2d6f896484d9fd1d4","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:27:50.399Z","source_record_sha256":"75bc10aabffa921d7baefe47658cf7b371196e6ee76a29059d64acad8687a267"} -{"schema_version":1,"benchmark_version":"v1","run_id":"4e925856-364c-4ca7-b769-c8d4d43e85c2","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-23T06:27:50.399Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8c4e8e2ab56f10ce503086b98981d6be57f293b4043dec00a86237144a107a5b","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:28:19.862Z","source_record_sha256":"06c1b8e490bc19f2da0294fb66c4db895a9fa3337d8ed4c8ed35438b17bb33b6"} -{"schema_version":1,"benchmark_version":"v1","run_id":"9c2b558d-b803-4299-9b8d-7778e56d0fcb","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-23T06:28:19.862Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8c4e8e2ab56f10ce503086b98981d6be57f293b4043dec00a86237144a107a5b","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:28:47.286Z","source_record_sha256":"0e1e42c0295011ad188ea782a3eda5887fa763faab0dbe0f5c3f399da3a5e1f8"} -{"schema_version":1,"benchmark_version":"v1","run_id":"6762a6af-2002-4349-9239-89c6c62a597b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-23T06:28:47.286Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8c4e8e2ab56f10ce503086b98981d6be57f293b4043dec00a86237144a107a5b","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:29:13.981Z","source_record_sha256":"fdea4303bae16790c241f8d7743d7ced5d74783b6681a8c8792d6ca6f9dd0c4c"} -{"schema_version":1,"benchmark_version":"v1","run_id":"aa24674f-2a54-49ce-9652-95a7afc5a139","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-23T06:29:13.981Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e78e9a2e15b7817ec95c9a9f2dd8863f05b1049624a65dd87d0387e113e06d29","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":100,"format":"JSON"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:29:40.942Z","source_record_sha256":"f0206728537187674d6417f5baa0210e91dc590443b30c2f658957890ba782f1"} -{"schema_version":1,"benchmark_version":"v1","run_id":"d038e88d-be8e-4d2e-810c-ec4f155b6871","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-23T06:29:40.943Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e78e9a2e15b7817ec95c9a9f2dd8863f05b1049624a65dd87d0387e113e06d29","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":100,"format":"JSON"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:30:10.765Z","source_record_sha256":"ffac927ecd59c4b15f6433f9e4a697caea50585c7a1f31b3fbff9f5b726e0da3"} -{"schema_version":1,"benchmark_version":"v1","run_id":"c54fe3bb-0b0a-4758-beec-b7a5ce2bb540","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-23T06:30:10.765Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e78e9a2e15b7817ec95c9a9f2dd8863f05b1049624a65dd87d0387e113e06d29","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:30:38.654Z","source_record_sha256":"5e3fff916b17629db0000dccb4d7df9783a86ba705981beeac9a3f061819e2df"} -{"schema_version":1,"benchmark_version":"v1","run_id":"6bede096-05fb-47c7-a2eb-5a2f12dad17d","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":1,"started_at":"2026-07-23T06:30:38.654Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"048ef068d6abc7e544421421e151018f4495f28c24194b2c149e5b5fdc19a19f","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20%22generative%20AI%22","from":"2026-07-16","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:31:10.595Z","source_record_sha256":"0e8d2e4dec3b62efe1c8dc241d837790f077bf855a59e34e146ba36fb9d87157"} -{"schema_version":1,"benchmark_version":"v1","run_id":"44918a28-652b-430d-ac7e-53daadb0e69b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":2,"started_at":"2026-07-23T06:31:10.595Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"048ef068d6abc7e544421421e151018f4495f28c24194b2c149e5b5fdc19a19f","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"%28%22artificial%20intelligence%22%20OR%20%22generative%20AI%22%20OR%20OpenAI%20OR%20Anthropic%29","from":"2026-07-16T00:00:00","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:31:44.443Z","source_record_sha256":"7d26eae6c76e79caa8b781a941faa89d732adc2180de4aca7b4a3ee2e1040558"} -{"schema_version":1,"benchmark_version":"v1","run_id":"a306ce69-4682-4778-a8e5-47b0c598e895","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":3,"started_at":"2026-07-23T06:31:44.443Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"86c0120a22c9e323afefb1ad57da1301f45e352b84c74122ca3bdcbbbd59ffb7","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20%22generative%20AI%22","from":"2026-07-16","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:32:18.995Z","source_record_sha256":"42521c23757219360713aa5afcf829bf3b9f312d45a0d82f0534446f5c0fe40a"} -{"schema_version":1,"benchmark_version":"v1","run_id":"9377cc54-be7e-42b4-bf14-f4d14d013c54","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T06:32:18.995Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"301a7dca171878182649e610f180ae3c6f7db7f92485847b354714b924f969e4","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:32:52.395Z","source_record_sha256":"44b9472e368322522d38d3d194dc4a259d7d86e92fd6b3ebc65af3335e3d8fbf"} -{"schema_version":1,"benchmark_version":"v1","run_id":"c7815fa0-d6c8-4346-8463-1ae287470639","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T06:32:52.395Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"301a7dca171878182649e610f180ae3c6f7db7f92485847b354714b924f969e4","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:33:23.255Z","source_record_sha256":"1706ab297ad8e5469871e4c5dabc845a2a6bc64322945ea48954e30d43e3f099"} -{"schema_version":1,"benchmark_version":"v1","run_id":"844e8bbf-ae65-4047-9349-cbd1c1bf7b50","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T06:33:23.255Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"301a7dca171878182649e610f180ae3c6f7db7f92485847b354714b924f969e4","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:33:51.402Z","source_record_sha256":"7d86a9a686125488ce2a9c9827e7d8615a48aa4589c6f196894dec97f271cde3"} -{"schema_version":1,"benchmark_version":"v1","run_id":"cc32c600-4ef5-4fd8-b37c-6ae16c084f7e","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T06:33:51.402Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"850daeb30805dbaca59081825c8550458c4087938c88b78fb51cbc9e06aa928f","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:34:27.740Z","source_record_sha256":"e9825ffa157918d1b5bc4d9a98b4e2287745715b9d48bfb78f75999a5729c28d"} -{"schema_version":1,"benchmark_version":"v1","run_id":"de6093e2-ce59-4692-ae3c-da0801ac1675","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T06:34:27.740Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"850daeb30805dbaca59081825c8550458c4087938c88b78fb51cbc9e06aa928f","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:34:55.284Z","source_record_sha256":"14985dc8546323241dd9218ff7d0d7d60342c29e2f67882fc90ca3c9fdb9d946"} -{"schema_version":1,"benchmark_version":"v1","run_id":"a66e8303-b561-4d85-a2fa-baf46e0a953b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T06:34:55.284Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"850daeb30805dbaca59081825c8550458c4087938c88b78fb51cbc9e06aa928f","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:35:32.765Z","source_record_sha256":"187dfceaab10938e6587cda861b728789d536bf3b28d28b0cd4ba929ae1283d4"} -{"schema_version":1,"benchmark_version":"v1","run_id":"28718552-be81-486c-b313-75a5550ff3cc","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T06:35:32.765Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5783235b11728761fe41459111710a4cb2120d0d39e45f52a0cf8924ce54f453","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京","output":"JSON"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:36:03.749Z","source_record_sha256":"c6b25e36cd3b6ba2ed3b94a5041676171634d2ea7861d065351eab66243722ae"} -{"schema_version":1,"benchmark_version":"v1","run_id":"26e071b2-d139-4ee3-879c-5900578b1233","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T06:36:03.749Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5783235b11728761fe41459111710a4cb2120d0d39e45f52a0cf8924ce54f453","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:36:38.806Z","source_record_sha256":"bc95771cdcc075b15bbab52498067471e1ec7bf88d9d018f6906c25212f49e33"} -{"schema_version":1,"benchmark_version":"v1","run_id":"e2f6f4c2-a8e7-4967-98b1-1a8c29aa3410","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T06:36:38.806Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5783235b11728761fe41459111710a4cb2120d0d39e45f52a0cf8924ce54f453","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:37:14.052Z","source_record_sha256":"17081aa679bee8e6eea626e26319507fcf79576ed5dea927e450a7f1ec7a3d7a"} -{"schema_version":1,"benchmark_version":"v1","run_id":"fde5c3d4-8ff3-4813-a8c5-5e6a61ed3c2e","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-23T06:37:14.052Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"554e0d179f358300caff88da68d3aa9a452bb6b0b2fd0c9b3d7c4e72105773bf","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false},"finished_at":"2026-07-23T06:37:44.040Z","source_record_sha256":"f91cf8ce9139290d5c6da675f04813d6e6ee21b29963bfe35b8997feb17074b6"} -{"schema_version":1,"benchmark_version":"v1","run_id":"cb085acd-e437-40df-9e07-e91343e2af9d","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-23T06:37:44.040Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"554e0d179f358300caff88da68d3aa9a452bb6b0b2fd0c9b3d7c4e72105773bf","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false},"finished_at":"2026-07-23T06:38:14.212Z","source_record_sha256":"bf745350830585628ac0756e443a10066e9bb18c4ef7b5b5a4a19b4595e9e1bf"} -{"schema_version":1,"benchmark_version":"v1","run_id":"143359c4-589b-437f-ae30-9a99b9f1636b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-23T06:38:14.212Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"554e0d179f358300caff88da68d3aa9a452bb6b0b2fd0c9b3d7c4e72105773bf","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false},"finished_at":"2026-07-23T06:38:55.364Z","source_record_sha256":"f4d2ba49e80c77e7e6ea19c287b2c01827f9ae5a37900e71462cdfabf3d2ca97"} -{"schema_version":1,"benchmark_version":"v1","run_id":"58e4b1fd-8de8-4f85-8db1-478b57309b00","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":1,"started_at":"2026-07-23T06:38:55.364Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d6fafa5dcc5b80de51ca0ad0cd3e975827e3281c0aa5dbb07e57ede5fb7c75d6","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:39:36.286Z","source_record_sha256":"b3b701d6ebe68b0c1cc8c50a5cb2d0dd1f99fd925aab1c5195e0757bd216732f"} -{"schema_version":1,"benchmark_version":"v1","run_id":"f498a668-e254-4a1f-a109-0b11043a9a77","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":2,"started_at":"2026-07-23T06:39:36.286Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d6fafa5dcc5b80de51ca0ad0cd3e975827e3281c0aa5dbb07e57ede5fb7c75d6","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:40:06.523Z","source_record_sha256":"0b053eb5eb7799ea2e73b49aae1f3c395b57cb29bb4357530c73f123d47db293"} -{"schema_version":1,"benchmark_version":"v1","run_id":"3b48193b-6026-491e-8472-51034ae93749","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":3,"started_at":"2026-07-23T06:40:06.523Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d6fafa5dcc5b80de51ca0ad0cd3e975827e3281c0aa5dbb07e57ede5fb7c75d6","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:40:52.281Z","source_record_sha256":"ed953a426b8091ac61b9d3f826d950ca92c83ece434f088c6a5144b0378dd77d"} -{"schema_version":1,"benchmark_version":"v1","run_id":"3d70bc1e-3063-41e6-8b57-40c6e6b963e2","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":1,"started_at":"2026-07-23T06:40:52.281Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7785fb29a5fc11f07890b56ab20c7fc1235e8dca0651ca769eaf65c71fb42f5f","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:41:28.119Z","source_record_sha256":"a95a691983bf0e0c7d529f7562a557be3507430d675a4bd3d839df3216f6a5e9"} -{"schema_version":1,"benchmark_version":"v1","run_id":"39c78597-f3db-4d54-b56e-9ed6e6b3d412","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":2,"started_at":"2026-07-23T06:41:28.119Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7785fb29a5fc11f07890b56ab20c7fc1235e8dca0651ca769eaf65c71fb42f5f","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:42:00.561Z","source_record_sha256":"a0b6604983a5cb7cb672c3b78d0ab62371334c2073304ac0557bc2198840ad9f"} -{"schema_version":1,"benchmark_version":"v1","run_id":"cf50c76f-316f-4793-877b-2f51474c6981","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":3,"started_at":"2026-07-23T06:42:00.561Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1cedb5127086b90b58363d555e4053bea288d9ebbab0631b1e0947958d156844","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:42:34.259Z","source_record_sha256":"b785d59be2fe7f270be2cf59048ad17214b13a2d6387d0d28a2176dfda9a7d91"} -{"schema_version":1,"benchmark_version":"v1","run_id":"5bf899ee-7846-44cf-8e8a-ebeb8bfbff65","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":1,"started_at":"2026-07-23T06:42:34.259Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:43:06.453Z","source_record_sha256":"4f635a3e21f7ec6a43b30500f049ff3208cce239ac1be3ace62b2c03ff541897"} -{"schema_version":1,"benchmark_version":"v1","run_id":"673ccda1-c286-49cd-9299-75ce17706e92","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":2,"started_at":"2026-07-23T06:43:06.453Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:43:43.650Z","source_record_sha256":"d6f433e283139085ada4f2415ec556e220ec66bd7f11fc504f2c51f0584e89c5"} -{"schema_version":1,"benchmark_version":"v1","run_id":"859aee1a-1670-4140-b9ac-91435124e6dc","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":3,"started_at":"2026-07-23T06:43:43.650Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:44:12.213Z","source_record_sha256":"db5ddc2bef71959a0abfe67ed9111c89e5d7a91db808692068481910ae1c49a2"} -{"schema_version":1,"benchmark_version":"v1","run_id":"70a47304-ae9e-4cb5-8331-5df6a8e83a43","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":1,"started_at":"2026-07-23T06:44:12.213Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e5da17c29aa94866e3abc93b7ae7169f66e151ae69893d952e89fcee78460d21","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:44:39.351Z","source_record_sha256":"3f8d52a2dd4bbf58e9f9be36f1e434b694eb757da69d8159f0243888a598eb4b"} -{"schema_version":1,"benchmark_version":"v1","run_id":"73355630-bc92-4e0a-a1c8-4f25f895a949","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":2,"started_at":"2026-07-23T06:44:39.351Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e5da17c29aa94866e3abc93b7ae7169f66e151ae69893d952e89fcee78460d21","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:45:14.542Z","source_record_sha256":"3a19bbc3b8533c11da052cc57dac3f7c2d2f5f04a555da0668605d0369dd1b6e"} -{"schema_version":1,"benchmark_version":"v1","run_id":"f05cea11-e23c-463a-98f1-e5c0b4f51e88","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":3,"started_at":"2026-07-23T06:45:14.542Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e5da17c29aa94866e3abc93b7ae7169f66e151ae69893d952e89fcee78460d21","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:45:40.767Z","source_record_sha256":"b1f8a12fd2910883acab983ca94f2d489b62427c0daa9759d3ed850c590cde0d"} -{"schema_version":1,"benchmark_version":"v1","run_id":"ae3e60aa-8ee9-4650-b3e2-6d906a42da8b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T06:45:40.767Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":null},"finished_at":"2026-07-23T06:46:30.257Z","source_record_sha256":"a662ab99b03038d4da9d8e89149c4d3180ec80a1ddba685c896089c14c660114"} -{"schema_version":1,"benchmark_version":"v1","run_id":"cc1eebe6-e68b-45ba-8f32-827f98baa240","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T06:46:30.257Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":null},"finished_at":"2026-07-23T06:47:52.318Z","source_record_sha256":"e8e64e193750c200d9940be5a99c552cad0ecb01e495121124aa07804da7b2d0"} -{"schema_version":1,"benchmark_version":"v1","run_id":"19382342-d937-46dc-ba4a-c71139081673","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T06:47:52.318Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":null},"finished_at":"2026-07-23T06:48:55.681Z","source_record_sha256":"487eeb948773c4df6e60d2333f6e54cfa05a7e143dbd87ea81953b8cfcf02f6f"} -{"schema_version":1,"benchmark_version":"v1","run_id":"48e281c3-3a7a-4327-9911-427072bc6fcd","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":1,"started_at":"2026-07-23T06:48:55.681Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"47a10fc51eaffa7d634cfeb673e3977424960edb526cf36296eab022d673e074","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:49:28.588Z","source_record_sha256":"8429545504e4b2eef7082c3c1fce7c561bd1be1de082be34fd96e2c5149690c5"} -{"schema_version":1,"benchmark_version":"v1","run_id":"d23e1fc9-03a5-4c76-8761-22c9de0e6231","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":2,"started_at":"2026-07-23T06:49:28.588Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"47a10fc51eaffa7d634cfeb673e3977424960edb526cf36296eab022d673e074","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":10,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:49:55.653Z","source_record_sha256":"7164bd4f310c9dda3a27e9ff046018d44184c359ec7c7f0db3880e16476eee65"} -{"schema_version":1,"benchmark_version":"v1","run_id":"262b0d23-181c-4b1c-8241-8cfd530dbb96","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":3,"started_at":"2026-07-23T06:49:55.653Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"47a10fc51eaffa7d634cfeb673e3977424960edb526cf36296eab022d673e074","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:50:22.831Z","source_record_sha256":"8620cabf967667758d4370970f8e6d82026af991bf28e2b2dfbf7fc602d9c6b1"} -{"schema_version":1,"benchmark_version":"v1","run_id":"3d110e2a-4ee1-47f7-a018-f5ae9d166e46","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":1,"started_at":"2026-07-23T06:50:22.831Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5d74987403b7f6009ae5a6da3cdcfac27dc35aa8f8dc5c333e65725c97f0cd0a","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:50:57.424Z","source_record_sha256":"1862e1000eaaa7832e064d1857baa8f7ecbe97a1739265e5b4666a0a4622ec05"} -{"schema_version":1,"benchmark_version":"v1","run_id":"b619dc10-8cc0-4413-9219-1aea84479b72","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":2,"started_at":"2026-07-23T06:50:57.424Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5d74987403b7f6009ae5a6da3cdcfac27dc35aa8f8dc5c333e65725c97f0cd0a","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:51:30.279Z","source_record_sha256":"c85b8d8d773950ae3a9274f3b0bb65b300c1639dc132b34c2a053531392c8a6c"} -{"schema_version":1,"benchmark_version":"v1","run_id":"0ab6d79a-6510-4b11-966c-7063b6646f73","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":3,"started_at":"2026-07-23T06:51:30.279Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5d74987403b7f6009ae5a6da3cdcfac27dc35aa8f8dc5c333e65725c97f0cd0a","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:51:58.663Z","source_record_sha256":"5c935b04a6e98677b01766c543ce5eea027cdf4b2f27d6f12677d78d1bbfa0a7"} -{"schema_version":1,"benchmark_version":"v1","run_id":"245b91ee-988d-474f-8fd1-82da7c62183a","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-23T06:51:58.663Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:52:30.880Z","source_record_sha256":"d86e68f4637cad728737fb4b0e4283ba4766a64304df4ed5b0843bb5ffc8185a"} -{"schema_version":1,"benchmark_version":"v1","run_id":"dc38691f-d8f6-40f8-9e77-9288c33a7e68","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-23T06:52:30.881Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:53:04.476Z","source_record_sha256":"c97ae4740814eb914f3859c0d16a16dec1298fa3ff5caf4ae386299734840da7"} -{"schema_version":1,"benchmark_version":"v1","run_id":"c1624700-5aba-4cfc-8a68-acabf3de024a","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T06:53:04.476Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":null},"finished_at":"2026-07-23T06:55:18.009Z","source_record_sha256":"d567ee1f534ced827b85bb21fc9c7c8b41325ea27130743f8e95d0396532f4bd"} -{"schema_version":1,"benchmark_version":"v1","run_id":"0e485876-22f8-4265-91e0-51990c3644e9","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":1,"started_at":"2026-07-23T06:55:18.009Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fa45aa3f97145ca8594d0a5ff1f0fba0f1abb533dfb4cfabbf84a07e4a76fba8","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false},"finished_at":"2026-07-23T06:55:45.466Z","source_record_sha256":"e8d8d305c2052be04c186b5fb864515ed6c6066c552f64a7df60e6b7aec061a6"} -{"schema_version":1,"benchmark_version":"v1","run_id":"ea693fd1-b596-40b1-bd16-eec213eecb12","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":2,"started_at":"2026-07-23T06:55:45.466Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fa45aa3f97145ca8594d0a5ff1f0fba0f1abb533dfb4cfabbf84a07e4a76fba8","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false},"finished_at":"2026-07-23T06:56:19.074Z","source_record_sha256":"0ebe2c1c3adbf481010ee766b90c35e838b2c50bfd738c866806459e727f03b5"} -{"schema_version":1,"benchmark_version":"v1","run_id":"abf65b68-8d35-453f-89b6-d30447a432e7","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":3,"started_at":"2026-07-23T06:56:19.074Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fa45aa3f97145ca8594d0a5ff1f0fba0f1abb533dfb4cfabbf84a07e4a76fba8","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false},"finished_at":"2026-07-23T06:56:53.486Z","source_record_sha256":"83d6dc9000481394c87614ff3a0dff0fcf553b0b50f7e2f7b556a5f986c5db06"} -{"schema_version":1,"benchmark_version":"v1","run_id":"a6ccdda6-dc1f-4765-873e-bd8c22197a6f","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":1,"started_at":"2026-07-23T06:56:53.486Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fcf3a79fe980fdf6f8b86e5034754e48d6405528a3a58b79d4ce0c7533cc5a2d","selection_grounded":true},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"selection_grounded":true,"required_parameters":["season"]},"parameters":{"league":39,"season":2025},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:57:26.157Z","source_record_sha256":"a592538886a42e90179278a71fc1bbc54d59f461ad659fcf68c646f88bd8af74"} -{"schema_version":1,"benchmark_version":"v1","run_id":"a025004c-c94c-444b-a2c9-bb176ed51ea3","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":2,"started_at":"2026-07-23T06:57:26.157Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fcf3a79fe980fdf6f8b86e5034754e48d6405528a3a58b79d4ce0c7533cc5a2d","selection_grounded":true},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"selection_grounded":true,"required_parameters":["season"]},"parameters":{"league":39,"season":2025},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:57:50.791Z","source_record_sha256":"b633b44046b54479076c360986169b474cd47cfd3baffcab34502a7a1a143287"} -{"schema_version":1,"benchmark_version":"v1","run_id":"f15c0986-0c0c-4eab-8dac-99cc4f6f7dea","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":3,"started_at":"2026-07-23T06:57:50.791Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fcf3a79fe980fdf6f8b86e5034754e48d6405528a3a58b79d4ce0c7533cc5a2d","selection_grounded":true},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"selection_grounded":true,"required_parameters":["season"]},"parameters":{"league":39,"season":2025},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:58:17.129Z","source_record_sha256":"ac3e0c86a18c7046786aeddaaee62ddca5f592e7d47b05c1076884e962aeb5de"} +{"schema_version":1,"benchmark_version":"v1","run_id":"ac88304b-0662-4d2b-bb4b-778a65d4d84d","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":1,"started_at":"2026-07-23T06:26:39.032Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8ff1444540e79a77ed35ac1b5cb0b6de9b5cbfad78aca4b2d6f896484d9fd1d4","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:27:00.228Z","source_record_sha256":"e5922113293b9cbaef44499c19030629c3cd20fb1d48276eb4ef820887c78c9d","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"5b911181-4d6c-472a-a2a0-4cb12e433f06","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":2,"started_at":"2026-07-23T06:27:00.228Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8ff1444540e79a77ed35ac1b5cb0b6de9b5cbfad78aca4b2d6f896484d9fd1d4","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:27:23.671Z","source_record_sha256":"daa24fddaeade477213fd0cfdafdaf2a07333f517ebf55f0d5d65dcb0bbb9f6b","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"a8268395-01a9-413b-b2af-ba30d2e71f9a","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":3,"started_at":"2026-07-23T06:27:23.671Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8ff1444540e79a77ed35ac1b5cb0b6de9b5cbfad78aca4b2d6f896484d9fd1d4","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:27:50.399Z","source_record_sha256":"75bc10aabffa921d7baefe47658cf7b371196e6ee76a29059d64acad8687a267","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"4e925856-364c-4ca7-b769-c8d4d43e85c2","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-23T06:27:50.399Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8c4e8e2ab56f10ce503086b98981d6be57f293b4043dec00a86237144a107a5b","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:28:19.862Z","source_record_sha256":"06c1b8e490bc19f2da0294fb66c4db895a9fa3337d8ed4c8ed35438b17bb33b6","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"9c2b558d-b803-4299-9b8d-7778e56d0fcb","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-23T06:28:19.862Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8c4e8e2ab56f10ce503086b98981d6be57f293b4043dec00a86237144a107a5b","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:28:47.286Z","source_record_sha256":"0e1e42c0295011ad188ea782a3eda5887fa763faab0dbe0f5c3f399da3a5e1f8","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"6762a6af-2002-4349-9239-89c6c62a597b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-23T06:28:47.286Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8c4e8e2ab56f10ce503086b98981d6be57f293b4043dec00a86237144a107a5b","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:29:13.981Z","source_record_sha256":"fdea4303bae16790c241f8d7743d7ced5d74783b6681a8c8792d6ca6f9dd0c4c","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"aa24674f-2a54-49ce-9652-95a7afc5a139","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-23T06:29:13.981Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e78e9a2e15b7817ec95c9a9f2dd8863f05b1049624a65dd87d0387e113e06d29","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:29:40.942Z","source_record_sha256":"f0206728537187674d6417f5baa0210e91dc590443b30c2f658957890ba782f1","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"d038e88d-be8e-4d2e-810c-ec4f155b6871","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-23T06:29:40.943Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e78e9a2e15b7817ec95c9a9f2dd8863f05b1049624a65dd87d0387e113e06d29","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:30:10.765Z","source_record_sha256":"ffac927ecd59c4b15f6433f9e4a697caea50585c7a1f31b3fbff9f5b726e0da3","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"c54fe3bb-0b0a-4758-beec-b7a5ce2bb540","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-23T06:30:10.765Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e78e9a2e15b7817ec95c9a9f2dd8863f05b1049624a65dd87d0387e113e06d29","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:30:38.654Z","source_record_sha256":"5e3fff916b17629db0000dccb4d7df9783a86ba705981beeac9a3f061819e2df","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"6bede096-05fb-47c7-a2eb-5a2f12dad17d","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":1,"started_at":"2026-07-23T06:30:38.654Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"048ef068d6abc7e544421421e151018f4495f28c24194b2c149e5b5fdc19a19f","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:31:10.595Z","source_record_sha256":"0e8d2e4dec3b62efe1c8dc241d837790f077bf855a59e34e146ba36fb9d87157","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"44918a28-652b-430d-ac7e-53daadb0e69b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":2,"started_at":"2026-07-23T06:31:10.595Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"048ef068d6abc7e544421421e151018f4495f28c24194b2c149e5b5fdc19a19f","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:31:44.443Z","source_record_sha256":"7d26eae6c76e79caa8b781a941faa89d732adc2180de4aca7b4a3ee2e1040558","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"a306ce69-4682-4778-a8e5-47b0c598e895","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":3,"started_at":"2026-07-23T06:31:44.443Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"86c0120a22c9e323afefb1ad57da1301f45e352b84c74122ca3bdcbbbd59ffb7","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:32:18.995Z","source_record_sha256":"42521c23757219360713aa5afcf829bf3b9f312d45a0d82f0534446f5c0fe40a","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"9377cc54-be7e-42b4-bf14-f4d14d013c54","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T06:32:18.995Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"301a7dca171878182649e610f180ae3c6f7db7f92485847b354714b924f969e4","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:32:52.395Z","source_record_sha256":"44b9472e368322522d38d3d194dc4a259d7d86e92fd6b3ebc65af3335e3d8fbf","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"c7815fa0-d6c8-4346-8463-1ae287470639","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T06:32:52.395Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"301a7dca171878182649e610f180ae3c6f7db7f92485847b354714b924f969e4","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:33:23.255Z","source_record_sha256":"1706ab297ad8e5469871e4c5dabc845a2a6bc64322945ea48954e30d43e3f099","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"844e8bbf-ae65-4047-9349-cbd1c1bf7b50","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T06:33:23.255Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"301a7dca171878182649e610f180ae3c6f7db7f92485847b354714b924f969e4","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:33:51.402Z","source_record_sha256":"7d86a9a686125488ce2a9c9827e7d8615a48aa4589c6f196894dec97f271cde3","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"cc32c600-4ef5-4fd8-b37c-6ae16c084f7e","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T06:33:51.402Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"850daeb30805dbaca59081825c8550458c4087938c88b78fb51cbc9e06aa928f","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:34:27.740Z","source_record_sha256":"e9825ffa157918d1b5bc4d9a98b4e2287745715b9d48bfb78f75999a5729c28d","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"de6093e2-ce59-4692-ae3c-da0801ac1675","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T06:34:27.740Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"850daeb30805dbaca59081825c8550458c4087938c88b78fb51cbc9e06aa928f","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:34:55.284Z","source_record_sha256":"14985dc8546323241dd9218ff7d0d7d60342c29e2f67882fc90ca3c9fdb9d946","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"a66e8303-b561-4d85-a2fa-baf46e0a953b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T06:34:55.284Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"850daeb30805dbaca59081825c8550458c4087938c88b78fb51cbc9e06aa928f","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:35:32.765Z","source_record_sha256":"187dfceaab10938e6587cda861b728789d536bf3b28d28b0cd4ba929ae1283d4","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"28718552-be81-486c-b313-75a5550ff3cc","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T06:35:32.765Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5783235b11728761fe41459111710a4cb2120d0d39e45f52a0cf8924ce54f453","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:36:03.749Z","source_record_sha256":"c6b25e36cd3b6ba2ed3b94a5041676171634d2ea7861d065351eab66243722ae","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"26e071b2-d139-4ee3-879c-5900578b1233","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T06:36:03.749Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5783235b11728761fe41459111710a4cb2120d0d39e45f52a0cf8924ce54f453","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:36:38.806Z","source_record_sha256":"bc95771cdcc075b15bbab52498067471e1ec7bf88d9d018f6906c25212f49e33","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"e2f6f4c2-a8e7-4967-98b1-1a8c29aa3410","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T06:36:38.806Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5783235b11728761fe41459111710a4cb2120d0d39e45f52a0cf8924ce54f453","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:37:14.052Z","source_record_sha256":"17081aa679bee8e6eea626e26319507fcf79576ed5dea927e450a7f1ec7a3d7a","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"fde5c3d4-8ff3-4813-a8c5-5e6a61ed3c2e","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-23T06:37:14.052Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"554e0d179f358300caff88da68d3aa9a452bb6b0b2fd0c9b3d7c4e72105773bf","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":false},"finished_at":"2026-07-23T06:37:44.040Z","source_record_sha256":"f91cf8ce9139290d5c6da675f04813d6e6ee21b29963bfe35b8997feb17074b6","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"cb085acd-e437-40df-9e07-e91343e2af9d","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-23T06:37:44.040Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"554e0d179f358300caff88da68d3aa9a452bb6b0b2fd0c9b3d7c4e72105773bf","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":false},"finished_at":"2026-07-23T06:38:14.212Z","source_record_sha256":"bf745350830585628ac0756e443a10066e9bb18c4ef7b5b5a4a19b4595e9e1bf","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"143359c4-589b-437f-ae30-9a99b9f1636b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-23T06:38:14.212Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"554e0d179f358300caff88da68d3aa9a452bb6b0b2fd0c9b3d7c4e72105773bf","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":false},"finished_at":"2026-07-23T06:38:55.364Z","source_record_sha256":"f4d2ba49e80c77e7e6ea19c287b2c01827f9ae5a37900e71462cdfabf3d2ca97","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"58e4b1fd-8de8-4f85-8db1-478b57309b00","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":1,"started_at":"2026-07-23T06:38:55.364Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d6fafa5dcc5b80de51ca0ad0cd3e975827e3281c0aa5dbb07e57ede5fb7c75d6","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:39:36.286Z","source_record_sha256":"b3b701d6ebe68b0c1cc8c50a5cb2d0dd1f99fd925aab1c5195e0757bd216732f","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"f498a668-e254-4a1f-a109-0b11043a9a77","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":2,"started_at":"2026-07-23T06:39:36.286Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d6fafa5dcc5b80de51ca0ad0cd3e975827e3281c0aa5dbb07e57ede5fb7c75d6","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:40:06.523Z","source_record_sha256":"0b053eb5eb7799ea2e73b49aae1f3c395b57cb29bb4357530c73f123d47db293","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"3b48193b-6026-491e-8472-51034ae93749","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":3,"started_at":"2026-07-23T06:40:06.523Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d6fafa5dcc5b80de51ca0ad0cd3e975827e3281c0aa5dbb07e57ede5fb7c75d6","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:40:52.281Z","source_record_sha256":"ed953a426b8091ac61b9d3f826d950ca92c83ece434f088c6a5144b0378dd77d","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"3d70bc1e-3063-41e6-8b57-40c6e6b963e2","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":1,"started_at":"2026-07-23T06:40:52.281Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7785fb29a5fc11f07890b56ab20c7fc1235e8dca0651ca769eaf65c71fb42f5f","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:41:28.119Z","source_record_sha256":"a95a691983bf0e0c7d529f7562a557be3507430d675a4bd3d839df3216f6a5e9","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"39c78597-f3db-4d54-b56e-9ed6e6b3d412","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":2,"started_at":"2026-07-23T06:41:28.119Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7785fb29a5fc11f07890b56ab20c7fc1235e8dca0651ca769eaf65c71fb42f5f","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:42:00.561Z","source_record_sha256":"a0b6604983a5cb7cb672c3b78d0ab62371334c2073304ac0557bc2198840ad9f","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"cf50c76f-316f-4793-877b-2f51474c6981","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":3,"started_at":"2026-07-23T06:42:00.561Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1cedb5127086b90b58363d555e4053bea288d9ebbab0631b1e0947958d156844","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:42:34.259Z","source_record_sha256":"b785d59be2fe7f270be2cf59048ad17214b13a2d6387d0d28a2176dfda9a7d91","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"5bf899ee-7846-44cf-8e8a-ebeb8bfbff65","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":1,"started_at":"2026-07-23T06:42:34.259Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:43:06.453Z","source_record_sha256":"4f635a3e21f7ec6a43b30500f049ff3208cce239ac1be3ace62b2c03ff541897","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"673ccda1-c286-49cd-9299-75ce17706e92","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":2,"started_at":"2026-07-23T06:43:06.453Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:43:43.650Z","source_record_sha256":"d6f433e283139085ada4f2415ec556e220ec66bd7f11fc504f2c51f0584e89c5","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"859aee1a-1670-4140-b9ac-91435124e6dc","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":3,"started_at":"2026-07-23T06:43:43.650Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:44:12.213Z","source_record_sha256":"db5ddc2bef71959a0abfe67ed9111c89e5d7a91db808692068481910ae1c49a2","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"70a47304-ae9e-4cb5-8331-5df6a8e83a43","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":1,"started_at":"2026-07-23T06:44:12.213Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e5da17c29aa94866e3abc93b7ae7169f66e151ae69893d952e89fcee78460d21","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:44:39.351Z","source_record_sha256":"3f8d52a2dd4bbf58e9f9be36f1e434b694eb757da69d8159f0243888a598eb4b","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"73355630-bc92-4e0a-a1c8-4f25f895a949","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":2,"started_at":"2026-07-23T06:44:39.351Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e5da17c29aa94866e3abc93b7ae7169f66e151ae69893d952e89fcee78460d21","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:45:14.542Z","source_record_sha256":"3a19bbc3b8533c11da052cc57dac3f7c2d2f5f04a555da0668605d0369dd1b6e","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"f05cea11-e23c-463a-98f1-e5c0b4f51e88","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":3,"started_at":"2026-07-23T06:45:14.542Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e5da17c29aa94866e3abc93b7ae7169f66e151ae69893d952e89fcee78460d21","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:45:40.767Z","source_record_sha256":"b1f8a12fd2910883acab983ca94f2d489b62427c0daa9759d3ed850c590cde0d","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"ae3e60aa-8ee9-4650-b3e2-6d906a42da8b","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T06:45:40.767Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true},"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":null},"finished_at":"2026-07-23T06:46:30.257Z","source_record_sha256":"a662ab99b03038d4da9d8e89149c4d3180ec80a1ddba685c896089c14c660114","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"cc1eebe6-e68b-45ba-8f32-827f98baa240","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T06:46:30.257Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true},"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":null},"finished_at":"2026-07-23T06:47:52.318Z","source_record_sha256":"e8e64e193750c200d9940be5a99c552cad0ecb01e495121124aa07804da7b2d0","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"19382342-d937-46dc-ba4a-c71139081673","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T06:47:52.318Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true},"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":null},"finished_at":"2026-07-23T06:48:55.681Z","source_record_sha256":"487eeb948773c4df6e60d2333f6e54cfa05a7e143dbd87ea81953b8cfcf02f6f","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"48e281c3-3a7a-4327-9911-427072bc6fcd","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":1,"started_at":"2026-07-23T06:48:55.681Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"47a10fc51eaffa7d634cfeb673e3977424960edb526cf36296eab022d673e074","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:49:28.588Z","source_record_sha256":"8429545504e4b2eef7082c3c1fce7c561bd1be1de082be34fd96e2c5149690c5","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"d23e1fc9-03a5-4c76-8761-22c9de0e6231","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":2,"started_at":"2026-07-23T06:49:28.588Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"47a10fc51eaffa7d634cfeb673e3977424960edb526cf36296eab022d673e074","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:49:55.653Z","source_record_sha256":"7164bd4f310c9dda3a27e9ff046018d44184c359ec7c7f0db3880e16476eee65","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"262b0d23-181c-4b1c-8241-8cfd530dbb96","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":3,"started_at":"2026-07-23T06:49:55.653Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"47a10fc51eaffa7d634cfeb673e3977424960edb526cf36296eab022d673e074","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:50:22.831Z","source_record_sha256":"8620cabf967667758d4370970f8e6d82026af991bf28e2b2dfbf7fc602d9c6b1","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"3d110e2a-4ee1-47f7-a018-f5ae9d166e46","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":1,"started_at":"2026-07-23T06:50:22.831Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5d74987403b7f6009ae5a6da3cdcfac27dc35aa8f8dc5c333e65725c97f0cd0a","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:50:57.424Z","source_record_sha256":"1862e1000eaaa7832e064d1857baa8f7ecbe97a1739265e5b4666a0a4622ec05","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"b619dc10-8cc0-4413-9219-1aea84479b72","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":2,"started_at":"2026-07-23T06:50:57.424Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5d74987403b7f6009ae5a6da3cdcfac27dc35aa8f8dc5c333e65725c97f0cd0a","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:51:30.279Z","source_record_sha256":"c85b8d8d773950ae3a9274f3b0bb65b300c1639dc132b34c2a053531392c8a6c","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"0ab6d79a-6510-4b11-966c-7063b6646f73","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":3,"started_at":"2026-07-23T06:51:30.279Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5d74987403b7f6009ae5a6da3cdcfac27dc35aa8f8dc5c333e65725c97f0cd0a","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:51:58.663Z","source_record_sha256":"5c935b04a6e98677b01766c543ce5eea027cdf4b2f27d6f12677d78d1bbfa0a7","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"245b91ee-988d-474f-8fd1-82da7c62183a","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-23T06:51:58.663Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:52:30.880Z","source_record_sha256":"d86e68f4637cad728737fb4b0e4283ba4766a64304df4ed5b0843bb5ffc8185a","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"dc38691f-d8f6-40f8-9e77-9288c33a7e68","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-23T06:52:30.881Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:53:04.476Z","source_record_sha256":"c97ae4740814eb914f3859c0d16a16dec1298fa3ff5caf4ae386299734840da7","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"c1624700-5aba-4cfc-8a68-acabf3de024a","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T06:53:04.476Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":null},"finished_at":"2026-07-23T06:55:18.009Z","source_record_sha256":"d567ee1f534ced827b85bb21fc9c7c8b41325ea27130743f8e95d0396532f4bd","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"0e485876-22f8-4265-91e0-51990c3644e9","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":1,"started_at":"2026-07-23T06:55:18.009Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fa45aa3f97145ca8594d0a5ff1f0fba0f1abb533dfb4cfabbf84a07e4a76fba8","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":false},"finished_at":"2026-07-23T06:55:45.466Z","source_record_sha256":"e8d8d305c2052be04c186b5fb864515ed6c6066c552f64a7df60e6b7aec061a6","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"ea693fd1-b596-40b1-bd16-eec213eecb12","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":2,"started_at":"2026-07-23T06:55:45.466Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fa45aa3f97145ca8594d0a5ff1f0fba0f1abb533dfb4cfabbf84a07e4a76fba8","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":false},"finished_at":"2026-07-23T06:56:19.074Z","source_record_sha256":"0ebe2c1c3adbf481010ee766b90c35e838b2c50bfd738c866806459e727f03b5","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"abf65b68-8d35-453f-89b6-d30447a432e7","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":3,"started_at":"2026-07-23T06:56:19.074Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fa45aa3f97145ca8594d0a5ff1f0fba0f1abb533dfb4cfabbf84a07e4a76fba8","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":false},"finished_at":"2026-07-23T06:56:53.486Z","source_record_sha256":"83d6dc9000481394c87614ff3a0dff0fcf553b0b50f7e2f7b556a5f986c5db06","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"a6ccdda6-dc1f-4765-873e-bd8c22197a6f","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":1,"started_at":"2026-07-23T06:56:53.486Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fcf3a79fe980fdf6f8b86e5034754e48d6405528a3a58b79d4ce0c7533cc5a2d","selection_grounded":true},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:57:26.157Z","source_record_sha256":"a592538886a42e90179278a71fc1bbc54d59f461ad659fcf68c646f88bd8af74","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"a025004c-c94c-444b-a2c9-bb176ed51ea3","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":2,"started_at":"2026-07-23T06:57:26.157Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fcf3a79fe980fdf6f8b86e5034754e48d6405528a3a58b79d4ce0c7533cc5a2d","selection_grounded":true},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:57:50.791Z","source_record_sha256":"b633b44046b54479076c360986169b474cd47cfd3baffcab34502a7a1a143287","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"f15c0986-0c0c-4eab-8dac-99cc4f6f7dea","model":"gpt-5.6-sol","metadata":{"adapter_revision":"72672d22f3852349e166ab930efa3617945d82f0/codex-cli-0.144.1/medium","toolkit_revision":"fd08165e979a8ffaa030c56a4bd4853d805bf095","task_set_sha256":"67ce3685c911781f91d9978696eb4ec64192aa03da5df8fb320f8680b6e411ba","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"lane":"model","model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"50813b2a37d9024abddde21c614110c3fa926c66df833d039bf68a69fe9c7349","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":3,"started_at":"2026-07-23T06:57:50.791Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fcf3a79fe980fdf6f8b86e5034754e48d6405528a3a58b79d4ce0c7533cc5a2d","selection_grounded":true},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true},"finished_at":"2026-07-23T06:58:17.129Z","source_record_sha256":"ac3e0c86a18c7046786aeddaaee62ddca5f592e7d47b05c1076884e962aeb5de","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} diff --git a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.runs.jsonl b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.runs.jsonl index 730278e..4877967 100644 --- a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.runs.jsonl +++ b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.runs.jsonl @@ -1,54 +1,54 @@ -{"schema_version":1,"benchmark_version":"v1","run_id":"23187609-52d7-44b6-b60e-91f68d32777c","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":1,"started_at":"2026-07-23T09:40:25.828Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"28e564a72f02a0b6744c9086e328ba4b829e39246dca5821577adcb20a5f3a72","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:40:51.455Z","source_record_sha256":"0394fb5b9e153aa501a8d781643292841d812c05c0986a4661ed3866e242ab03"} -{"schema_version":1,"benchmark_version":"v1","run_id":"d2bfdd97-1ac6-4388-aab6-dc04bd389b4d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":2,"started_at":"2026-07-23T09:40:51.455Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"28e564a72f02a0b6744c9086e328ba4b829e39246dca5821577adcb20a5f3a72","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:41:19.367Z","source_record_sha256":"a132a571a86762281fbf49fdd7506aaa19fc15a736d6837127da51c0cab30f16"} -{"schema_version":1,"benchmark_version":"v1","run_id":"9dc77ec7-6fea-42e6-8786-f8970a33da5d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":3,"started_at":"2026-07-23T09:41:19.367Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"28e564a72f02a0b6744c9086e328ba4b829e39246dca5821577adcb20a5f3a72","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:41:45.578Z","source_record_sha256":"9b8eb9c3f5b12d221b1a3051a045f8cb17bdf481f796131e344089ac7ad58e16"} -{"schema_version":1,"benchmark_version":"v1","run_id":"98d4488a-8f0e-48fe-8743-ed08df8940f8","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-23T09:41:45.578Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"65423dcd48e9ba551f0500805a38535f2428d65ef165a26c778463c01921a519","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:42:10.683Z","source_record_sha256":"7dc8a173d225a7254741f7a0591960ebcf3c3ff7c8a7d18ba52e38dae92fef57"} -{"schema_version":1,"benchmark_version":"v1","run_id":"450cf5b6-8404-4be2-b73e-c28a6ae3227d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-23T09:42:10.683Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"65423dcd48e9ba551f0500805a38535f2428d65ef165a26c778463c01921a519","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:42:35.790Z","source_record_sha256":"088cb3c49cd699c6f9a9033404a71866daac47829c9a2ec696e3f5aed6bb6e89"} -{"schema_version":1,"benchmark_version":"v1","run_id":"456b1477-ec0a-4772-887c-8971ac437a95","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-23T09:42:35.790Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"65423dcd48e9ba551f0500805a38535f2428d65ef165a26c778463c01921a519","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:43:04.020Z","source_record_sha256":"6f10781492c58eeb6c351dfa9db214028bc0558b31b1e442e0568aa5f9611edf"} -{"schema_version":1,"benchmark_version":"v1","run_id":"850f6ae9-de46-4e92-a722-81edd34dd191","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-23T09:43:04.020Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"b98b9cb4476c56c6553ff1ea2bfd5e1976911c8c14142f76ef97c691d368414f","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":100,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:43:40.323Z","source_record_sha256":"18b9cf40ca54aac96152a60268192d8144aac03c1ab87fa134dd57bfffd5ce1e"} -{"schema_version":1,"benchmark_version":"v1","run_id":"378c4fcc-1e86-4149-8b38-82d8230e5d11","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-23T09:43:40.323Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"b98b9cb4476c56c6553ff1ea2bfd5e1976911c8c14142f76ef97c691d368414f","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:44:16.949Z","source_record_sha256":"f36668206832ea60ec935daeee36c7f48d9b4fad9c4a2740d6b347e3c6cda748"} -{"schema_version":1,"benchmark_version":"v1","run_id":"beb58cdc-281f-42cf-a388-9d9f30fa74be","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-23T09:44:16.949Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"b98b9cb4476c56c6553ff1ea2bfd5e1976911c8c14142f76ef97c691d368414f","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:44:53.121Z","source_record_sha256":"07f3fe927e57423d7eae4fa52ee878da564f4142ea03fe803b0b3307fb1a319d"} -{"schema_version":1,"benchmark_version":"v1","run_id":"b602eea9-1fa5-4b43-adf2-59adff282a87","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":1,"started_at":"2026-07-23T09:44:53.121Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"4ca8303398d379bc94e129f5da06d77fc7aebb56b424abeea7e4a3feb3910636","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20AI","from":"2026-07-16","to":"2026-07-23T23:59:59+08:00","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:45:25.145Z","source_record_sha256":"06a9d53c3fec8a8ec22cd6fb65cca82d18632ad97ceefbff36c9c18572724017"} -{"schema_version":1,"benchmark_version":"v1","run_id":"bfc0ab6d-9a15-4768-a672-e466f0d23414","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":2,"started_at":"2026-07-23T09:45:25.145Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"4ca8303398d379bc94e129f5da06d77fc7aebb56b424abeea7e4a3feb3910636","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20AI","from":"2026-07-16","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:46:05.132Z","source_record_sha256":"406f86260cf08286855e783c689cd90f3e1f976e84b0277cbff522aa6526c029"} -{"schema_version":1,"benchmark_version":"v1","run_id":"c2bab891-e16c-49ce-af35-9ba5220ef929","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":3,"started_at":"2026-07-23T09:46:05.132Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"4ca8303398d379bc94e129f5da06d77fc7aebb56b424abeea7e4a3feb3910636","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"%22artificial%20intelligence%22%20OR%20%22generative%20AI%22","from":"2026-07-20","to":"2026-07-23T23:59:59","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:46:35.462Z","source_record_sha256":"e0be7b97e788ce36f9643c04d2f4911b3bba211890d0915b07f01e579b6ff811"} -{"schema_version":1,"benchmark_version":"v1","run_id":"7c3b1d86-cdf3-42aa-94d0-bfa2024284ec","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T09:46:35.462Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7935de8cea6f84dce9a4df111df427079b97af7dd653fecdcba159947c48d136","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:47:02.518Z","source_record_sha256":"24485ca6593e17388a3346e0f9209b4846153324491a991e8ea7f222a144ab4a"} -{"schema_version":1,"benchmark_version":"v1","run_id":"0dfd6d1c-ca90-4916-8b8f-4ae4d3e7f273","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T09:47:02.518Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7935de8cea6f84dce9a4df111df427079b97af7dd653fecdcba159947c48d136","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:47:32.328Z","source_record_sha256":"f7107e2e7957bb86f20b63cd1c0eb6a4de8d5158aa8f68d2f1d8a3ba01fe3179"} -{"schema_version":1,"benchmark_version":"v1","run_id":"ce6fbc71-c894-461a-adb1-4a607e7d4bdc","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T09:47:32.328Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7935de8cea6f84dce9a4df111df427079b97af7dd653fecdcba159947c48d136","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:47:58.422Z","source_record_sha256":"f2cfd462aceb106b5154662f6f3e86d2e0d7afcc77db75ad43ba5d04ccf1aa56"} -{"schema_version":1,"benchmark_version":"v1","run_id":"b43508b4-6a54-487e-b3fd-dd511201af19","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T09:47:58.422Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"0c9d6a51ec70c89cafcbd7ccfca456093271999f98a8c42b6423e996fc6f48a4","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:48:24.635Z","source_record_sha256":"4da0464a226f7e73e86a16071182c1ecf42f06113bf5d0c77ee7068624a913db"} -{"schema_version":1,"benchmark_version":"v1","run_id":"704ee017-af2e-469b-a055-de7b367c3ffb","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T09:48:24.635Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"0c9d6a51ec70c89cafcbd7ccfca456093271999f98a8c42b6423e996fc6f48a4","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:48:55.300Z","source_record_sha256":"fb66f6302a91edfe07cfb60666e18fc797143c7c62980963704afd6dfdffc5fb"} -{"schema_version":1,"benchmark_version":"v1","run_id":"495bda63-87f8-4ada-bcd2-a2a4076f6431","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T09:48:55.300Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"0c9d6a51ec70c89cafcbd7ccfca456093271999f98a8c42b6423e996fc6f48a4","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:49:26.689Z","source_record_sha256":"d0640e38ed193059bf9d325cd9f756d76ff72556413bce3721d34f795afd0f0f"} -{"schema_version":1,"benchmark_version":"v1","run_id":"10a88d8d-1053-4d50-bab5-4b7114dd0ee7","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T09:49:26.689Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3dcb23db74a0c80244544838d4968450a29fa18516b5d8c8e17b0a38af6919cd","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:49:55.414Z","source_record_sha256":"35054cbc3c8f2f9e2e047104c7fce1e4b687d34d2a9fe24367477a5158e97af4"} -{"schema_version":1,"benchmark_version":"v1","run_id":"252d6b99-8605-4a66-aaec-d2b63a4fe36f","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T09:49:55.414Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3dcb23db74a0c80244544838d4968450a29fa18516b5d8c8e17b0a38af6919cd","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:50:31.198Z","source_record_sha256":"c5ed77b70f27ba4bba45540ec1d3ace713757cd7f600ec78f5aae4a0dd2b244b"} -{"schema_version":1,"benchmark_version":"v1","run_id":"3b2b6bf9-ad89-4298-92a7-3622fb115a13","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T09:50:31.198Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3dcb23db74a0c80244544838d4968450a29fa18516b5d8c8e17b0a38af6919cd","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:51:04.973Z","source_record_sha256":"ea2151bd097ea5a08d2fb3134f458e21af30728d07a85b1b08d3df56f4b816af"} -{"schema_version":1,"benchmark_version":"v1","run_id":"aaac2eb2-646e-42ff-ae32-1326ee9682b4","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-23T09:51:04.973Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2798b2551f89d750e90bc78b840801a716942fd2cc7241b33575fc13b5e1cc98","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T09:51:37.056Z","source_record_sha256":"6236fb0c65c08f023b41589cf7aed9f3194cb6782081175abfa6382589e9a535"} -{"schema_version":1,"benchmark_version":"v1","run_id":"3e947ba6-ca5a-47dc-abe3-404e9464239b","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-23T09:51:37.056Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"954802b741ae1296ad98d5a92faa78b0b02b7549ad05ec06ce4544a982fca525","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T09:52:06.521Z","source_record_sha256":"8ea3c9edf5c876f7243d43804235e2665a74f7cc290af4b8ba61e605a3ca21a5"} -{"schema_version":1,"benchmark_version":"v1","run_id":"69e303d3-d9ea-4f86-9fa3-0b4bd9c3087c","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-23T09:52:06.521Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"954802b741ae1296ad98d5a92faa78b0b02b7549ad05ec06ce4544a982fca525","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"8.8.8.8"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T09:52:38.206Z","source_record_sha256":"8d56e8fc444982b3c01b8ecf2017638d7194161bcebbf1ab788bc5169290a2d5"} -{"schema_version":1,"benchmark_version":"v1","run_id":"16873a4e-2cd3-47d9-845e-0067aeb15a42","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":1,"started_at":"2026-07-23T09:52:38.206Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2099bfa8887678c84902e7ec565c63edeef86ac5127e75b290713f8ff3a4f197","selection_grounded":true},"selection":{"tool_id":"coinmarketcap.tools.price-conversion.execute.v1.2b99a4df"},"inspection":{"selection_grounded":true,"required_parameters":["amount"]},"parameters":{"amount":1,"id":"1","convert":"USD"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:53:13.658Z","source_record_sha256":"ee67bc1787845c7185516013d6c9a0f7e5ac4e6116d9ef80bb8fb9cbb0d926ec"} -{"schema_version":1,"benchmark_version":"v1","run_id":"190b85e5-c226-468b-91d5-99a08a96768d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":2,"started_at":"2026-07-23T09:53:13.658Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2099bfa8887678c84902e7ec565c63edeef86ac5127e75b290713f8ff3a4f197","selection_grounded":true},"selection":{"tool_id":"coinmarketcap.tools.price-conversion.execute.v1.2b99a4df"},"inspection":{"selection_grounded":true,"required_parameters":["amount"]},"parameters":{"amount":1,"id":"1","convert":"USD"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:53:52.934Z","source_record_sha256":"d9583032847d73d7c87252026831b092ad84839a82152af3ba4f8958db2484e8"} -{"schema_version":1,"benchmark_version":"v1","run_id":"533afeba-6917-4bdb-83f5-6314d89c1207","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":3,"started_at":"2026-07-23T09:53:52.934Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2099bfa8887678c84902e7ec565c63edeef86ac5127e75b290713f8ff3a4f197","selection_grounded":true},"selection":{"tool_id":"coinmarketcap.tools.price-conversion.execute.v1.2b99a4df"},"inspection":{"selection_grounded":true,"required_parameters":["amount"]},"parameters":{"amount":1,"id":"1","convert":"USD"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:54:29.273Z","source_record_sha256":"c26269c83b8733a84b0b0fa0afd38a548f64e455d12753e98f3064158726eb8d"} -{"schema_version":1,"benchmark_version":"v1","run_id":"4e7bf26c-1907-441c-8103-49071ebcef84","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":1,"started_at":"2026-07-23T09:54:29.273Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9eb927c2b3d94415d2534d541127a937adb20aca4f0cefdc0b7c6948c064c9f9","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:54:58.710Z","source_record_sha256":"cbf18b1fae7e417ae99b83844c38d0c8ade8811fb732e5ccecde52d12d1ae414"} -{"schema_version":1,"benchmark_version":"v1","run_id":"7c57619b-bf15-4504-8587-08a778c4c502","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":2,"started_at":"2026-07-23T09:54:58.710Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9eb927c2b3d94415d2534d541127a937adb20aca4f0cefdc0b7c6948c064c9f9","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:55:30.042Z","source_record_sha256":"deeb3853b2c38377a75619dfad3b69859fb62bf5e4e5ebc5212f9464b79c9a08"} -{"schema_version":1,"benchmark_version":"v1","run_id":"b0c765a8-abef-4d88-b1d1-583d7a7083f5","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":3,"started_at":"2026-07-23T09:55:30.042Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9eb927c2b3d94415d2534d541127a937adb20aca4f0cefdc0b7c6948c064c9f9","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:56:01.974Z","source_record_sha256":"d4d546dc07a8cfebca12c2b75db4e5ba54f00289b241423a33213091bf58efa1"} -{"schema_version":1,"benchmark_version":"v1","run_id":"3573e6ae-8c2b-4c45-9b9f-a7dbdaf459d8","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":1,"started_at":"2026-07-23T09:56:01.974Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d664c4f2fd3f9d9e0d0d8f66a450ec57e2aa075d57bba3883ac678920ca072c6","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:56:42.264Z","source_record_sha256":"edd01cfdc6a189fb72df609f9a303603b01a19fad462ba18acb337cc42591b97"} -{"schema_version":1,"benchmark_version":"v1","run_id":"7835af93-e41c-428a-995f-4447571133fe","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":2,"started_at":"2026-07-23T09:56:42.264Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d664c4f2fd3f9d9e0d0d8f66a450ec57e2aa075d57bba3883ac678920ca072c6","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:57:10.056Z","source_record_sha256":"4f1e3e4bf201c9a85fe1a4c523b17c3b8e06c57dc637a19767f1872cc9de5d15"} -{"schema_version":1,"benchmark_version":"v1","run_id":"73d62bf3-09c9-4c6f-af68-cbe62631e4c3","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":3,"started_at":"2026-07-23T09:57:10.056Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d664c4f2fd3f9d9e0d0d8f66a450ec57e2aa075d57bba3883ac678920ca072c6","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:57:36.396Z","source_record_sha256":"3be7b15dcd9a6869bf6202dee1c262faa553589f0f1d34ed4fc3b0d3d860b9ef"} -{"schema_version":1,"benchmark_version":"v1","run_id":"0e125c12-7a7b-4c87-9e64-54a8be79f774","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":1,"started_at":"2026-07-23T09:57:36.396Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8d10f08954c40da564662fec151d75440ec8391aefdf74cfc8e2c32a1ee26dbb","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:58:04.308Z","source_record_sha256":"40c87aa07d78b9b835112c13584563870d4b64eb7a23230de8085d7a344f1851"} -{"schema_version":1,"benchmark_version":"v1","run_id":"7a2ca54b-b346-4343-b6fb-54a264c5b778","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":2,"started_at":"2026-07-23T09:58:04.308Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8d10f08954c40da564662fec151d75440ec8391aefdf74cfc8e2c32a1ee26dbb","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:58:29.894Z","source_record_sha256":"b0b04c429f3abe332a69253f6d6e831d1702da29097ab1285883cf4945269647"} -{"schema_version":1,"benchmark_version":"v1","run_id":"e98dc7fd-0ed4-4487-a753-c6c197254603","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":3,"started_at":"2026-07-23T09:58:29.894Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8d10f08954c40da564662fec151d75440ec8391aefdf74cfc8e2c32a1ee26dbb","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:58:58.074Z","source_record_sha256":"4dc549878c034c8440e01e24b20ced68fd25617caef32d35519f5ee2750ec0d3"} -{"schema_version":1,"benchmark_version":"v1","run_id":"a22d10aa-0f23-4f76-b2a2-692576b4b2a2","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T09:58:58.074Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected"},"finished_at":"2026-07-23T09:59:40.997Z","source_record_sha256":"8e0f1a8dd393b2e3e3f8480ede541b2effe403ed92acfa0e2fab9f4269ed9246"} -{"schema_version":1,"benchmark_version":"v1","run_id":"97361151-f794-478a-8b90-e8e544c484f7","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T09:59:40.997Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:00:15.027Z","source_record_sha256":"f9b49fcdc5f1e3d00fdb7185a218da04e37fc6e60da566fc3aa2e9073ab191a1"} -{"schema_version":1,"benchmark_version":"v1","run_id":"e877975e-2fff-4ec9-a487-0bddba738e4f","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T10:00:15.027Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected"},"finished_at":"2026-07-23T10:00:55.710Z","source_record_sha256":"d6bd4b821637a43316f544a37fc975c0b59797e77cd24ef7fae03280ddaa3721"} -{"schema_version":1,"benchmark_version":"v1","run_id":"d9fe796d-cd3d-4d0c-8bb3-0cd1fc7e128d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":1,"started_at":"2026-07-23T10:00:55.710Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"406f82f0151e2aab38d28e5369d6e3b6b8f1e5a40414bd6ea7cafc3eb9f40259","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:01:21.601Z","source_record_sha256":"61bcf3bc4829f5e23fb25aa989ab9e25e8bdc1b9066d6e8e7dae7ec7d3fe39a4"} -{"schema_version":1,"benchmark_version":"v1","run_id":"e3d0d1fd-e32c-4701-b87a-7d2907aae773","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":2,"started_at":"2026-07-23T10:01:21.601Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"406f82f0151e2aab38d28e5369d6e3b6b8f1e5a40414bd6ea7cafc3eb9f40259","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:01:46.789Z","source_record_sha256":"0c98fe2b9fab21c84a6018f2c17577e36c6f1db959d8959006e8a27a1d509c9d"} -{"schema_version":1,"benchmark_version":"v1","run_id":"109d0ded-433f-4ef6-a7ed-f8cf19b897be","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":3,"started_at":"2026-07-23T10:01:46.789Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"413e70c9b613711904d82925006e89c88e507640e6650646084db3a76f64b8ce","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge","safesearch":"strict","count":20,"search_lang":"en","country":"us","spellcheck":"1"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:02:17.125Z","source_record_sha256":"25f730ed07cea7449205d61271acef2544d40eae7a33b9a76bc2ed2d6eb53a7d"} -{"schema_version":1,"benchmark_version":"v1","run_id":"e55b1799-5e11-481c-a18f-e4be01b4e744","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":1,"started_at":"2026-07-23T10:02:17.125Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5d7380bab4c7ce4e2289dc764fc2cae8ba0d5cbe190f62fc09dee13517796d3d","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:02:44.222Z","source_record_sha256":"b47fbff96f2cbd4f32ae3c1d02eae4ad3bb17da02f61fd7f1d9018cef02ce863"} -{"schema_version":1,"benchmark_version":"v1","run_id":"a88a3226-9842-4032-ac67-fcae35d59a03","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":2,"started_at":"2026-07-23T10:02:44.222Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5d7380bab4c7ce4e2289dc764fc2cae8ba0d5cbe190f62fc09dee13517796d3d","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:03:10.533Z","source_record_sha256":"c44ab638aca59dd5eb6650b581ed3e0bc1b085e81a01b9e06fa1ebde07edc1ad"} -{"schema_version":1,"benchmark_version":"v1","run_id":"bd6bfb47-687b-4d08-9a70-f019316e623a","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":3,"started_at":"2026-07-23T10:03:10.533Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5d7380bab4c7ce4e2289dc764fc2cae8ba0d5cbe190f62fc09dee13517796d3d","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:03:37.293Z","source_record_sha256":"f1f1cec153e7696804111aaa35815a177088c70905bb2e81effa38b9f7dc0afa"} -{"schema_version":1,"benchmark_version":"v1","run_id":"c28bc1c8-94d0-4bed-a562-af06253fc011","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-23T10:03:37.293Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:04:19.806Z","source_record_sha256":"558fe12898a5153ca789f01f8d5dff4733850bbd892e4e13ceaecd52a9909ccd"} -{"schema_version":1,"benchmark_version":"v1","run_id":"9eff8a69-cb9a-4931-ae30-4230a29c6b2c","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-23T10:04:19.806Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:04:54.838Z","source_record_sha256":"222b9822b0bf52fdd42c8e4ef55761abc6715c95ef4b0a67a99937a60c59da23"} -{"schema_version":1,"benchmark_version":"v1","run_id":"81d9ac77-2d8f-4363-be1a-09193951a1b1","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T10:04:54.838Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:05:33.311Z","source_record_sha256":"8b485bf1ce1cf1c78376b3b005c724bc529b3ddd6789b540138dbae1c9a0303d"} -{"schema_version":1,"benchmark_version":"v1","run_id":"abc19032-2dc3-46fc-a861-05132c7eda65","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":1,"started_at":"2026-07-23T10:05:33.311Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f9568410d64286b9cc625ae8f4a38b3b350dee873f626cc44bb01cd92e5e7652","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T10:06:00.664Z","source_record_sha256":"9841f36f25b9775a95a122bfd640f89bbe141c6a0b8cda802a39479dd4b7a510"} -{"schema_version":1,"benchmark_version":"v1","run_id":"66977b1e-3302-469e-9e30-590d4b8cd542","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":2,"started_at":"2026-07-23T10:06:00.664Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f9568410d64286b9cc625ae8f4a38b3b350dee873f626cc44bb01cd92e5e7652","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T10:06:50.608Z","source_record_sha256":"5233af85fb24de9f413a64d9c076732b4b2f658cb5ee949299dc62dfe2a97544"} -{"schema_version":1,"benchmark_version":"v1","run_id":"b3001ec6-f8da-4dcc-b420-a40d0991337b","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":3,"started_at":"2026-07-23T10:06:50.608Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f9568410d64286b9cc625ae8f4a38b3b350dee873f626cc44bb01cd92e5e7652","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T10:07:33.561Z","source_record_sha256":"a5bad4a51d0d339497fed6f904985b0ebb406c88b9a791b07a9a735845003e09"} -{"schema_version":1,"benchmark_version":"v1","run_id":"7e26fd5b-b2f6-40bb-90dc-0d32665ffadd","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":1,"started_at":"2026-07-23T10:07:33.561Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66bd65e09a5a8cadf11017997e8362fae34ffb37649c18058a462de424f6e78b","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.98a153bd"},"inspection":{"selection_grounded":true,"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:08:06.053Z","source_record_sha256":"4f5f11c913c72853033c5cc8b76b8a599895f5000a28ca566ecbcfdfa5aedca0"} -{"schema_version":1,"benchmark_version":"v1","run_id":"936fb637-9c45-40c3-bfe5-33fb0bf5bc9a","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":2,"started_at":"2026-07-23T10:08:06.053Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66bd65e09a5a8cadf11017997e8362fae34ffb37649c18058a462de424f6e78b","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.98a153bd"},"inspection":{"selection_grounded":true,"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:08:47.521Z","source_record_sha256":"e7a26c6bcdbf34636feff685eb07deebf1ce0c47f985051dbd2b896f753ab3e3"} -{"schema_version":1,"benchmark_version":"v1","run_id":"85b215db-7bde-4512-aad4-781a905c7c38","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":3,"started_at":"2026-07-23T10:08:47.521Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66bd65e09a5a8cadf11017997e8362fae34ffb37649c18058a462de424f6e78b","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.98a153bd"},"inspection":{"selection_grounded":true,"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:09:22.117Z","source_record_sha256":"4af6c1eee5e48b043149b271dc644e1f7e6b575823f62f617d7479e9044e3313"} +{"schema_version":1,"benchmark_version":"v1","run_id":"23187609-52d7-44b6-b60e-91f68d32777c","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":1,"started_at":"2026-07-23T09:40:25.828Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"28e564a72f02a0b6744c9086e328ba4b829e39246dca5821577adcb20a5f3a72","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:40:51.455Z","source_record_sha256":"0394fb5b9e153aa501a8d781643292841d812c05c0986a4661ed3866e242ab03","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"d2bfdd97-1ac6-4388-aab6-dc04bd389b4d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":2,"started_at":"2026-07-23T09:40:51.455Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"28e564a72f02a0b6744c9086e328ba4b829e39246dca5821577adcb20a5f3a72","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:41:19.367Z","source_record_sha256":"a132a571a86762281fbf49fdd7506aaa19fc15a736d6837127da51c0cab30f16","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"9dc77ec7-6fea-42e6-8786-f8970a33da5d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":3,"started_at":"2026-07-23T09:41:19.367Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"28e564a72f02a0b6744c9086e328ba4b829e39246dca5821577adcb20a5f3a72","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:41:45.578Z","source_record_sha256":"9b8eb9c3f5b12d221b1a3051a045f8cb17bdf481f796131e344089ac7ad58e16","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"98d4488a-8f0e-48fe-8743-ed08df8940f8","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-23T09:41:45.578Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"65423dcd48e9ba551f0500805a38535f2428d65ef165a26c778463c01921a519","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:42:10.683Z","source_record_sha256":"7dc8a173d225a7254741f7a0591960ebcf3c3ff7c8a7d18ba52e38dae92fef57","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"450cf5b6-8404-4be2-b73e-c28a6ae3227d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-23T09:42:10.683Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"65423dcd48e9ba551f0500805a38535f2428d65ef165a26c778463c01921a519","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:42:35.790Z","source_record_sha256":"088cb3c49cd699c6f9a9033404a71866daac47829c9a2ec696e3f5aed6bb6e89","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"456b1477-ec0a-4772-887c-8971ac437a95","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-23T09:42:35.790Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"65423dcd48e9ba551f0500805a38535f2428d65ef165a26c778463c01921a519","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:43:04.020Z","source_record_sha256":"6f10781492c58eeb6c351dfa9db214028bc0558b31b1e442e0568aa5f9611edf","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"850f6ae9-de46-4e92-a722-81edd34dd191","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-23T09:43:04.020Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"b98b9cb4476c56c6553ff1ea2bfd5e1976911c8c14142f76ef97c691d368414f","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:43:40.323Z","source_record_sha256":"18b9cf40ca54aac96152a60268192d8144aac03c1ab87fa134dd57bfffd5ce1e","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"378c4fcc-1e86-4149-8b38-82d8230e5d11","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-23T09:43:40.323Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"b98b9cb4476c56c6553ff1ea2bfd5e1976911c8c14142f76ef97c691d368414f","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:44:16.949Z","source_record_sha256":"f36668206832ea60ec935daeee36c7f48d9b4fad9c4a2740d6b347e3c6cda748","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"beb58cdc-281f-42cf-a388-9d9f30fa74be","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-23T09:44:16.949Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"b98b9cb4476c56c6553ff1ea2bfd5e1976911c8c14142f76ef97c691d368414f","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:44:53.121Z","source_record_sha256":"07f3fe927e57423d7eae4fa52ee878da564f4142ea03fe803b0b3307fb1a319d","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"b602eea9-1fa5-4b43-adf2-59adff282a87","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":1,"started_at":"2026-07-23T09:44:53.121Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"4ca8303398d379bc94e129f5da06d77fc7aebb56b424abeea7e4a3feb3910636","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:45:25.145Z","source_record_sha256":"06a9d53c3fec8a8ec22cd6fb65cca82d18632ad97ceefbff36c9c18572724017","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"bfc0ab6d-9a15-4768-a672-e466f0d23414","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":2,"started_at":"2026-07-23T09:45:25.145Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"4ca8303398d379bc94e129f5da06d77fc7aebb56b424abeea7e4a3feb3910636","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:46:05.132Z","source_record_sha256":"406f86260cf08286855e783c689cd90f3e1f976e84b0277cbff522aa6526c029","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"c2bab891-e16c-49ce-af35-9ba5220ef929","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":3,"started_at":"2026-07-23T09:46:05.132Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"4ca8303398d379bc94e129f5da06d77fc7aebb56b424abeea7e4a3feb3910636","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:46:35.462Z","source_record_sha256":"e0be7b97e788ce36f9643c04d2f4911b3bba211890d0915b07f01e579b6ff811","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"7c3b1d86-cdf3-42aa-94d0-bfa2024284ec","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T09:46:35.462Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7935de8cea6f84dce9a4df111df427079b97af7dd653fecdcba159947c48d136","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:47:02.518Z","source_record_sha256":"24485ca6593e17388a3346e0f9209b4846153324491a991e8ea7f222a144ab4a","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"0dfd6d1c-ca90-4916-8b8f-4ae4d3e7f273","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T09:47:02.518Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7935de8cea6f84dce9a4df111df427079b97af7dd653fecdcba159947c48d136","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:47:32.328Z","source_record_sha256":"f7107e2e7957bb86f20b63cd1c0eb6a4de8d5158aa8f68d2f1d8a3ba01fe3179","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"ce6fbc71-c894-461a-adb1-4a607e7d4bdc","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T09:47:32.328Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7935de8cea6f84dce9a4df111df427079b97af7dd653fecdcba159947c48d136","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:47:58.422Z","source_record_sha256":"f2cfd462aceb106b5154662f6f3e86d2e0d7afcc77db75ad43ba5d04ccf1aa56","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"b43508b4-6a54-487e-b3fd-dd511201af19","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T09:47:58.422Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"0c9d6a51ec70c89cafcbd7ccfca456093271999f98a8c42b6423e996fc6f48a4","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:48:24.635Z","source_record_sha256":"4da0464a226f7e73e86a16071182c1ecf42f06113bf5d0c77ee7068624a913db","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"704ee017-af2e-469b-a055-de7b367c3ffb","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T09:48:24.635Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"0c9d6a51ec70c89cafcbd7ccfca456093271999f98a8c42b6423e996fc6f48a4","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:48:55.300Z","source_record_sha256":"fb66f6302a91edfe07cfb60666e18fc797143c7c62980963704afd6dfdffc5fb","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"495bda63-87f8-4ada-bcd2-a2a4076f6431","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T09:48:55.300Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"0c9d6a51ec70c89cafcbd7ccfca456093271999f98a8c42b6423e996fc6f48a4","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:49:26.689Z","source_record_sha256":"d0640e38ed193059bf9d325cd9f756d76ff72556413bce3721d34f795afd0f0f","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"10a88d8d-1053-4d50-bab5-4b7114dd0ee7","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T09:49:26.689Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3dcb23db74a0c80244544838d4968450a29fa18516b5d8c8e17b0a38af6919cd","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:49:55.414Z","source_record_sha256":"35054cbc3c8f2f9e2e047104c7fce1e4b687d34d2a9fe24367477a5158e97af4","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"252d6b99-8605-4a66-aaec-d2b63a4fe36f","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T09:49:55.414Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3dcb23db74a0c80244544838d4968450a29fa18516b5d8c8e17b0a38af6919cd","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:50:31.198Z","source_record_sha256":"c5ed77b70f27ba4bba45540ec1d3ace713757cd7f600ec78f5aae4a0dd2b244b","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"3b2b6bf9-ad89-4298-92a7-3622fb115a13","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T09:50:31.198Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3dcb23db74a0c80244544838d4968450a29fa18516b5d8c8e17b0a38af6919cd","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:51:04.973Z","source_record_sha256":"ea2151bd097ea5a08d2fb3134f458e21af30728d07a85b1b08d3df56f4b816af","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"aaac2eb2-646e-42ff-ae32-1326ee9682b4","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-23T09:51:04.973Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2798b2551f89d750e90bc78b840801a716942fd2cc7241b33575fc13b5e1cc98","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T09:51:37.056Z","source_record_sha256":"6236fb0c65c08f023b41589cf7aed9f3194cb6782081175abfa6382589e9a535","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"3e947ba6-ca5a-47dc-abe3-404e9464239b","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-23T09:51:37.056Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"954802b741ae1296ad98d5a92faa78b0b02b7549ad05ec06ce4544a982fca525","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T09:52:06.521Z","source_record_sha256":"8ea3c9edf5c876f7243d43804235e2665a74f7cc290af4b8ba61e605a3ca21a5","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"69e303d3-d9ea-4f86-9fa3-0b4bd9c3087c","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-23T09:52:06.521Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"954802b741ae1296ad98d5a92faa78b0b02b7549ad05ec06ce4544a982fca525","selection_grounded":true},"selection":{"tool_id":"weather_api.ip_lookup.retrieve.v1.e095d904"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T09:52:38.206Z","source_record_sha256":"8d56e8fc444982b3c01b8ecf2017638d7194161bcebbf1ab788bc5169290a2d5","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"16873a4e-2cd3-47d9-845e-0067aeb15a42","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":1,"started_at":"2026-07-23T09:52:38.206Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2099bfa8887678c84902e7ec565c63edeef86ac5127e75b290713f8ff3a4f197","selection_grounded":true},"selection":{"tool_id":"coinmarketcap.tools.price-conversion.execute.v1.2b99a4df"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:53:13.658Z","source_record_sha256":"ee67bc1787845c7185516013d6c9a0f7e5ac4e6116d9ef80bb8fb9cbb0d926ec","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"190b85e5-c226-468b-91d5-99a08a96768d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":2,"started_at":"2026-07-23T09:53:13.658Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2099bfa8887678c84902e7ec565c63edeef86ac5127e75b290713f8ff3a4f197","selection_grounded":true},"selection":{"tool_id":"coinmarketcap.tools.price-conversion.execute.v1.2b99a4df"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:53:52.934Z","source_record_sha256":"d9583032847d73d7c87252026831b092ad84839a82152af3ba4f8958db2484e8","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"533afeba-6917-4bdb-83f5-6314d89c1207","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":3,"started_at":"2026-07-23T09:53:52.934Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"2099bfa8887678c84902e7ec565c63edeef86ac5127e75b290713f8ff3a4f197","selection_grounded":true},"selection":{"tool_id":"coinmarketcap.tools.price-conversion.execute.v1.2b99a4df"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:54:29.273Z","source_record_sha256":"c26269c83b8733a84b0b0fa0afd38a548f64e455d12753e98f3064158726eb8d","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"4e7bf26c-1907-441c-8103-49071ebcef84","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":1,"started_at":"2026-07-23T09:54:29.273Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9eb927c2b3d94415d2534d541127a937adb20aca4f0cefdc0b7c6948c064c9f9","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:54:58.710Z","source_record_sha256":"cbf18b1fae7e417ae99b83844c38d0c8ade8811fb732e5ccecde52d12d1ae414","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"7c57619b-bf15-4504-8587-08a778c4c502","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":2,"started_at":"2026-07-23T09:54:58.710Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9eb927c2b3d94415d2534d541127a937adb20aca4f0cefdc0b7c6948c064c9f9","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:55:30.042Z","source_record_sha256":"deeb3853b2c38377a75619dfad3b69859fb62bf5e4e5ebc5212f9464b79c9a08","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"b0c765a8-abef-4d88-b1d1-583d7a7083f5","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":3,"started_at":"2026-07-23T09:55:30.042Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9eb927c2b3d94415d2534d541127a937adb20aca4f0cefdc0b7c6948c064c9f9","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:56:01.974Z","source_record_sha256":"d4d546dc07a8cfebca12c2b75db4e5ba54f00289b241423a33213091bf58efa1","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"3573e6ae-8c2b-4c45-9b9f-a7dbdaf459d8","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":1,"started_at":"2026-07-23T09:56:01.974Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d664c4f2fd3f9d9e0d0d8f66a450ec57e2aa075d57bba3883ac678920ca072c6","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:56:42.264Z","source_record_sha256":"edd01cfdc6a189fb72df609f9a303603b01a19fad462ba18acb337cc42591b97","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"7835af93-e41c-428a-995f-4447571133fe","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":2,"started_at":"2026-07-23T09:56:42.264Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d664c4f2fd3f9d9e0d0d8f66a450ec57e2aa075d57bba3883ac678920ca072c6","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:57:10.056Z","source_record_sha256":"4f1e3e4bf201c9a85fe1a4c523b17c3b8e06c57dc637a19767f1872cc9de5d15","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"73d62bf3-09c9-4c6f-af68-cbe62631e4c3","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":3,"started_at":"2026-07-23T09:57:10.056Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d664c4f2fd3f9d9e0d0d8f66a450ec57e2aa075d57bba3883ac678920ca072c6","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:57:36.396Z","source_record_sha256":"3be7b15dcd9a6869bf6202dee1c262faa553589f0f1d34ed4fc3b0d3d860b9ef","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"0e125c12-7a7b-4c87-9e64-54a8be79f774","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":1,"started_at":"2026-07-23T09:57:36.396Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8d10f08954c40da564662fec151d75440ec8391aefdf74cfc8e2c32a1ee26dbb","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:58:04.308Z","source_record_sha256":"40c87aa07d78b9b835112c13584563870d4b64eb7a23230de8085d7a344f1851","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"7a2ca54b-b346-4343-b6fb-54a264c5b778","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":2,"started_at":"2026-07-23T09:58:04.308Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8d10f08954c40da564662fec151d75440ec8391aefdf74cfc8e2c32a1ee26dbb","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:58:29.894Z","source_record_sha256":"b0b04c429f3abe332a69253f6d6e831d1702da29097ab1285883cf4945269647","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"e98dc7fd-0ed4-4487-a753-c6c197254603","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":3,"started_at":"2026-07-23T09:58:29.894Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8d10f08954c40da564662fec151d75440ec8391aefdf74cfc8e2c32a1ee26dbb","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T09:58:58.074Z","source_record_sha256":"4dc549878c034c8440e01e24b20ced68fd25617caef32d35519f5ee2750ec0d3","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"a22d10aa-0f23-4f76-b2a2-692576b4b2a2","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T09:58:58.074Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true},"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected"},"finished_at":"2026-07-23T09:59:40.997Z","source_record_sha256":"8e0f1a8dd393b2e3e3f8480ede541b2effe403ed92acfa0e2fab9f4269ed9246","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"97361151-f794-478a-8b90-e8e544c484f7","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T09:59:40.997Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:00:15.027Z","source_record_sha256":"f9b49fcdc5f1e3d00fdb7185a218da04e37fc6e60da566fc3aa2e9073ab191a1","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"e877975e-2fff-4ec9-a487-0bddba738e4f","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T10:00:15.027Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true},"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected"},"finished_at":"2026-07-23T10:00:55.710Z","source_record_sha256":"d6bd4b821637a43316f544a37fc975c0b59797e77cd24ef7fae03280ddaa3721","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"d9fe796d-cd3d-4d0c-8bb3-0cd1fc7e128d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":1,"started_at":"2026-07-23T10:00:55.710Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"406f82f0151e2aab38d28e5369d6e3b6b8f1e5a40414bd6ea7cafc3eb9f40259","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:01:21.601Z","source_record_sha256":"61bcf3bc4829f5e23fb25aa989ab9e25e8bdc1b9066d6e8e7dae7ec7d3fe39a4","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"e3d0d1fd-e32c-4701-b87a-7d2907aae773","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":2,"started_at":"2026-07-23T10:01:21.601Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"406f82f0151e2aab38d28e5369d6e3b6b8f1e5a40414bd6ea7cafc3eb9f40259","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:01:46.789Z","source_record_sha256":"0c98fe2b9fab21c84a6018f2c17577e36c6f1db959d8959006e8a27a1d509c9d","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"109d0ded-433f-4ef6-a7ed-f8cf19b897be","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":3,"started_at":"2026-07-23T10:01:46.789Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"413e70c9b613711904d82925006e89c88e507640e6650646084db3a76f64b8ce","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:02:17.125Z","source_record_sha256":"25f730ed07cea7449205d61271acef2544d40eae7a33b9a76bc2ed2d6eb53a7d","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"e55b1799-5e11-481c-a18f-e4be01b4e744","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":1,"started_at":"2026-07-23T10:02:17.125Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5d7380bab4c7ce4e2289dc764fc2cae8ba0d5cbe190f62fc09dee13517796d3d","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:02:44.222Z","source_record_sha256":"b47fbff96f2cbd4f32ae3c1d02eae4ad3bb17da02f61fd7f1d9018cef02ce863","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"a88a3226-9842-4032-ac67-fcae35d59a03","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":2,"started_at":"2026-07-23T10:02:44.222Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5d7380bab4c7ce4e2289dc764fc2cae8ba0d5cbe190f62fc09dee13517796d3d","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:03:10.533Z","source_record_sha256":"c44ab638aca59dd5eb6650b581ed3e0bc1b085e81a01b9e06fa1ebde07edc1ad","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"bd6bfb47-687b-4d08-9a70-f019316e623a","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":3,"started_at":"2026-07-23T10:03:10.533Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"5d7380bab4c7ce4e2289dc764fc2cae8ba0d5cbe190f62fc09dee13517796d3d","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:03:37.293Z","source_record_sha256":"f1f1cec153e7696804111aaa35815a177088c70905bb2e81effa38b9f7dc0afa","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"c28bc1c8-94d0-4bed-a562-af06253fc011","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-23T10:03:37.293Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:04:19.806Z","source_record_sha256":"558fe12898a5153ca789f01f8d5dff4733850bbd892e4e13ceaecd52a9909ccd","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"9eff8a69-cb9a-4931-ae30-4230a29c6b2c","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-23T10:04:19.806Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:04:54.838Z","source_record_sha256":"222b9822b0bf52fdd42c8e4ef55761abc6715c95ef4b0a67a99937a60c59da23","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"81d9ac77-2d8f-4363-be1a-09193951a1b1","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T10:04:54.838Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:05:33.311Z","source_record_sha256":"8b485bf1ce1cf1c78376b3b005c724bc529b3ddd6789b540138dbae1c9a0303d","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"abc19032-2dc3-46fc-a861-05132c7eda65","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":1,"started_at":"2026-07-23T10:05:33.311Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f9568410d64286b9cc625ae8f4a38b3b350dee873f626cc44bb01cd92e5e7652","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T10:06:00.664Z","source_record_sha256":"9841f36f25b9775a95a122bfd640f89bbe141c6a0b8cda802a39479dd4b7a510","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"66977b1e-3302-469e-9e30-590d4b8cd542","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":2,"started_at":"2026-07-23T10:06:00.664Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f9568410d64286b9cc625ae8f4a38b3b350dee873f626cc44bb01cd92e5e7652","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T10:06:50.608Z","source_record_sha256":"5233af85fb24de9f413a64d9c076732b4b2f658cb5ee949299dc62dfe2a97544","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"b3001ec6-f8da-4dcc-b420-a40d0991337b","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":3,"started_at":"2026-07-23T10:06:50.608Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f9568410d64286b9cc625ae8f4a38b3b350dee873f626cc44bb01cd92e5e7652","selection_grounded":true},"selection":{"tool_id":"tradefeeds.compinfo.retrieve.v1.c2314765"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":false,"result_nonempty":false},"finished_at":"2026-07-23T10:07:33.561Z","source_record_sha256":"a5bad4a51d0d339497fed6f904985b0ebb406c88b9a791b07a9a735845003e09","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"7e26fd5b-b2f6-40bb-90dc-0d32665ffadd","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":1,"started_at":"2026-07-23T10:07:33.561Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66bd65e09a5a8cadf11017997e8362fae34ffb37649c18058a462de424f6e78b","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.98a153bd"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:08:06.053Z","source_record_sha256":"4f5f11c913c72853033c5cc8b76b8a599895f5000a28ca566ecbcfdfa5aedca0","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"936fb637-9c45-40c3-bfe5-33fb0bf5bc9a","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":2,"started_at":"2026-07-23T10:08:06.053Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66bd65e09a5a8cadf11017997e8362fae34ffb37649c18058a462de424f6e78b","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.98a153bd"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:08:47.521Z","source_record_sha256":"e7a26c6bcdbf34636feff685eb07deebf1ce0c47f985051dbd2b896f753ab3e3","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"85b215db-7bde-4512-aad4-781a905c7c38","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/codex-cli-0.144.1/medium","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"8d59a5211c759a7567caf54b37186ebc3925ffb6a3c0fa5ce2e0c2cd3737c996","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":3,"started_at":"2026-07-23T10:08:47.521Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66bd65e09a5a8cadf11017997e8362fae34ffb37649c18058a462de424f6e78b","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.98a153bd"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T10:09:22.117Z","source_record_sha256":"4af6c1eee5e48b043149b271dc644e1f7e6b575823f62f617d7479e9044e3313","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} diff --git a/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.runs.jsonl b/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.runs.jsonl index 0f518d2..a531c3c 100644 --- a/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.runs.jsonl +++ b/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.runs.jsonl @@ -1,54 +1,54 @@ -{"schema_version":1,"benchmark_version":"v1","run_id":"8ba500d0-14cb-46a7-a347-c9ba9be3a3a8","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":1,"started_at":"2026-07-23T07:52:46.344Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"b4772dc9d4f828e88d87b4ace08a968439e097cee7c9613886d037cc95f7eb04","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:52:51.394Z","source_record_sha256":"0af61ea34d3519146acf43ad3a651907d4379f14dbc9311939a1e0f223b4f560"} -{"schema_version":1,"benchmark_version":"v1","run_id":"1f240f6b-6c51-4970-94c8-86a99dd6a36e","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":2,"started_at":"2026-07-23T07:52:51.394Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"b4772dc9d4f828e88d87b4ace08a968439e097cee7c9613886d037cc95f7eb04","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:52:55.748Z","source_record_sha256":"9fe1cf51eda32858b6eae402db46b71a4a67b87a7f3482a6fa3e747497931fe6"} -{"schema_version":1,"benchmark_version":"v1","run_id":"891080de-9c5f-45d3-897f-94db7557d1c1","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":3,"started_at":"2026-07-23T07:52:55.748Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"b4772dc9d4f828e88d87b4ace08a968439e097cee7c9613886d037cc95f7eb04","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:52:59.070Z","source_record_sha256":"de18dbbb42fd505950f46e6f353c3c611431bc728b3a6c50fcceaced1ecb20eb"} -{"schema_version":1,"benchmark_version":"v1","run_id":"f32129b5-3004-4733-90c2-6137c3a74d0f","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-23T07:52:59.070Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8c4e8e2ab56f10ce503086b98981d6be57f293b4043dec00a86237144a107a5b","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:02.034Z","source_record_sha256":"bceda6101f8eeaa37088617b0b9b7b5aa4280285a8b9db10657cfcbfe7a96a0e"} -{"schema_version":1,"benchmark_version":"v1","run_id":"28f35db3-bab5-4b00-9aa3-e29371acf680","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-23T07:53:02.035Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8c4e8e2ab56f10ce503086b98981d6be57f293b4043dec00a86237144a107a5b","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:05.383Z","source_record_sha256":"dc3d2d8c9423bb629806b01d70116a244e2edb3c225af885af5c54638aaf1e42"} -{"schema_version":1,"benchmark_version":"v1","run_id":"770c7467-c2b6-4090-8826-0182ccfae9da","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-23T07:53:05.383Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8c4e8e2ab56f10ce503086b98981d6be57f293b4043dec00a86237144a107a5b","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:08.429Z","source_record_sha256":"9450c107d0c58da627d2c3b1ed2f9a115d286340bfb5011576b0ca1091a6a0f0"} -{"schema_version":1,"benchmark_version":"v1","run_id":"c4624c8d-fb7b-4d9e-9248-209a6679336b","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-23T07:53:08.429Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"84ea39c886ae326a41eb2ef257097f3e0ad96d07404ab489516a62c468e23e0e","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:11.737Z","source_record_sha256":"7a7f0968645bd379de70740d7fd9a67a8b52b5bd637e9981a4cddace3fcd2c0a"} -{"schema_version":1,"benchmark_version":"v1","run_id":"4d565c9f-3a0c-4dc2-ad7c-4848b1e3f561","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-23T07:53:11.737Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"84ea39c886ae326a41eb2ef257097f3e0ad96d07404ab489516a62c468e23e0e","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:14.963Z","source_record_sha256":"76a8ffbec9cf85c811492d2a7b89ed017fe0e6a4bacdc435f335acb9424f123f"} -{"schema_version":1,"benchmark_version":"v1","run_id":"c8ab76aa-1e1b-4c55-8f77-f583d283d344","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-23T07:53:14.963Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"84ea39c886ae326a41eb2ef257097f3e0ad96d07404ab489516a62c468e23e0e","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:18.126Z","source_record_sha256":"a9ad392d5968fb6dc60ca31ee93bf238579d4fe58710f3512f0c279fe29aa197"} -{"schema_version":1,"benchmark_version":"v1","run_id":"3a5d9a59-34e8-4209-b700-633819ddcedf","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":1,"started_at":"2026-07-23T07:53:18.126Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"15c3ab376fd0853e249e998c30c4632d06a8cb26ae60ae74080fdeaeab085a29","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"artificial intelligence","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:20.737Z","source_record_sha256":"06d2a66d7ec4df06e6881e8568069ca35541904373ef7d9cd1df8a43b3433167"} -{"schema_version":1,"benchmark_version":"v1","run_id":"1190b998-aceb-484b-8e96-8f711b737082","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":2,"started_at":"2026-07-23T07:53:20.737Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"15c3ab376fd0853e249e998c30c4632d06a8cb26ae60ae74080fdeaeab085a29","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"artificial intelligence","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:23.488Z","source_record_sha256":"e7fa7c4d1a74281901b6142fee9b9f4124addb0f964f8a93a201664ea0e93784"} -{"schema_version":1,"benchmark_version":"v1","run_id":"9e1336ba-27ab-4d05-b4e2-72f24e2d9fd2","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":3,"started_at":"2026-07-23T07:53:23.488Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"15c3ab376fd0853e249e998c30c4632d06a8cb26ae60ae74080fdeaeab085a29","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"artificial intelligence","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:26.274Z","source_record_sha256":"e1eb12914dfef2709c88243eec74e97e1a6be81f60bbb60878f71aa16d238500"} -{"schema_version":1,"benchmark_version":"v1","run_id":"7ddaaf21-9eb5-4cbf-9e9e-ad0cf2b4da30","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T07:53:26.275Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:30.277Z","source_record_sha256":"7f876ab3626ab04941230d16c52d7ad76831f5c7eca25e27ced0795f330b21b5"} -{"schema_version":1,"benchmark_version":"v1","run_id":"aa7da099-31bf-4f14-94b5-04b148b81bde","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T07:53:30.278Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:34.089Z","source_record_sha256":"14497b8df3945d1204c9f850aff21659fed945adf62f0282570a7fd03b000b8d"} -{"schema_version":1,"benchmark_version":"v1","run_id":"b42f18f4-c0fe-443f-9d07-cb4870e88af6","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T07:53:34.089Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:37.784Z","source_record_sha256":"6671f9b91d3b83f7ad972c4764083b7aeedfbcc2e2f0483c454d8d2b9d90e2b0"} -{"schema_version":1,"benchmark_version":"v1","run_id":"ab798371-3cf3-4945-87c6-4240b8b9e36f","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T07:53:37.784Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"97d9f951a9f193786150828de15857112d2439f3832ea0b92efcd5108ada92af","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id"},"finished_at":"2026-07-23T07:53:39.282Z","source_record_sha256":"334143d7288f3d7bd9e0e1542e391875145a4f8d296ac3deab16320d4de545ee"} -{"schema_version":1,"benchmark_version":"v1","run_id":"3879f2c1-7e03-4819-b2b4-4dc353c9b6b6","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T07:53:39.282Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"97d9f951a9f193786150828de15857112d2439f3832ea0b92efcd5108ada92af","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id"},"finished_at":"2026-07-23T07:53:41.087Z","source_record_sha256":"664918ea59c543cacbe4d218fc05fbd56995b603c5981355eb040dd1b64da7af"} -{"schema_version":1,"benchmark_version":"v1","run_id":"cb7160d0-c6ea-4e8b-a60e-bffc3e6747df","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T07:53:41.087Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"97d9f951a9f193786150828de15857112d2439f3832ea0b92efcd5108ada92af","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id"},"finished_at":"2026-07-23T07:53:42.486Z","source_record_sha256":"f020400865f1d1eee9552674edd440cdb237b26d9dbe01217fd1704109c7bc5c"} -{"schema_version":1,"benchmark_version":"v1","run_id":"c004662d-e17c-441a-8439-d18a4a1988c5","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T07:53:42.486Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f3a89453c750d6ddfee82c21bfd021c18cbefd53654879c10f93d3b6200c7d13","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:46.409Z","source_record_sha256":"81cc941960ad0e0e7d1ca50a6703be7197c26fa3190096771441100819fb438b"} -{"schema_version":1,"benchmark_version":"v1","run_id":"aa9437b4-4a68-4467-9195-e421a33d242d","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T07:53:46.409Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f3a89453c750d6ddfee82c21bfd021c18cbefd53654879c10f93d3b6200c7d13","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:51.677Z","source_record_sha256":"98c2f8f11295550da5dc09caaeb1cbba0e058a4e0672402acb9052882f0162ce"} -{"schema_version":1,"benchmark_version":"v1","run_id":"c3632840-0ea8-46f3-9c0b-d721c14211ed","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T07:53:51.677Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f3a89453c750d6ddfee82c21bfd021c18cbefd53654879c10f93d3b6200c7d13","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:55.919Z","source_record_sha256":"07899db418e2d3ff161a1468af71d0d8d9f0d5ecb0e3aaeda74c8501b5c858a3"} -{"schema_version":1,"benchmark_version":"v1","run_id":"a0d8e047-f8fa-4579-bec2-de6526299e1a","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-23T07:53:55.919Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"bed0baef996d3d7151e059d230ea273b29388e076cd6fd76c201699fe0e77d92","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true,"required_parameters":["ip"]},"parameters":{"ip":"8.8.8.8"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:58.844Z","source_record_sha256":"417a087f5fff31d941d1450f386d7f2b7cf26493e2b56a32a0c89aceba03110b"} -{"schema_version":1,"benchmark_version":"v1","run_id":"bc8d8e9b-d160-465f-bda4-3b185ef33115","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-23T07:53:58.844Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"bed0baef996d3d7151e059d230ea273b29388e076cd6fd76c201699fe0e77d92","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true,"required_parameters":["ip"]},"parameters":{"ip":"8.8.8.8"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:02.027Z","source_record_sha256":"7b3a08f02c7ebbc54497ee554d7c7508d0a36d537fe86a0da0e967108153a0a6"} -{"schema_version":1,"benchmark_version":"v1","run_id":"cd7516ab-c10d-4af9-8063-273a79f68af3","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-23T07:54:02.027Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"bed0baef996d3d7151e059d230ea273b29388e076cd6fd76c201699fe0e77d92","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true,"required_parameters":["ip"]},"parameters":{"ip":"8.8.8.8"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:05.637Z","source_record_sha256":"554942cdc248efa73636668cf133e807fe106f6ba81b17b6a3bbc0792846c1ae"} -{"schema_version":1,"benchmark_version":"v1","run_id":"be7b9cb2-88b4-4148-b1aa-4a17f035c27a","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":1,"started_at":"2026-07-23T07:54:05.637Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66d9bb9820728c627264f0ade0a1ca2560a70f6bcb8e608cc1ff1dcba98fc27c","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:09.422Z","source_record_sha256":"894f6a0e29bb8e829274f97ccda1360b23dddd9642e8967b73c7164c0fbd7438"} -{"schema_version":1,"benchmark_version":"v1","run_id":"fa26dcf3-d647-434e-8fc4-cfb726806704","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":2,"started_at":"2026-07-23T07:54:09.422Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66d9bb9820728c627264f0ade0a1ca2560a70f6bcb8e608cc1ff1dcba98fc27c","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:12.396Z","source_record_sha256":"7e6204b967be6bed5da44ba84fc66b245ec98f6ee5b919d4c80dcb436dc3a203"} -{"schema_version":1,"benchmark_version":"v1","run_id":"5a70425d-4ab1-48a2-a484-b91463c77e34","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":3,"started_at":"2026-07-23T07:54:12.396Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66d9bb9820728c627264f0ade0a1ca2560a70f6bcb8e608cc1ff1dcba98fc27c","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:15.174Z","source_record_sha256":"b90d6f2b430710e87f66e708b63dd2e3907a7110314961ef14dcef2e799b98f9"} -{"schema_version":1,"benchmark_version":"v1","run_id":"cc44232f-8d5d-40e4-87d0-914004721dfc","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":1,"started_at":"2026-07-23T07:54:15.174Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6ce1454e6f0eb9e506f81f2b5680e6c466aa63d6ebe9a2fd2dcbc8a9ef43cbfb","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:17.456Z","source_record_sha256":"30451fa2e418594b6e586f1d65e1d271d69c88a98333fe7fa180ec7a579bfc68"} -{"schema_version":1,"benchmark_version":"v1","run_id":"0ecbcede-88b3-403b-934f-8e1ce9909259","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":2,"started_at":"2026-07-23T07:54:17.456Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6ce1454e6f0eb9e506f81f2b5680e6c466aa63d6ebe9a2fd2dcbc8a9ef43cbfb","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:20.037Z","source_record_sha256":"cc6f64a7dada4a2d453e01e4c97a9dcf587205e82aa0063be7b0e758b231c75e"} -{"schema_version":1,"benchmark_version":"v1","run_id":"7b1f0b31-d0a7-4fb7-b807-d8633bd42387","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":3,"started_at":"2026-07-23T07:54:20.037Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6ce1454e6f0eb9e506f81f2b5680e6c466aa63d6ebe9a2fd2dcbc8a9ef43cbfb","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:22.508Z","source_record_sha256":"982d57090dc3a49756ae97a55ad4c8b2d881887f167b439982fab4c075a75ebc"} -{"schema_version":1,"benchmark_version":"v1","run_id":"e85eed29-7b7c-4c24-875b-7c4ec6f2afc6","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":1,"started_at":"2026-07-23T07:54:22.508Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:25.490Z","source_record_sha256":"439b63730d6b09940fdf2a8ab4bdefd01f4d0500f874f0f43f9bbf718521d81e"} -{"schema_version":1,"benchmark_version":"v1","run_id":"203b9a62-2140-4edb-b40a-61c95d9c3146","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":2,"started_at":"2026-07-23T07:54:25.490Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:27.929Z","source_record_sha256":"faa4bdd3fc214b2cb211097805f866c9e9328d33548e9ebb5418c3b0cc7ce47d"} -{"schema_version":1,"benchmark_version":"v1","run_id":"95854287-bfed-486a-9aff-20d839999229","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":3,"started_at":"2026-07-23T07:54:27.929Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:30.101Z","source_record_sha256":"2f8fd3729b4ecadac2c9e08867dcccb1c46ef5f3014f4cc01e030c26b08401d4"} -{"schema_version":1,"benchmark_version":"v1","run_id":"ff1c9565-ffe8-4a70-adc2-b5215c3410ff","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":1,"started_at":"2026-07-23T07:54:30.101Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"a06d4241bd0f4717242da76bd0a368da5831e1734b134538a73c2bdaccb08f02","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:33.694Z","source_record_sha256":"6fd516f622f28c4308eb9a8fb0fc8ba0da6bf4e55ca0d0bccbaf314f8514c12d"} -{"schema_version":1,"benchmark_version":"v1","run_id":"56a668ab-8a69-4277-8729-4a1717af668c","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":2,"started_at":"2026-07-23T07:54:33.695Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"a06d4241bd0f4717242da76bd0a368da5831e1734b134538a73c2bdaccb08f02","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:37.509Z","source_record_sha256":"e9f371194aacbe14fdb754ae7ca81cfed9d449fe60bc2b0e7db8b4eced05d7e4"} -{"schema_version":1,"benchmark_version":"v1","run_id":"2fd403d8-a11c-41e1-9390-d076ca5dfdb0","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":3,"started_at":"2026-07-23T07:54:37.510Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"a06d4241bd0f4717242da76bd0a368da5831e1734b134538a73c2bdaccb08f02","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:41.063Z","source_record_sha256":"462b2a9d2956048e37863bb20115229e7cdf7eec0f0c38e926fa28e3e69f87d3"} -{"schema_version":1,"benchmark_version":"v1","run_id":"8b3f5092-fa39-4426-a908-35e2af2c9312","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T07:54:41.063Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"selection_grounded":true,"required_parameters":["domain"]},"parameters":{"domain":"example.com"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:44.215Z","source_record_sha256":"bb6cd41a0bc9404975ac0ce301c90dfa89db04f157f2baf52c386d078353248d"} -{"schema_version":1,"benchmark_version":"v1","run_id":"d454cf08-3fbd-4a50-bb4a-9af533a5675d","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T07:54:44.215Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"selection_grounded":true,"required_parameters":["domain"]},"parameters":{"domain":"example.com"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:47.714Z","source_record_sha256":"83adcd9d73f2cbcb2401cee3c0236f99274f1d94b4f5bcaae9f2b2b7fd0cea2d"} -{"schema_version":1,"benchmark_version":"v1","run_id":"d24fe5b1-5ffb-427b-a45a-37baffde5d36","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T07:54:47.714Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"selection_grounded":true,"required_parameters":["domain"]},"parameters":{"domain":"example.com"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:50.804Z","source_record_sha256":"f2c213a9cf5de6fb38b9a6b5c533ad4ec1a6782a088b14b16534373073244be9"} -{"schema_version":1,"benchmark_version":"v1","run_id":"66cecd06-6fce-4084-94a3-4997791e2303","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":1,"started_at":"2026-07-23T07:54:50.804Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"ee3ecf6abb4efbab753f21f53cf48773a835c92a3130aaa099de9ab94938c11f","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:54.158Z","source_record_sha256":"a890d50ba9be3b67f26af31bdfb78cef8f90c9114710fd6ea3de002d3571f13d"} -{"schema_version":1,"benchmark_version":"v1","run_id":"ffdc3990-5991-4e84-bfb9-4997434ec7e8","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":2,"started_at":"2026-07-23T07:54:54.158Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"ee3ecf6abb4efbab753f21f53cf48773a835c92a3130aaa099de9ab94938c11f","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:57.764Z","source_record_sha256":"f47672c5937d11c4dbc47389860eab92a1e9b524f175e2660244ff5de2f9562e"} -{"schema_version":1,"benchmark_version":"v1","run_id":"8fc69d41-be93-424e-ae30-754fb957f63b","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":3,"started_at":"2026-07-23T07:54:57.764Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"ee3ecf6abb4efbab753f21f53cf48773a835c92a3130aaa099de9ab94938c11f","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:01.885Z","source_record_sha256":"79a25af369d8e25cfe3d8675f3de51c3167bb7b6a295b2ba0dd6cc2086ae602f"} -{"schema_version":1,"benchmark_version":"v1","run_id":"5cd557cc-6800-41f3-b397-f194865cc601","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":1,"started_at":"2026-07-23T07:55:01.885Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"94c7741de8644b5da225afc2ab92aaca833f1b9970bd8ef60ab25afcbd393b4f","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:04.949Z","source_record_sha256":"96a966391d65a0f6043821c49d8a00b24c60d3dca3f384168389f3d6fe7790f6"} -{"schema_version":1,"benchmark_version":"v1","run_id":"5c5c80cb-d46e-42c9-bbb7-c173a487851d","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":2,"started_at":"2026-07-23T07:55:04.949Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"94c7741de8644b5da225afc2ab92aaca833f1b9970bd8ef60ab25afcbd393b4f","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:07.898Z","source_record_sha256":"e30ad712f81d01a8e10cc3cbb7aca35ba8691162103607f77c9f2b989249df29"} -{"schema_version":1,"benchmark_version":"v1","run_id":"e0657299-6f28-4851-8554-1b4d50126bfb","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":3,"started_at":"2026-07-23T07:55:07.898Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"94c7741de8644b5da225afc2ab92aaca833f1b9970bd8ef60ab25afcbd393b4f","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:11.714Z","source_record_sha256":"ddb4dd8f9c7e67b446b0afbac933185d49b2f9027b079dfd511548552f3e6df3"} -{"schema_version":1,"benchmark_version":"v1","run_id":"53649f10-7f15-439b-b58e-c8e01a811995","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-23T07:55:11.714Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:20.097Z","source_record_sha256":"59f4bd89d66c54fe7486331376c9ba579108485c9a765ecc207405b698402cae"} -{"schema_version":1,"benchmark_version":"v1","run_id":"cdefdbdd-5005-472b-b3e0-d33e3ecc51c5","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-23T07:55:20.097Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:28.789Z","source_record_sha256":"9cc943cb8d10c628630fb2425caccb1a8c3093774d7b7dda357ac6cb074b8d01"} -{"schema_version":1,"benchmark_version":"v1","run_id":"929db9db-2ef4-466d-a80b-2e14af1f4c7c","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T07:55:28.789Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:36.177Z","source_record_sha256":"7cb4afef06406a0e19a1ee831041bc440418e7a2819a80a18360e9386f6d7668"} -{"schema_version":1,"benchmark_version":"v1","run_id":"711a0f3a-7f40-4b3d-ab18-0fd661803757","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":1,"started_at":"2026-07-23T07:55:36.177Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9ca02b19c7d86f46ad7a35b1f996cc0789a5a1cb92c6c567fbf2233aefa59ed4","selection_grounded":true},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:40.433Z","source_record_sha256":"bb8a1ce15aaca2982405e532624ca02946339e8355c0a4b192b75a9a5e886b81"} -{"schema_version":1,"benchmark_version":"v1","run_id":"342d9249-b2e3-43e1-9db2-9d6584889277","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":2,"started_at":"2026-07-23T07:55:40.433Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9ca02b19c7d86f46ad7a35b1f996cc0789a5a1cb92c6c567fbf2233aefa59ed4","selection_grounded":true},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:43.901Z","source_record_sha256":"f8e45b5268313edd9fcf9dc8fc42facdac41bbf8aa973d87f286cab763f06ab8"} -{"schema_version":1,"benchmark_version":"v1","run_id":"063b8a50-9c2c-41dd-89df-4a639c354c74","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":3,"started_at":"2026-07-23T07:55:43.901Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9ca02b19c7d86f46ad7a35b1f996cc0789a5a1cb92c6c567fbf2233aefa59ed4","selection_grounded":true},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:47.413Z","source_record_sha256":"25401223352ec2b8591b28d187ddd4a8215e1ceec71e81d878195b48b3e11951"} -{"schema_version":1,"benchmark_version":"v1","run_id":"34788c2f-c12d-46b8-9a88-d4285ab363de","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":1,"started_at":"2026-07-23T07:55:47.413Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8533341a13a1599cae14cf51f2a2e2642be5fae6d57aa42b445fc87cf9803a1f","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"selection_grounded":true,"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:51.371Z","source_record_sha256":"2c20d520e1e846149c959212418c6111000c94379f2adac4e9f4a1f98c013e7c"} -{"schema_version":1,"benchmark_version":"v1","run_id":"1ae754a2-5fe1-4baa-85b8-e5c6e8664576","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":2,"started_at":"2026-07-23T07:55:51.371Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8533341a13a1599cae14cf51f2a2e2642be5fae6d57aa42b445fc87cf9803a1f","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"selection_grounded":true,"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:54.947Z","source_record_sha256":"a66bede429e003a5177bcab371c06cfb944263d91f57f05c2ebddbd991e58cb7"} -{"schema_version":1,"benchmark_version":"v1","run_id":"ddfc24d1-5ac5-4861-b959-7013b1160520","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":3,"started_at":"2026-07-23T07:55:54.947Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8533341a13a1599cae14cf51f2a2e2642be5fae6d57aa42b445fc87cf9803a1f","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"selection_grounded":true,"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:58.608Z","source_record_sha256":"f0eef74f6476cb2e6b010b7b0bef148fd16ead166a062cf4c11e3fcf8e5f2921"} +{"schema_version":1,"benchmark_version":"v1","run_id":"8ba500d0-14cb-46a7-a347-c9ba9be3a3a8","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":1,"started_at":"2026-07-23T07:52:46.344Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"b4772dc9d4f828e88d87b4ace08a968439e097cee7c9613886d037cc95f7eb04","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:52:51.394Z","source_record_sha256":"0af61ea34d3519146acf43ad3a651907d4379f14dbc9311939a1e0f223b4f560","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"1f240f6b-6c51-4970-94c8-86a99dd6a36e","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":2,"started_at":"2026-07-23T07:52:51.394Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"b4772dc9d4f828e88d87b4ace08a968439e097cee7c9613886d037cc95f7eb04","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:52:55.748Z","source_record_sha256":"9fe1cf51eda32858b6eae402db46b71a4a67b87a7f3482a6fa3e747497931fe6","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"891080de-9c5f-45d3-897f-94db7557d1c1","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":3,"started_at":"2026-07-23T07:52:55.748Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"b4772dc9d4f828e88d87b4ace08a968439e097cee7c9613886d037cc95f7eb04","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:52:59.070Z","source_record_sha256":"de18dbbb42fd505950f46e6f353c3c611431bc728b3a6c50fcceaced1ecb20eb","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"f32129b5-3004-4733-90c2-6137c3a74d0f","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-23T07:52:59.070Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8c4e8e2ab56f10ce503086b98981d6be57f293b4043dec00a86237144a107a5b","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:02.034Z","source_record_sha256":"bceda6101f8eeaa37088617b0b9b7b5aa4280285a8b9db10657cfcbfe7a96a0e","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"28f35db3-bab5-4b00-9aa3-e29371acf680","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-23T07:53:02.035Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8c4e8e2ab56f10ce503086b98981d6be57f293b4043dec00a86237144a107a5b","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:05.383Z","source_record_sha256":"dc3d2d8c9423bb629806b01d70116a244e2edb3c225af885af5c54638aaf1e42","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"770c7467-c2b6-4090-8826-0182ccfae9da","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-23T07:53:05.383Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8c4e8e2ab56f10ce503086b98981d6be57f293b4043dec00a86237144a107a5b","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:08.429Z","source_record_sha256":"9450c107d0c58da627d2c3b1ed2f9a115d286340bfb5011576b0ca1091a6a0f0","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"c4624c8d-fb7b-4d9e-9248-209a6679336b","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-23T07:53:08.429Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"84ea39c886ae326a41eb2ef257097f3e0ad96d07404ab489516a62c468e23e0e","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:11.737Z","source_record_sha256":"7a7f0968645bd379de70740d7fd9a67a8b52b5bd637e9981a4cddace3fcd2c0a","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"4d565c9f-3a0c-4dc2-ad7c-4848b1e3f561","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-23T07:53:11.737Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"84ea39c886ae326a41eb2ef257097f3e0ad96d07404ab489516a62c468e23e0e","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:14.963Z","source_record_sha256":"76a8ffbec9cf85c811492d2a7b89ed017fe0e6a4bacdc435f335acb9424f123f","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"c8ab76aa-1e1b-4c55-8f77-f583d283d344","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-23T07:53:14.963Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"84ea39c886ae326a41eb2ef257097f3e0ad96d07404ab489516a62c468e23e0e","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:18.126Z","source_record_sha256":"a9ad392d5968fb6dc60ca31ee93bf238579d4fe58710f3512f0c279fe29aa197","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"3a5d9a59-34e8-4209-b700-633819ddcedf","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":1,"started_at":"2026-07-23T07:53:18.126Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"15c3ab376fd0853e249e998c30c4632d06a8cb26ae60ae74080fdeaeab085a29","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:20.737Z","source_record_sha256":"06d2a66d7ec4df06e6881e8568069ca35541904373ef7d9cd1df8a43b3433167","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"1190b998-aceb-484b-8e96-8f711b737082","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":2,"started_at":"2026-07-23T07:53:20.737Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"15c3ab376fd0853e249e998c30c4632d06a8cb26ae60ae74080fdeaeab085a29","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:23.488Z","source_record_sha256":"e7fa7c4d1a74281901b6142fee9b9f4124addb0f964f8a93a201664ea0e93784","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"9e1336ba-27ab-4d05-b4e2-72f24e2d9fd2","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":3,"started_at":"2026-07-23T07:53:23.488Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"15c3ab376fd0853e249e998c30c4632d06a8cb26ae60ae74080fdeaeab085a29","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:26.274Z","source_record_sha256":"e1eb12914dfef2709c88243eec74e97e1a6be81f60bbb60878f71aa16d238500","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"7ddaaf21-9eb5-4cbf-9e9e-ad0cf2b4da30","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T07:53:26.275Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:30.277Z","source_record_sha256":"7f876ab3626ab04941230d16c52d7ad76831f5c7eca25e27ced0795f330b21b5","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"aa7da099-31bf-4f14-94b5-04b148b81bde","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T07:53:30.278Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:34.089Z","source_record_sha256":"14497b8df3945d1204c9f850aff21659fed945adf62f0282570a7fd03b000b8d","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"b42f18f4-c0fe-443f-9d07-cb4870e88af6","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T07:53:34.089Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:37.784Z","source_record_sha256":"6671f9b91d3b83f7ad972c4764083b7aeedfbcc2e2f0483c454d8d2b9d90e2b0","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"ab798371-3cf3-4945-87c6-4240b8b9e36f","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T07:53:37.784Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"97d9f951a9f193786150828de15857112d2439f3832ea0b92efcd5108ada92af","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false},"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id"},"finished_at":"2026-07-23T07:53:39.282Z","source_record_sha256":"334143d7288f3d7bd9e0e1542e391875145a4f8d296ac3deab16320d4de545ee","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"3879f2c1-7e03-4819-b2b4-4dc353c9b6b6","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T07:53:39.282Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"97d9f951a9f193786150828de15857112d2439f3832ea0b92efcd5108ada92af","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false},"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id"},"finished_at":"2026-07-23T07:53:41.087Z","source_record_sha256":"664918ea59c543cacbe4d218fc05fbd56995b603c5981355eb040dd1b64da7af","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"cb7160d0-c6ea-4e8b-a60e-bffc3e6747df","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T07:53:41.087Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"97d9f951a9f193786150828de15857112d2439f3832ea0b92efcd5108ada92af","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false},"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id"},"finished_at":"2026-07-23T07:53:42.486Z","source_record_sha256":"f020400865f1d1eee9552674edd440cdb237b26d9dbe01217fd1704109c7bc5c","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"c004662d-e17c-441a-8439-d18a4a1988c5","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T07:53:42.486Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f3a89453c750d6ddfee82c21bfd021c18cbefd53654879c10f93d3b6200c7d13","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:46.409Z","source_record_sha256":"81cc941960ad0e0e7d1ca50a6703be7197c26fa3190096771441100819fb438b","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"aa9437b4-4a68-4467-9195-e421a33d242d","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T07:53:46.409Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f3a89453c750d6ddfee82c21bfd021c18cbefd53654879c10f93d3b6200c7d13","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:51.677Z","source_record_sha256":"98c2f8f11295550da5dc09caaeb1cbba0e058a4e0672402acb9052882f0162ce","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"c3632840-0ea8-46f3-9c0b-d721c14211ed","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T07:53:51.677Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f3a89453c750d6ddfee82c21bfd021c18cbefd53654879c10f93d3b6200c7d13","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:55.919Z","source_record_sha256":"07899db418e2d3ff161a1468af71d0d8d9f0d5ecb0e3aaeda74c8501b5c858a3","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"a0d8e047-f8fa-4579-bec2-de6526299e1a","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-23T07:53:55.919Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"bed0baef996d3d7151e059d230ea273b29388e076cd6fd76c201699fe0e77d92","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:53:58.844Z","source_record_sha256":"417a087f5fff31d941d1450f386d7f2b7cf26493e2b56a32a0c89aceba03110b","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"bc8d8e9b-d160-465f-bda4-3b185ef33115","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-23T07:53:58.844Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"bed0baef996d3d7151e059d230ea273b29388e076cd6fd76c201699fe0e77d92","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:02.027Z","source_record_sha256":"7b3a08f02c7ebbc54497ee554d7c7508d0a36d537fe86a0da0e967108153a0a6","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"cd7516ab-c10d-4af9-8063-273a79f68af3","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-23T07:54:02.027Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"bed0baef996d3d7151e059d230ea273b29388e076cd6fd76c201699fe0e77d92","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:05.637Z","source_record_sha256":"554942cdc248efa73636668cf133e807fe106f6ba81b17b6a3bbc0792846c1ae","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"be7b9cb2-88b4-4148-b1aa-4a17f035c27a","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":1,"started_at":"2026-07-23T07:54:05.637Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66d9bb9820728c627264f0ade0a1ca2560a70f6bcb8e608cc1ff1dcba98fc27c","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:09.422Z","source_record_sha256":"894f6a0e29bb8e829274f97ccda1360b23dddd9642e8967b73c7164c0fbd7438","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"fa26dcf3-d647-434e-8fc4-cfb726806704","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":2,"started_at":"2026-07-23T07:54:09.422Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66d9bb9820728c627264f0ade0a1ca2560a70f6bcb8e608cc1ff1dcba98fc27c","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:12.396Z","source_record_sha256":"7e6204b967be6bed5da44ba84fc66b245ec98f6ee5b919d4c80dcb436dc3a203","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"5a70425d-4ab1-48a2-a484-b91463c77e34","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":3,"started_at":"2026-07-23T07:54:12.396Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"66d9bb9820728c627264f0ade0a1ca2560a70f6bcb8e608cc1ff1dcba98fc27c","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:15.174Z","source_record_sha256":"b90d6f2b430710e87f66e708b63dd2e3907a7110314961ef14dcef2e799b98f9","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"cc44232f-8d5d-40e4-87d0-914004721dfc","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":1,"started_at":"2026-07-23T07:54:15.174Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6ce1454e6f0eb9e506f81f2b5680e6c466aa63d6ebe9a2fd2dcbc8a9ef43cbfb","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:17.456Z","source_record_sha256":"30451fa2e418594b6e586f1d65e1d271d69c88a98333fe7fa180ec7a579bfc68","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"0ecbcede-88b3-403b-934f-8e1ce9909259","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":2,"started_at":"2026-07-23T07:54:17.456Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6ce1454e6f0eb9e506f81f2b5680e6c466aa63d6ebe9a2fd2dcbc8a9ef43cbfb","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:20.037Z","source_record_sha256":"cc6f64a7dada4a2d453e01e4c97a9dcf587205e82aa0063be7b0e758b231c75e","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"7b1f0b31-d0a7-4fb7-b807-d8633bd42387","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":3,"started_at":"2026-07-23T07:54:20.037Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6ce1454e6f0eb9e506f81f2b5680e6c466aa63d6ebe9a2fd2dcbc8a9ef43cbfb","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:22.508Z","source_record_sha256":"982d57090dc3a49756ae97a55ad4c8b2d881887f167b439982fab4c075a75ebc","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"e85eed29-7b7c-4c24-875b-7c4ec6f2afc6","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":1,"started_at":"2026-07-23T07:54:22.508Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:25.490Z","source_record_sha256":"439b63730d6b09940fdf2a8ab4bdefd01f4d0500f874f0f43f9bbf718521d81e","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"203b9a62-2140-4edb-b40a-61c95d9c3146","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":2,"started_at":"2026-07-23T07:54:25.490Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:27.929Z","source_record_sha256":"faa4bdd3fc214b2cb211097805f866c9e9328d33548e9ebb5418c3b0cc7ce47d","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"95854287-bfed-486a-9aff-20d839999229","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":3,"started_at":"2026-07-23T07:54:27.929Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:30.101Z","source_record_sha256":"2f8fd3729b4ecadac2c9e08867dcccb1c46ef5f3014f4cc01e030c26b08401d4","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"ff1c9565-ffe8-4a70-adc2-b5215c3410ff","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":1,"started_at":"2026-07-23T07:54:30.101Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"a06d4241bd0f4717242da76bd0a368da5831e1734b134538a73c2bdaccb08f02","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:33.694Z","source_record_sha256":"6fd516f622f28c4308eb9a8fb0fc8ba0da6bf4e55ca0d0bccbaf314f8514c12d","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"56a668ab-8a69-4277-8729-4a1717af668c","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":2,"started_at":"2026-07-23T07:54:33.695Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"a06d4241bd0f4717242da76bd0a368da5831e1734b134538a73c2bdaccb08f02","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:37.509Z","source_record_sha256":"e9f371194aacbe14fdb754ae7ca81cfed9d449fe60bc2b0e7db8b4eced05d7e4","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"2fd403d8-a11c-41e1-9390-d076ca5dfdb0","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":3,"started_at":"2026-07-23T07:54:37.510Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"a06d4241bd0f4717242da76bd0a368da5831e1734b134538a73c2bdaccb08f02","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:41.063Z","source_record_sha256":"462b2a9d2956048e37863bb20115229e7cdf7eec0f0c38e926fa28e3e69f87d3","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"8b3f5092-fa39-4426-a908-35e2af2c9312","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T07:54:41.063Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:44.215Z","source_record_sha256":"bb6cd41a0bc9404975ac0ce301c90dfa89db04f157f2baf52c386d078353248d","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"d454cf08-3fbd-4a50-bb4a-9af533a5675d","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T07:54:44.215Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:47.714Z","source_record_sha256":"83adcd9d73f2cbcb2401cee3c0236f99274f1d94b4f5bcaae9f2b2b7fd0cea2d","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"d24fe5b1-5ffb-427b-a45a-37baffde5d36","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T07:54:47.714Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c9321695aa12fd6a080fd01e6cb8a9c36976f082309395e16eb49d55965542bd","selection_grounded":true},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:50.804Z","source_record_sha256":"f2c213a9cf5de6fb38b9a6b5c533ad4ec1a6782a088b14b16534373073244be9","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"66cecd06-6fce-4084-94a3-4997791e2303","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":1,"started_at":"2026-07-23T07:54:50.804Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"ee3ecf6abb4efbab753f21f53cf48773a835c92a3130aaa099de9ab94938c11f","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:54.158Z","source_record_sha256":"a890d50ba9be3b67f26af31bdfb78cef8f90c9114710fd6ea3de002d3571f13d","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"ffdc3990-5991-4e84-bfb9-4997434ec7e8","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":2,"started_at":"2026-07-23T07:54:54.158Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"ee3ecf6abb4efbab753f21f53cf48773a835c92a3130aaa099de9ab94938c11f","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:54:57.764Z","source_record_sha256":"f47672c5937d11c4dbc47389860eab92a1e9b524f175e2660244ff5de2f9562e","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"8fc69d41-be93-424e-ae30-754fb957f63b","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":3,"started_at":"2026-07-23T07:54:57.764Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"ee3ecf6abb4efbab753f21f53cf48773a835c92a3130aaa099de9ab94938c11f","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:01.885Z","source_record_sha256":"79a25af369d8e25cfe3d8675f3de51c3167bb7b6a295b2ba0dd6cc2086ae602f","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"5cd557cc-6800-41f3-b397-f194865cc601","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":1,"started_at":"2026-07-23T07:55:01.885Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"94c7741de8644b5da225afc2ab92aaca833f1b9970bd8ef60ab25afcbd393b4f","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:04.949Z","source_record_sha256":"96a966391d65a0f6043821c49d8a00b24c60d3dca3f384168389f3d6fe7790f6","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"5c5c80cb-d46e-42c9-bbb7-c173a487851d","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":2,"started_at":"2026-07-23T07:55:04.949Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"94c7741de8644b5da225afc2ab92aaca833f1b9970bd8ef60ab25afcbd393b4f","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:07.898Z","source_record_sha256":"e30ad712f81d01a8e10cc3cbb7aca35ba8691162103607f77c9f2b989249df29","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"e0657299-6f28-4851-8554-1b4d50126bfb","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":3,"started_at":"2026-07-23T07:55:07.898Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"94c7741de8644b5da225afc2ab92aaca833f1b9970bd8ef60ab25afcbd393b4f","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:11.714Z","source_record_sha256":"ddb4dd8f9c7e67b446b0afbac933185d49b2f9027b079dfd511548552f3e6df3","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"53649f10-7f15-439b-b58e-c8e01a811995","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-23T07:55:11.714Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:20.097Z","source_record_sha256":"59f4bd89d66c54fe7486331376c9ba579108485c9a765ecc207405b698402cae","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"cdefdbdd-5005-472b-b3e0-d33e3ecc51c5","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-23T07:55:20.097Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:28.789Z","source_record_sha256":"9cc943cb8d10c628630fb2425caccb1a8c3093774d7b7dda357ac6cb074b8d01","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"929db9db-2ef4-466d-a80b-2e14af1f4c7c","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T07:55:28.789Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:36.177Z","source_record_sha256":"7cb4afef06406a0e19a1ee831041bc440418e7a2819a80a18360e9386f6d7668","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"711a0f3a-7f40-4b3d-ab18-0fd661803757","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":1,"started_at":"2026-07-23T07:55:36.177Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9ca02b19c7d86f46ad7a35b1f996cc0789a5a1cb92c6c567fbf2233aefa59ed4","selection_grounded":true},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:40.433Z","source_record_sha256":"bb8a1ce15aaca2982405e532624ca02946339e8355c0a4b192b75a9a5e886b81","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"342d9249-b2e3-43e1-9db2-9d6584889277","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":2,"started_at":"2026-07-23T07:55:40.433Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9ca02b19c7d86f46ad7a35b1f996cc0789a5a1cb92c6c567fbf2233aefa59ed4","selection_grounded":true},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:43.901Z","source_record_sha256":"f8e45b5268313edd9fcf9dc8fc42facdac41bbf8aa973d87f286cab763f06ab8","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"063b8a50-9c2c-41dd-89df-4a639c354c74","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":3,"started_at":"2026-07-23T07:55:43.901Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9ca02b19c7d86f46ad7a35b1f996cc0789a5a1cb92c6c567fbf2233aefa59ed4","selection_grounded":true},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:47.413Z","source_record_sha256":"25401223352ec2b8591b28d187ddd4a8215e1ceec71e81d878195b48b3e11951","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"34788c2f-c12d-46b8-9a88-d4285ab363de","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":1,"started_at":"2026-07-23T07:55:47.413Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8533341a13a1599cae14cf51f2a2e2642be5fae6d57aa42b445fc87cf9803a1f","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:51.371Z","source_record_sha256":"2c20d520e1e846149c959212418c6111000c94379f2adac4e9f4a1f98c013e7c","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"1ae754a2-5fe1-4baa-85b8-e5c6e8664576","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":2,"started_at":"2026-07-23T07:55:51.371Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8533341a13a1599cae14cf51f2a2e2642be5fae6d57aa42b445fc87cf9803a1f","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:54.947Z","source_record_sha256":"a66bede429e003a5177bcab371c06cfb944263d91f57f05c2ebddbd991e58cb7","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"ddfc24d1-5ac5-4861-b959-7013b1160520","model":"oracle-v1","metadata":{"lane":"reference","adapter_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f/oracle-v1","toolkit_revision":"5006c2a0789e2352944ffd87b1a154b96ec0566f","task_set_sha256":"d1e18b3affdecd65c09b632ab476b907d09b46ac11718d9aa92156bd0d6f8866","api_base_url":"https://qveris.ai/api/v1","discovery_limit":10,"execute":true,"model_revision":"unreported","api_revision":"unreported","catalog_revision":"unreported","catalog_observation_sha256":"dcf0141d9e09bb7655a69e31d54a0a755f129b0e81aeda11e5eebbd35946e591","runtime":{"node":"unreported","platform":"unreported","arch":"unreported"},"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":3,"started_at":"2026-07-23T07:55:54.947Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"8533341a13a1599cae14cf51f2a2e2642be5fae6d57aa42b445fc87cf9803a1f","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T07:55:58.608Z","source_record_sha256":"f0eef74f6476cb2e6b010b7b0bef148fd16ead166a062cf4c11e3fcf8e5f2921","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} diff --git a/benchmarks/discover-call/results/2026-07-23-reference-v1-v4.runs.jsonl b/benchmarks/discover-call/results/2026-07-23-reference-v1-v4.runs.jsonl index 208067e..bdf0dde 100644 --- a/benchmarks/discover-call/results/2026-07-23-reference-v1-v4.runs.jsonl +++ b/benchmarks/discover-call/results/2026-07-23-reference-v1-v4.runs.jsonl @@ -1,54 +1,54 @@ -{"schema_version":1,"benchmark_version":"v1","run_id":"47cb7805-14a0-4e3d-9467-eada2d3d5aaf","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":1,"started_at":"2026-07-23T11:48:57.201Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"60a9778d19cb8b02430702c056caa75f4e19deac6783e4f41e81045a80bb5f80","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:01.956Z","source_record_sha256":"bb3eccb640d33192f56ffdcd31e28cbb6808e52d855bdb7d98950e6badebff90"} -{"schema_version":1,"benchmark_version":"v1","run_id":"cae0eb3b-e6de-4378-9aaa-7d8f52e5d199","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":2,"started_at":"2026-07-23T11:49:01.956Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"60a9778d19cb8b02430702c056caa75f4e19deac6783e4f41e81045a80bb5f80","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:06.391Z","source_record_sha256":"c7c17c944dadb3959a97004cf2a43106cdf94599d5a268b4ae42607eb7134db6"} -{"schema_version":1,"benchmark_version":"v1","run_id":"2f81c94d-6247-47eb-abb3-23b5a66053be","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":3,"started_at":"2026-07-23T11:49:06.391Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"60a9778d19cb8b02430702c056caa75f4e19deac6783e4f41e81045a80bb5f80","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["location"]},"parameters":{"location":"London","unitGroup":"metric","contentType":"json"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:09.865Z","source_record_sha256":"1a0f663ae9ddbe3530a06e497cb34a4bee8a0aae462ff40d9d2c8eee412bc102"} -{"schema_version":1,"benchmark_version":"v1","run_id":"a6fd0659-6976-499d-90e2-4db0e7d51719","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-23T11:49:09.865Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9f5212bec5a91748d3d60cb1901272c20496f288ccbbbb3d826f3920e3d66592","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:13.050Z","source_record_sha256":"8972f96aa8c5b2bf909568698fe1344d8a94ec200a021c241c9a2ba66a013250"} -{"schema_version":1,"benchmark_version":"v1","run_id":"47ba4b70-b8fe-4549-99c8-6c3e644e684b","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-23T11:49:13.050Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9f5212bec5a91748d3d60cb1901272c20496f288ccbbbb3d826f3920e3d66592","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:16.054Z","source_record_sha256":"36c277092f214e6254183f8843225aa6d2c44f7d0a96160fec638013892c5a9d"} -{"schema_version":1,"benchmark_version":"v1","run_id":"b9c6a6f7-dbf2-49a3-b37f-076232390cd9","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-23T11:49:16.055Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9f5212bec5a91748d3d60cb1901272c20496f288ccbbbb3d826f3920e3d66592","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"AAPL","format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:19.027Z","source_record_sha256":"0714760ecc4029e7b6d3d2fb1d8570a223380760705bf7e0d3e03b68373a6a30"} -{"schema_version":1,"benchmark_version":"v1","run_id":"eaccc622-c533-45d9-a3ad-fb8a6061a563","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-23T11:49:19.027Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"05301a2d5f1dcd68a46a128056393fce873a9c954f50a904b20d44e0da086501","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:22.152Z","source_record_sha256":"b22dfc0edc9fcec92bd0c137a12f7a5d3dfdac26c5e6b227e45a584e1aed5e5e"} -{"schema_version":1,"benchmark_version":"v1","run_id":"84f3dd2a-9da0-4510-8f41-bd13698f84f0","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-23T11:49:22.152Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"05301a2d5f1dcd68a46a128056393fce873a9c954f50a904b20d44e0da086501","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:24.995Z","source_record_sha256":"9dbea2b37f2c81c3c65d061a0959a4367cdd20faef7fa1dbfe9854ee23d8c6d8"} -{"schema_version":1,"benchmark_version":"v1","run_id":"1b9a3943-a23b-4ee1-a637-e55204be18ea","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-23T11:49:24.995Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"05301a2d5f1dcd68a46a128056393fce873a9c954f50a904b20d44e0da086501","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true,"required_parameters":["symbol","amount"]},"parameters":{"symbol":"USD/EUR","amount":1,"format":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:28.646Z","source_record_sha256":"d9fcd63fb991a169e7effa261baf8111331948a9e2e808cb0fa460ad338c2cd8"} -{"schema_version":1,"benchmark_version":"v1","run_id":"e59a22e8-5b81-4233-97f9-204dc9d37cd8","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":1,"started_at":"2026-07-23T11:49:28.646Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c4bbd90fb5d55233f0f6f0ef99607a3f2607d18fc477747e948347693409157a","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"artificial intelligence","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:31.575Z","source_record_sha256":"c564a710dfa464fe7a61301356f3661fe4a97897e15ed38fca59f8c68a59cffb"} -{"schema_version":1,"benchmark_version":"v1","run_id":"53f4c366-ae9d-4b54-9837-f29f855998b5","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":2,"started_at":"2026-07-23T11:49:31.575Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c4bbd90fb5d55233f0f6f0ef99607a3f2607d18fc477747e948347693409157a","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"artificial intelligence","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:34.512Z","source_record_sha256":"efd288148efa0f03827fc210788bdd569559a8e24ae29af457f6c322e87693a9"} -{"schema_version":1,"benchmark_version":"v1","run_id":"ba35f756-d608-4e36-846e-da6c997c012b","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":3,"started_at":"2026-07-23T11:49:34.512Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c4bbd90fb5d55233f0f6f0ef99607a3f2607d18fc477747e948347693409157a","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"artificial intelligence","language":"en","sortBy":"publishedAt","pageSize":20,"page":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:37.192Z","source_record_sha256":"1b4176d0ea0fbf9d04d7e61c271b9a8b9c8203a95fd1177f83fc28399caf85ac"} -{"schema_version":1,"benchmark_version":"v1","run_id":"4f315c20-69d3-4932-b9fe-9ac53abb26b3","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T11:49:37.192Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:41.059Z","source_record_sha256":"9f789afec504d169e789dba41d995393630dde1699d7f443486e5bfd7c363ff7"} -{"schema_version":1,"benchmark_version":"v1","run_id":"994aa1d9-989a-4cbb-bb65-0a6748264e2c","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T11:49:41.059Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:45.414Z","source_record_sha256":"843c439d6403310a6b963c590a9486f153dd92c6433ee058a8586a5da3acb57a"} -{"schema_version":1,"benchmark_version":"v1","run_id":"1e562721-df0e-401f-afb0-97173f7723bb","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T11:49:45.414Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true,"required_parameters":["query"]},"parameters":{"query":"climate risk","limit":10},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:49.536Z","source_record_sha256":"3c03fe59e1241ad2a33f7b06102a6a57b4717775752fdda2877901831cb423f4"} -{"schema_version":1,"benchmark_version":"v1","run_id":"dabf80ee-36bc-405a-94cd-674a3a026f48","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T11:49:49.536Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"f4605d6b4c45644fc9cec341c593f461cdef28806d63c58187e65c1311fd88b9","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id"},"finished_at":"2026-07-23T11:49:51.286Z","source_record_sha256":"ca3514c8b130287b1fdeda4d4059b03e09dbf2dbb5e7b7c92a629ab4868e5cef"} -{"schema_version":1,"benchmark_version":"v1","run_id":"cc6c272d-971c-45da-b3f0-0a980620a770","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T11:49:51.286Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"f4605d6b4c45644fc9cec341c593f461cdef28806d63c58187e65c1311fd88b9","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id"},"finished_at":"2026-07-23T11:49:53.649Z","source_record_sha256":"508f234db9002ea4f1a4f542cf03a5ceecc68ef5b2220f3a1aac0afbf7f735d0"} -{"schema_version":1,"benchmark_version":"v1","run_id":"93cd2a79-0d68-4912-8b80-281ff8b5afcf","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T11:49:53.649Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"f4605d6b4c45644fc9cec341c593f461cdef28806d63c58187e65c1311fd88b9","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false,"required_parameters":[]},"parameters":null,"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id"},"finished_at":"2026-07-23T11:49:55.417Z","source_record_sha256":"06831aa8ae3762571f2cff2bf7cfb286373e6af28ed4fcf7792799f26a66b91c"} -{"schema_version":1,"benchmark_version":"v1","run_id":"fe761783-51bd-469f-a93d-e70b02c5aca4","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T11:49:55.417Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"70aa2fe386fa3968550e5521c201c217ff7c531c47da064ac80c7028098b0fc1","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:58.783Z","source_record_sha256":"c4b1f95c00f2162eb2ae0ea1a1c18d32bfff43c1aa66c0e090ef29801ca59674"} -{"schema_version":1,"benchmark_version":"v1","run_id":"e066fabb-8e6a-43f1-8de1-118b9b2685d6","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T11:49:58.783Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"70aa2fe386fa3968550e5521c201c217ff7c531c47da064ac80c7028098b0fc1","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:02.312Z","source_record_sha256":"8e8f70b308a50ad0586d20cdae1b4600f8e023f8c311df4a60bcfdc364ee9f8b"} -{"schema_version":1,"benchmark_version":"v1","run_id":"38a5dda0-7976-47cf-8360-8de4b6616f38","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T11:50:02.312Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"70aa2fe386fa3968550e5521c201c217ff7c531c47da064ac80c7028098b0fc1","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true,"required_parameters":["address"]},"parameters":{"address":"北京市海淀区上地十街10号","city":"北京市","output":"JSON"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:05.390Z","source_record_sha256":"80804429ff13575a752328940d66e93c675d37de2fb9a20abe083ac6a063f5f2"} -{"schema_version":1,"benchmark_version":"v1","run_id":"a9efa250-3048-4843-9625-16b3719e367c","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-23T11:50:05.390Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e945d876b1c1f057c618eb121c5016982e2a640663a00218afdabf96a2f871c5","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true,"required_parameters":["ip"]},"parameters":{"ip":"8.8.8.8"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:08.634Z","source_record_sha256":"a655366007019129793bc3b859fd6ddbd995cb75a67e7df8e0f9e204aaa115f8"} -{"schema_version":1,"benchmark_version":"v1","run_id":"f78bf98f-2a48-4d41-a0be-6fee18aa0e6d","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-23T11:50:08.634Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e945d876b1c1f057c618eb121c5016982e2a640663a00218afdabf96a2f871c5","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true,"required_parameters":["ip"]},"parameters":{"ip":"8.8.8.8"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:11.669Z","source_record_sha256":"5794c2964429894a7ff28e15719e302d5c1b00006e0d46a48a0091aa0373d249"} -{"schema_version":1,"benchmark_version":"v1","run_id":"de52544a-c3ce-4ba2-9a30-136ff600bd46","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-23T11:50:11.669Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e945d876b1c1f057c618eb121c5016982e2a640663a00218afdabf96a2f871c5","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true,"required_parameters":["ip"]},"parameters":{"ip":"8.8.8.8"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:15.049Z","source_record_sha256":"3caa2c5028a16b205dc50eaa713fd02faacbce50e4b6e927d2e406716654467e"} -{"schema_version":1,"benchmark_version":"v1","run_id":"8b9d5b99-2532-4c71-bcbe-86fe01addec9","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":1,"started_at":"2026-07-23T11:50:15.049Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d6fafa5dcc5b80de51ca0ad0cd3e975827e3281c0aa5dbb07e57ede5fb7c75d6","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:18.517Z","source_record_sha256":"b0bd21d4827c242b5d7ac49306b3836434d584b594ccdd31d69aa98a8b5a2eaf"} -{"schema_version":1,"benchmark_version":"v1","run_id":"5750acdd-7fe8-42b7-8055-4e0b493a3563","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":2,"started_at":"2026-07-23T11:50:18.517Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d6fafa5dcc5b80de51ca0ad0cd3e975827e3281c0aa5dbb07e57ede5fb7c75d6","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:21.713Z","source_record_sha256":"9f5b324ec84f9a90f260e24eac25dfd714ca0342a5b446135e26d1d7ae90d649"} -{"schema_version":1,"benchmark_version":"v1","run_id":"9a5d4dd7-0b65-4fcb-8c0c-9236ad8e4385","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":3,"started_at":"2026-07-23T11:50:21.713Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d6fafa5dcc5b80de51ca0ad0cd3e975827e3281c0aa5dbb07e57ede5fb7c75d6","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"symbol":"BTCUSDT"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:24.746Z","source_record_sha256":"f095e22ade3257eea4e57f61ce50ad30bc703b39b7f3d40938e727c2da3a9864"} -{"schema_version":1,"benchmark_version":"v1","run_id":"ba98ef8c-c4a6-4451-9ddf-eeaf399350e6","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":1,"started_at":"2026-07-23T11:50:24.746Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c09c84639eec63b361d1f798b2ce033c380f12dcacaec1d15585f303caca3f52","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:28.102Z","source_record_sha256":"d2b118b09a4620806f4f7c95b0af97295280a9283ac1aa0047b285031a081d5f"} -{"schema_version":1,"benchmark_version":"v1","run_id":"89d87d76-b9b2-43a9-8e97-9b9e585abd2f","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":2,"started_at":"2026-07-23T11:50:28.102Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c09c84639eec63b361d1f798b2ce033c380f12dcacaec1d15585f303caca3f52","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:30.318Z","source_record_sha256":"44ef22f9bc5dc52ef9433772a2fcf042186d2b0728cfdece9f765562e95a23e3"} -{"schema_version":1,"benchmark_version":"v1","run_id":"84e9641c-d425-455f-9c5f-78af187aa8a0","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":3,"started_at":"2026-07-23T11:50:30.318Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c09c84639eec63b361d1f798b2ce033c380f12dcacaec1d15585f303caca3f52","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"model context protocol","count":"10"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:32.939Z","source_record_sha256":"388a156cfce0edd62ce1aea09c593fb1cd1bbe633286fe3e401233b3751a5bd5"} -{"schema_version":1,"benchmark_version":"v1","run_id":"bbc436b0-0423-438b-8c99-09c149b108c1","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":1,"started_at":"2026-07-23T11:50:32.939Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:35.293Z","source_record_sha256":"9d9d4b3c45840e35b730cdaf7e0dc764f5687407a14530008be2b0ce5739c3ca"} -{"schema_version":1,"benchmark_version":"v1","run_id":"6e8d097d-729b-49ff-9ff8-26d269b4c392","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":2,"started_at":"2026-07-23T11:50:35.293Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:37.516Z","source_record_sha256":"ea41ada6eef7178da3d7a67c61d73eeb45ed8012c9394e269ed2f9df4488abb9"} -{"schema_version":1,"benchmark_version":"v1","run_id":"36d8ce65-a35b-4f18-bce3-2130cb6bb18f","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":3,"started_at":"2026-07-23T11:50:37.516Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true,"required_parameters":["lat","lon"]},"parameters":{"lat":51.5,"lon":-0.12},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:39.898Z","source_record_sha256":"1023b919340a08acb80c944f97c62089c481d96b1701532db9e0a9a2c9531db5"} -{"schema_version":1,"benchmark_version":"v1","run_id":"2d63ffe2-9364-4a38-9143-310d2253f292","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":1,"started_at":"2026-07-23T11:50:39.898Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"bfe110e90706c2afbad72202d763dfe7ecc090a0acdd71d13ee5cd939020de3d","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:43.417Z","source_record_sha256":"345fecac7ee3c2f461a1883ab78429bf9c16530e648df0de26062da55ca1f3d6"} -{"schema_version":1,"benchmark_version":"v1","run_id":"ac9707ba-8ec7-4621-9bd3-f1924b19fc39","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":2,"started_at":"2026-07-23T11:50:43.417Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"bfe110e90706c2afbad72202d763dfe7ecc090a0acdd71d13ee5cd939020de3d","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:46.844Z","source_record_sha256":"412e4f494e67085a1b12d2a2f75ffda2f56119cbb5720e46acaf2d5e0d3704a9"} -{"schema_version":1,"benchmark_version":"v1","run_id":"5d5face2-a17f-4ebd-955e-f74779ff753b","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":3,"started_at":"2026-07-23T11:50:46.844Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"bfe110e90706c2afbad72202d763dfe7ecc090a0acdd71d13ee5cd939020de3d","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["category"]},"parameters":{"category":"general"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:50.621Z","source_record_sha256":"ab2435871876cb232c3f0ed08d7e54dccf7d2eb3d0ff732927fa2aca3f7225f5"} -{"schema_version":1,"benchmark_version":"v1","run_id":"571bc24e-6bfe-45c0-a24c-751d93c62aa5","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T11:50:50.621Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7c6eca1d49ca307d5d3c65dbae2d15a95977f722f7378d9d78ed7592e69a7e52","selection_grounded":true},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"selection_grounded":true,"required_parameters":["domain"]},"parameters":{"domain":"example.com"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:53.813Z","source_record_sha256":"7cf298e0f4bacd47caf7364fd7332bc0d9229efab746c328741892de23d0d398"} -{"schema_version":1,"benchmark_version":"v1","run_id":"7333c00c-ad5d-4355-9885-60619358b073","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T11:50:53.813Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7c6eca1d49ca307d5d3c65dbae2d15a95977f722f7378d9d78ed7592e69a7e52","selection_grounded":true},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"selection_grounded":true,"required_parameters":["domain"]},"parameters":{"domain":"example.com"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:56.683Z","source_record_sha256":"672e5cc17bf7fe7faf572e699f6b57e82403654fa42dd09ad94f28cb498cb521"} -{"schema_version":1,"benchmark_version":"v1","run_id":"305235ce-03f2-4ad3-a771-f88f78fdd385","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T11:50:56.683Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7c6eca1d49ca307d5d3c65dbae2d15a95977f722f7378d9d78ed7592e69a7e52","selection_grounded":true},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"selection_grounded":true,"required_parameters":["domain"]},"parameters":{"domain":"example.com"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:59.655Z","source_record_sha256":"164657ab106df8fac53297f629233beeb6eb774b0dd7f3eb33c7222660922809"} -{"schema_version":1,"benchmark_version":"v1","run_id":"bbd74432-588a-4d16-bada-e4719e2939aa","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":1,"started_at":"2026-07-23T11:50:59.655Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f226b4b6b337cb8cc47181f72cca9d36b6927f00b68525ffbabb1c97bff58d26","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:03.539Z","source_record_sha256":"1e9330a654eb3c466ed208b1ec3c9fbdad93b81a1c8b580e72da65774f9086c1"} -{"schema_version":1,"benchmark_version":"v1","run_id":"180041f2-2807-4ebe-b2b0-6fff70caad34","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":2,"started_at":"2026-07-23T11:51:03.539Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f226b4b6b337cb8cc47181f72cca9d36b6927f00b68525ffbabb1c97bff58d26","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:06.997Z","source_record_sha256":"2c8cbf34315ebea332cee35646f4553dd14347eef4e2f0c8e682586b69e0d799"} -{"schema_version":1,"benchmark_version":"v1","run_id":"09697d0a-d5dc-45c9-a58e-0f83265ad2a7","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":3,"started_at":"2026-07-23T11:51:06.997Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f226b4b6b337cb8cc47181f72cca9d36b6927f00b68525ffbabb1c97bff58d26","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":["q"]},"parameters":{"q":"Golden Gate Bridge"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:10.362Z","source_record_sha256":"87788e797974bcd217f59d742b914ce6feb80a617c7aa485bce22926d61f67b9"} -{"schema_version":1,"benchmark_version":"v1","run_id":"b2313e18-6665-425e-8282-d5c0dffc7269","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":1,"started_at":"2026-07-23T11:51:10.362Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6e380480779ccd9d93e77a96eac5da3e446beaaad9ad28b51fa3ba6c0de70b7b","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:13.250Z","source_record_sha256":"f179c532a1434f7d4b084dce5f38e3bf9af627b201bcbaccca5546a98bb673a3"} -{"schema_version":1,"benchmark_version":"v1","run_id":"0604aab2-f4dd-4ec5-80e2-07f0a46394a3","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":2,"started_at":"2026-07-23T11:51:13.250Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6e380480779ccd9d93e77a96eac5da3e446beaaad9ad28b51fa3ba6c0de70b7b","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:16.159Z","source_record_sha256":"8e86270c79ecc8f891910b1d651587290a81536167854a173d2195c5ac48eb37"} -{"schema_version":1,"benchmark_version":"v1","run_id":"73935f3c-23be-4b2a-9114-5a911d01c2ca","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":3,"started_at":"2026-07-23T11:51:16.159Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6e380480779ccd9d93e77a96eac5da3e446beaaad9ad28b51fa3ba6c0de70b7b","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true,"required_parameters":["exchange"]},"parameters":{"exchange":"US"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:18.898Z","source_record_sha256":"a5fb91158fbef2e1d09903059b5e697e9d0c96d38bb04fd7ad70b38ea73716e4"} -{"schema_version":1,"benchmark_version":"v1","run_id":"dad96c5e-3b67-40e4-8871-02d941c0f743","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-23T11:51:18.898Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:25.589Z","source_record_sha256":"49c08423482e2e2ff25d7ab791708b746e64610cd2d92e4f634753abbe3b872b"} -{"schema_version":1,"benchmark_version":"v1","run_id":"3fc5fb2c-f4ca-4362-9936-1758bde06182","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-23T11:51:25.589Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:32.866Z","source_record_sha256":"de289675bf86659570273300f6c458c005c184eee2ccfd418bde189ef3c7b4e5"} -{"schema_version":1,"benchmark_version":"v1","run_id":"698dc8bb-6506-4b16-a949-ce9468d44b63","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T11:51:32.866Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"format":"geojson"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:41.337Z","source_record_sha256":"e1168aa5bc62229b06e59c9e1062b95a3087434e90e9e344c107da4552c3e66d"} -{"schema_version":1,"benchmark_version":"v1","run_id":"9514047d-0a9a-4f1b-a14a-141f7676f466","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":1,"started_at":"2026-07-23T11:51:41.337Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e15912f4cbddea8b650e6fb9eecb8e09d60f2dd18d61bd0f1df0989c4a9f0cac","selection_grounded":true},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:45.787Z","source_record_sha256":"3d97bd5527879d8a88299cbf26255350fb2fcc3db92750ec339f613377f60049"} -{"schema_version":1,"benchmark_version":"v1","run_id":"bcce8d9a-66cc-40d3-b72b-13d1b8b306df","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":2,"started_at":"2026-07-23T11:51:45.787Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e15912f4cbddea8b650e6fb9eecb8e09d60f2dd18d61bd0f1df0989c4a9f0cac","selection_grounded":true},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:49.856Z","source_record_sha256":"b12a8f59abbe680b852bfabe6ca10816598effd87d19acf457132baffe58e4c6"} -{"schema_version":1,"benchmark_version":"v1","run_id":"da6af207-0055-474a-baa8-5286967891e1","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":3,"started_at":"2026-07-23T11:51:49.856Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e15912f4cbddea8b650e6fb9eecb8e09d60f2dd18d61bd0f1df0989c4a9f0cac","selection_grounded":true},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"selection_grounded":true,"required_parameters":[]},"parameters":{"cik":"0000320193"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:53.639Z","source_record_sha256":"ad82e114ec205a72a5ea96cfa3072470b6685e9237a832d134b535fc294eb66e"} -{"schema_version":1,"benchmark_version":"v1","run_id":"8cf8bcdd-bb75-4572-91db-44b6f075668b","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":1,"started_at":"2026-07-23T11:51:53.639Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"61aa7a6fac8a6b83b4e3502ad86f65d037077df0a9a9f50440a747046d39f06c","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"selection_grounded":true,"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:57.466Z","source_record_sha256":"f7966fd1f2a1dd3290af8a93df96ba4a6418e20a6254d7a4e1a936f463142dd2"} -{"schema_version":1,"benchmark_version":"v1","run_id":"9d55a677-d6ec-4061-b647-977bcb370dfc","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":2,"started_at":"2026-07-23T11:51:57.466Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"61aa7a6fac8a6b83b4e3502ad86f65d037077df0a9a9f50440a747046d39f06c","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"selection_grounded":true,"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:52:00.460Z","source_record_sha256":"b5fb410551a6642c3618ab6281dc8a97a3914fd7ce4d3ff7380da2e65427009f"} -{"schema_version":1,"benchmark_version":"v1","run_id":"dd1b7396-4eaa-4c46-af88-db113b37c68f","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":3,"started_at":"2026-07-23T11:52:00.460Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"61aa7a6fac8a6b83b4e3502ad86f65d037077df0a9a9f50440a747046d39f06c","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"selection_grounded":true,"required_parameters":["league","season"]},"parameters":{"league":39,"season":"2025"},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:52:03.442Z","source_record_sha256":"a16c58332263a82ffd97a879f731261d8ccbf4e3a18437b0e9883ea6e6ddb407"} +{"schema_version":1,"benchmark_version":"v1","run_id":"47cb7805-14a0-4e3d-9467-eada2d3d5aaf","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":1,"started_at":"2026-07-23T11:48:57.201Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"60a9778d19cb8b02430702c056caa75f4e19deac6783e4f41e81045a80bb5f80","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:01.956Z","source_record_sha256":"bb3eccb640d33192f56ffdcd31e28cbb6808e52d855bdb7d98950e6badebff90","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"cae0eb3b-e6de-4378-9aaa-7d8f52e5d199","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":2,"started_at":"2026-07-23T11:49:01.956Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"60a9778d19cb8b02430702c056caa75f4e19deac6783e4f41e81045a80bb5f80","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:06.391Z","source_record_sha256":"c7c17c944dadb3959a97004cf2a43106cdf94599d5a268b4ae42607eb7134db6","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"2f81c94d-6247-47eb-abb3-23b5a66053be","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":3,"started_at":"2026-07-23T11:49:06.391Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"60a9778d19cb8b02430702c056caa75f4e19deac6783e4f41e81045a80bb5f80","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:09.865Z","source_record_sha256":"1a0f663ae9ddbe3530a06e497cb34a4bee8a0aae462ff40d9d2c8eee412bc102","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"a6fd0659-6976-499d-90e2-4db0e7d51719","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-23T11:49:09.865Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9f5212bec5a91748d3d60cb1901272c20496f288ccbbbb3d826f3920e3d66592","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:13.050Z","source_record_sha256":"8972f96aa8c5b2bf909568698fe1344d8a94ec200a021c241c9a2ba66a013250","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"47ba4b70-b8fe-4549-99c8-6c3e644e684b","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-23T11:49:13.050Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9f5212bec5a91748d3d60cb1901272c20496f288ccbbbb3d826f3920e3d66592","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:16.054Z","source_record_sha256":"36c277092f214e6254183f8843225aa6d2c44f7d0a96160fec638013892c5a9d","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"b9c6a6f7-dbf2-49a3-b37f-076232390cd9","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-23T11:49:16.055Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9f5212bec5a91748d3d60cb1901272c20496f288ccbbbb3d826f3920e3d66592","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:19.027Z","source_record_sha256":"0714760ecc4029e7b6d3d2fb1d8570a223380760705bf7e0d3e03b68373a6a30","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"eaccc622-c533-45d9-a3ad-fb8a6061a563","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-23T11:49:19.027Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"05301a2d5f1dcd68a46a128056393fce873a9c954f50a904b20d44e0da086501","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:22.152Z","source_record_sha256":"b22dfc0edc9fcec92bd0c137a12f7a5d3dfdac26c5e6b227e45a584e1aed5e5e","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"84f3dd2a-9da0-4510-8f41-bd13698f84f0","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-23T11:49:22.152Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"05301a2d5f1dcd68a46a128056393fce873a9c954f50a904b20d44e0da086501","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:24.995Z","source_record_sha256":"9dbea2b37f2c81c3c65d061a0959a4367cdd20faef7fa1dbfe9854ee23d8c6d8","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"1b9a3943-a23b-4ee1-a637-e55204be18ea","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-23T11:49:24.995Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"05301a2d5f1dcd68a46a128056393fce873a9c954f50a904b20d44e0da086501","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:28.646Z","source_record_sha256":"d9fcd63fb991a169e7effa261baf8111331948a9e2e808cb0fa460ad338c2cd8","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"e59a22e8-5b81-4233-97f9-204dc9d37cd8","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":1,"started_at":"2026-07-23T11:49:28.646Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c4bbd90fb5d55233f0f6f0ef99607a3f2607d18fc477747e948347693409157a","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:31.575Z","source_record_sha256":"c564a710dfa464fe7a61301356f3661fe4a97897e15ed38fca59f8c68a59cffb","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"53f4c366-ae9d-4b54-9837-f29f855998b5","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":2,"started_at":"2026-07-23T11:49:31.575Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c4bbd90fb5d55233f0f6f0ef99607a3f2607d18fc477747e948347693409157a","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:34.512Z","source_record_sha256":"efd288148efa0f03827fc210788bdd569559a8e24ae29af457f6c322e87693a9","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"ba35f756-d608-4e36-846e-da6c997c012b","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":3,"started_at":"2026-07-23T11:49:34.512Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c4bbd90fb5d55233f0f6f0ef99607a3f2607d18fc477747e948347693409157a","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:37.192Z","source_record_sha256":"1b4176d0ea0fbf9d04d7e61c271b9a8b9c8203a95fd1177f83fc28399caf85ac","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"4f315c20-69d3-4932-b9fe-9ac53abb26b3","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":1,"started_at":"2026-07-23T11:49:37.192Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:41.059Z","source_record_sha256":"9f789afec504d169e789dba41d995393630dde1699d7f443486e5bfd7c363ff7","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"994aa1d9-989a-4cbb-bb65-0a6748264e2c","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":2,"started_at":"2026-07-23T11:49:41.059Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:45.414Z","source_record_sha256":"843c439d6403310a6b963c590a9486f153dd92c6433ee058a8586a5da3acb57a","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"1e562721-df0e-401f-afb0-97173f7723bb","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":3,"started_at":"2026-07-23T11:49:45.414Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:49.536Z","source_record_sha256":"3c03fe59e1241ad2a33f7b06102a6a57b4717775752fdda2877901831cb423f4","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"dabf80ee-36bc-405a-94cd-674a3a026f48","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-23T11:49:49.536Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"f4605d6b4c45644fc9cec341c593f461cdef28806d63c58187e65c1311fd88b9","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false},"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id"},"finished_at":"2026-07-23T11:49:51.286Z","source_record_sha256":"ca3514c8b130287b1fdeda4d4059b03e09dbf2dbb5e7b7c92a629ab4868e5cef","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"cc6c272d-971c-45da-b3f0-0a980620a770","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-23T11:49:51.286Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"f4605d6b4c45644fc9cec341c593f461cdef28806d63c58187e65c1311fd88b9","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false},"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id"},"finished_at":"2026-07-23T11:49:53.649Z","source_record_sha256":"508f234db9002ea4f1a4f542cf03a5ceecc68ef5b2220f3a1aac0afbf7f735d0","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"93cd2a79-0d68-4912-8b80-281ff8b5afcf","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-23T11:49:53.649Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"f4605d6b4c45644fc9cec341c593f461cdef28806d63c58187e65c1311fd88b9","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false},"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id"},"finished_at":"2026-07-23T11:49:55.417Z","source_record_sha256":"06831aa8ae3762571f2cff2bf7cfb286373e6af28ed4fcf7792799f26a66b91c","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0}} +{"schema_version":1,"benchmark_version":"v1","run_id":"fe761783-51bd-469f-a93d-e70b02c5aca4","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":1,"started_at":"2026-07-23T11:49:55.417Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"70aa2fe386fa3968550e5521c201c217ff7c531c47da064ac80c7028098b0fc1","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:49:58.783Z","source_record_sha256":"c4b1f95c00f2162eb2ae0ea1a1c18d32bfff43c1aa66c0e090ef29801ca59674","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"e066fabb-8e6a-43f1-8de1-118b9b2685d6","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":2,"started_at":"2026-07-23T11:49:58.783Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"70aa2fe386fa3968550e5521c201c217ff7c531c47da064ac80c7028098b0fc1","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:02.312Z","source_record_sha256":"8e8f70b308a50ad0586d20cdae1b4600f8e023f8c311df4a60bcfdc364ee9f8b","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"38a5dda0-7976-47cf-8360-8de4b6616f38","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":3,"started_at":"2026-07-23T11:50:02.312Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"70aa2fe386fa3968550e5521c201c217ff7c531c47da064ac80c7028098b0fc1","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:05.390Z","source_record_sha256":"80804429ff13575a752328940d66e93c675d37de2fb9a20abe083ac6a063f5f2","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"a9efa250-3048-4843-9625-16b3719e367c","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-23T11:50:05.390Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e945d876b1c1f057c618eb121c5016982e2a640663a00218afdabf96a2f871c5","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:08.634Z","source_record_sha256":"a655366007019129793bc3b859fd6ddbd995cb75a67e7df8e0f9e204aaa115f8","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"f78bf98f-2a48-4d41-a0be-6fee18aa0e6d","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-23T11:50:08.634Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e945d876b1c1f057c618eb121c5016982e2a640663a00218afdabf96a2f871c5","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:11.669Z","source_record_sha256":"5794c2964429894a7ff28e15719e302d5c1b00006e0d46a48a0091aa0373d249","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"de52544a-c3ce-4ba2-9a30-136ff600bd46","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-23T11:50:11.669Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e945d876b1c1f057c618eb121c5016982e2a640663a00218afdabf96a2f871c5","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:15.049Z","source_record_sha256":"3caa2c5028a16b205dc50eaa713fd02faacbce50e4b6e927d2e406716654467e","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"8b9d5b99-2532-4c71-bcbe-86fe01addec9","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":1,"started_at":"2026-07-23T11:50:15.049Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d6fafa5dcc5b80de51ca0ad0cd3e975827e3281c0aa5dbb07e57ede5fb7c75d6","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:18.517Z","source_record_sha256":"b0bd21d4827c242b5d7ac49306b3836434d584b594ccdd31d69aa98a8b5a2eaf","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"5750acdd-7fe8-42b7-8055-4e0b493a3563","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":2,"started_at":"2026-07-23T11:50:18.517Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d6fafa5dcc5b80de51ca0ad0cd3e975827e3281c0aa5dbb07e57ede5fb7c75d6","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:21.713Z","source_record_sha256":"9f5b324ec84f9a90f260e24eac25dfd714ca0342a5b446135e26d1d7ae90d649","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"9a5d4dd7-0b65-4fcb-8c0c-9236ad8e4385","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":3,"started_at":"2026-07-23T11:50:21.713Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d6fafa5dcc5b80de51ca0ad0cd3e975827e3281c0aa5dbb07e57ede5fb7c75d6","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:24.746Z","source_record_sha256":"f095e22ade3257eea4e57f61ce50ad30bc703b39b7f3d40938e727c2da3a9864","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"ba98ef8c-c4a6-4451-9ddf-eeaf399350e6","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":1,"started_at":"2026-07-23T11:50:24.746Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c09c84639eec63b361d1f798b2ce033c380f12dcacaec1d15585f303caca3f52","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:28.102Z","source_record_sha256":"d2b118b09a4620806f4f7c95b0af97295280a9283ac1aa0047b285031a081d5f","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"89d87d76-b9b2-43a9-8e97-9b9e585abd2f","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":2,"started_at":"2026-07-23T11:50:28.102Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c09c84639eec63b361d1f798b2ce033c380f12dcacaec1d15585f303caca3f52","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:30.318Z","source_record_sha256":"44ef22f9bc5dc52ef9433772a2fcf042186d2b0728cfdece9f765562e95a23e3","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"84e9641c-d425-455f-9c5f-78af187aa8a0","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":3,"started_at":"2026-07-23T11:50:30.318Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"c09c84639eec63b361d1f798b2ce033c380f12dcacaec1d15585f303caca3f52","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:32.939Z","source_record_sha256":"388a156cfce0edd62ce1aea09c593fb1cd1bbe633286fe3e401233b3751a5bd5","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"bbc436b0-0423-438b-8c99-09c149b108c1","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":1,"started_at":"2026-07-23T11:50:32.939Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:35.293Z","source_record_sha256":"9d9d4b3c45840e35b730cdaf7e0dc764f5687407a14530008be2b0ce5739c3ca","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"6e8d097d-729b-49ff-9ff8-26d269b4c392","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":2,"started_at":"2026-07-23T11:50:35.293Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:37.516Z","source_record_sha256":"ea41ada6eef7178da3d7a67c61d73eeb45ed8012c9394e269ed2f9df4488abb9","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"36d8ce65-a35b-4f18-bce3-2130cb6bb18f","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":3,"started_at":"2026-07-23T11:50:37.516Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:39.898Z","source_record_sha256":"1023b919340a08acb80c944f97c62089c481d96b1701532db9e0a9a2c9531db5","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"2d63ffe2-9364-4a38-9143-310d2253f292","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":1,"started_at":"2026-07-23T11:50:39.898Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"bfe110e90706c2afbad72202d763dfe7ecc090a0acdd71d13ee5cd939020de3d","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:43.417Z","source_record_sha256":"345fecac7ee3c2f461a1883ab78429bf9c16530e648df0de26062da55ca1f3d6","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"ac9707ba-8ec7-4621-9bd3-f1924b19fc39","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":2,"started_at":"2026-07-23T11:50:43.417Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"bfe110e90706c2afbad72202d763dfe7ecc090a0acdd71d13ee5cd939020de3d","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:46.844Z","source_record_sha256":"412e4f494e67085a1b12d2a2f75ffda2f56119cbb5720e46acaf2d5e0d3704a9","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"5d5face2-a17f-4ebd-955e-f74779ff753b","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":3,"started_at":"2026-07-23T11:50:46.844Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"bfe110e90706c2afbad72202d763dfe7ecc090a0acdd71d13ee5cd939020de3d","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:50.621Z","source_record_sha256":"ab2435871876cb232c3f0ed08d7e54dccf7d2eb3d0ff732927fa2aca3f7225f5","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"571bc24e-6bfe-45c0-a24c-751d93c62aa5","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-23T11:50:50.621Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7c6eca1d49ca307d5d3c65dbae2d15a95977f722f7378d9d78ed7592e69a7e52","selection_grounded":true},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:53.813Z","source_record_sha256":"7cf298e0f4bacd47caf7364fd7332bc0d9229efab746c328741892de23d0d398","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"7333c00c-ad5d-4355-9885-60619358b073","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":2,"started_at":"2026-07-23T11:50:53.813Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7c6eca1d49ca307d5d3c65dbae2d15a95977f722f7378d9d78ed7592e69a7e52","selection_grounded":true},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:56.683Z","source_record_sha256":"672e5cc17bf7fe7faf572e699f6b57e82403654fa42dd09ad94f28cb498cb521","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"305235ce-03f2-4ad3-a771-f88f78fdd385","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-23T11:50:56.683Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7c6eca1d49ca307d5d3c65dbae2d15a95977f722f7378d9d78ed7592e69a7e52","selection_grounded":true},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:50:59.655Z","source_record_sha256":"164657ab106df8fac53297f629233beeb6eb774b0dd7f3eb33c7222660922809","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"bbd74432-588a-4d16-bada-e4719e2939aa","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":1,"started_at":"2026-07-23T11:50:59.655Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f226b4b6b337cb8cc47181f72cca9d36b6927f00b68525ffbabb1c97bff58d26","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:03.539Z","source_record_sha256":"1e9330a654eb3c466ed208b1ec3c9fbdad93b81a1c8b580e72da65774f9086c1","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"180041f2-2807-4ebe-b2b0-6fff70caad34","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":2,"started_at":"2026-07-23T11:51:03.539Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f226b4b6b337cb8cc47181f72cca9d36b6927f00b68525ffbabb1c97bff58d26","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:06.997Z","source_record_sha256":"2c8cbf34315ebea332cee35646f4553dd14347eef4e2f0c8e682586b69e0d799","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"09697d0a-d5dc-45c9-a58e-0f83265ad2a7","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":3,"started_at":"2026-07-23T11:51:06.997Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"f226b4b6b337cb8cc47181f72cca9d36b6927f00b68525ffbabb1c97bff58d26","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:10.362Z","source_record_sha256":"87788e797974bcd217f59d742b914ce6feb80a617c7aa485bce22926d61f67b9","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"b2313e18-6665-425e-8282-d5c0dffc7269","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":1,"started_at":"2026-07-23T11:51:10.362Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6e380480779ccd9d93e77a96eac5da3e446beaaad9ad28b51fa3ba6c0de70b7b","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:13.250Z","source_record_sha256":"f179c532a1434f7d4b084dce5f38e3bf9af627b201bcbaccca5546a98bb673a3","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"0604aab2-f4dd-4ec5-80e2-07f0a46394a3","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":2,"started_at":"2026-07-23T11:51:13.250Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6e380480779ccd9d93e77a96eac5da3e446beaaad9ad28b51fa3ba6c0de70b7b","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:16.159Z","source_record_sha256":"8e86270c79ecc8f891910b1d651587290a81536167854a173d2195c5ac48eb37","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"73935f3c-23be-4b2a-9114-5a911d01c2ca","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":3,"started_at":"2026-07-23T11:51:16.159Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6e380480779ccd9d93e77a96eac5da3e446beaaad9ad28b51fa3ba6c0de70b7b","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:18.898Z","source_record_sha256":"a5fb91158fbef2e1d09903059b5e697e9d0c96d38bb04fd7ad70b38ea73716e4","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"dad96c5e-3b67-40e4-8871-02d941c0f743","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-23T11:51:18.898Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:25.589Z","source_record_sha256":"49c08423482e2e2ff25d7ab791708b746e64610cd2d92e4f634753abbe3b872b","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"3fc5fb2c-f4ca-4362-9936-1758bde06182","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-23T11:51:25.589Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:32.866Z","source_record_sha256":"de289675bf86659570273300f6c458c005c184eee2ccfd418bde189ef3c7b4e5","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"698dc8bb-6506-4b16-a949-ce9468d44b63","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-23T11:51:32.866Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:41.337Z","source_record_sha256":"e1168aa5bc62229b06e59c9e1062b95a3087434e90e9e344c107da4552c3e66d","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"9514047d-0a9a-4f1b-a14a-141f7676f466","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":1,"started_at":"2026-07-23T11:51:41.337Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e15912f4cbddea8b650e6fb9eecb8e09d60f2dd18d61bd0f1df0989c4a9f0cac","selection_grounded":true},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:45.787Z","source_record_sha256":"3d97bd5527879d8a88299cbf26255350fb2fcc3db92750ec339f613377f60049","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"bcce8d9a-66cc-40d3-b72b-13d1b8b306df","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":2,"started_at":"2026-07-23T11:51:45.787Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e15912f4cbddea8b650e6fb9eecb8e09d60f2dd18d61bd0f1df0989c4a9f0cac","selection_grounded":true},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:49.856Z","source_record_sha256":"b12a8f59abbe680b852bfabe6ca10816598effd87d19acf457132baffe58e4c6","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"da6af207-0055-474a-baa8-5286967891e1","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":3,"started_at":"2026-07-23T11:51:49.856Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e15912f4cbddea8b650e6fb9eecb8e09d60f2dd18d61bd0f1df0989c4a9f0cac","selection_grounded":true},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:53.639Z","source_record_sha256":"ad82e114ec205a72a5ea96cfa3072470b6685e9237a832d134b535fc294eb66e","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"8cf8bcdd-bb75-4572-91db-44b6f075668b","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":1,"started_at":"2026-07-23T11:51:53.639Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"61aa7a6fac8a6b83b4e3502ad86f65d037077df0a9a9f50440a747046d39f06c","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:51:57.466Z","source_record_sha256":"f7966fd1f2a1dd3290af8a93df96ba4a6418e20a6254d7a4e1a936f463142dd2","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"9d55a677-d6ec-4061-b647-977bcb370dfc","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":2,"started_at":"2026-07-23T11:51:57.466Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"61aa7a6fac8a6b83b4e3502ad86f65d037077df0a9a9f50440a747046d39f06c","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:52:00.460Z","source_record_sha256":"b5fb410551a6642c3618ab6281dc8a97a3914fd7ce4d3ff7380da2e65427009f","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} +{"schema_version":1,"benchmark_version":"v1","run_id":"dd1b7396-4eaa-4c46-af88-db113b37c68f","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"deeda797e50a0dae649bf91d7503d09ec497b324/reference-v1","toolkit_revision":"deeda797e50a0dae649bf91d7503d09ec497b324","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-22.1","catalog_revision":"unreported","catalog_observation_sha256":"7b342ab2738b78597c650fec975b92a919883cf3ad87dadc3fc5316360e6e9a1","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":3,"started_at":"2026-07-23T11:52:00.460Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"61aa7a6fac8a6b83b4e3502ad86f65d037077df0a9a9f50440a747046d39f06c","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"selection_grounded":true},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-23T11:52:03.442Z","source_record_sha256":"a16c58332263a82ffd97a879f731261d8ccbf4e3a18437b0e9883ea6e6ddb407","parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1}} diff --git a/benchmarks/discover-call/results/README.md b/benchmarks/discover-call/results/README.md index 797dce9..ed2c274 100644 --- a/benchmarks/discover-call/results/README.md +++ b/benchmarks/discover-call/results/README.md @@ -5,6 +5,9 @@ The v4 baseline compares the curated reference route and the configured model over immutable `tasks/v4.jsonl`: 18 tasks, three trials per task, Top 10 discovery, and real calls. All 54 trials in each lane are retained. +Public run records omit raw parameter values and retain only their scored +required-parameter and constraint-accuracy attestations. Inspected parameter +names are also omitted. | Metric | Curated reference route | `gpt-5.6-sol` configured model | | --- | ---: | ---: | diff --git a/benchmarks/discover-call/src/publication.mjs b/benchmarks/discover-call/src/publication.mjs index 467bfff..c120589 100644 --- a/benchmarks/discover-call/src/publication.mjs +++ b/benchmarks/discover-call/src/publication.mjs @@ -1,5 +1,29 @@ import { createHash } from 'node:crypto'; +import { scoreRecord } from './scoring.mjs'; + +const PUBLIC_API_HOSTS = new Set(['qveris.ai', 'qveris.cn']); + +const PUBLIC_RECORD_FIELDS = new Set([ + 'schema_version', + 'benchmark_version', + 'run_id', + 'model', + 'metadata', + 'task_id', + 'trial', + 'started_at', + 'status', + 'discovery', + 'selection', + 'inspection', + 'parameterization', + 'call', + 'error', + 'finished_at', + 'source_record_sha256', +]); + const PUBLIC_METADATA_FIELDS = new Set([ 'lane', 'model_revision', @@ -16,6 +40,21 @@ const PUBLIC_METADATA_FIELDS = new Set([ 'publication_policy', ]); +const PUBLIC_RUNTIME_FIELDS = new Set(['node', 'platform', 'arch']); +const PUBLIC_DISCOVERY_FIELDS = new Set([ + 'result_count', + 'snapshot_sha256', + 'selection_grounded', +]); +const PUBLIC_SELECTION_FIELDS = new Set(['tool_id', 'tool_id_sha256']); +const PUBLIC_INSPECTION_FIELDS = new Set(['selection_grounded']); +const PUBLIC_PARAMETERIZATION_FIELDS = new Set([ + 'required_parameter_accuracy', + 'constraint_accuracy', +]); +const PUBLIC_CALL_FIELDS = new Set(['attempted', 'success', 'result_nonempty']); +const PUBLIC_ERROR_FIELDS = new Set(['stage', 'reason_code']); + const REQUIRED_FORBIDDEN_FIELDS = [ 'execution_id', 'search_id', @@ -30,10 +69,13 @@ const REQUIRED_FORBIDDEN_FIELDS = [ 'authorization', 'client_secret', 'password', + 'parameters', + 'required_parameters', ]; -export function sanitizePublicRecords(records, policy) { +export function sanitizePublicRecords(records, policy, tasks) { validatePolicy(policy); + const taskById = taskMap(tasks); const approvedToolIds = new Set(policy.approved_selected_tool_ids); const catalogObservationSha256 = commonMetadataValue(records, 'catalog_observation_sha256') ?? @@ -47,18 +89,30 @@ export function sanitizePublicRecords(records, policy) { ), ); return records.map((record) => - sanitizePublicRecord(record, policy, approvedToolIds, catalogObservationSha256), + sanitizePublicRecord( + record, + taskById.get(record.task_id), + policy, + approvedToolIds, + catalogObservationSha256, + ), ); } -export function sanitizePublicRecord( +function sanitizePublicRecord( record, + task, policy, approvedToolIds = new Set(policy.approved_selected_tool_ids), catalogObservationSha256 = record.metadata?.catalog_observation_sha256 ?? 'unreported', ) { + if (!task) { + throw new Error(`Public artifact references unknown task: ${record?.task_id ?? ''}`); + } const selectedToolId = record.selection?.tool_id ?? null; const selectedToolApproved = selectedToolId ? approvedToolIds.has(selectedToolId) : false; + const { parameterization: _untrustedParameterization, ...rawScoringRecord } = record; + const scored = scoreRecord(task, rawScoringRecord); const resultToolIds = array(record.discovery?.result_tool_ids); const returnedToolIds = array(record.inspection?.returned_tool_ids); @@ -97,9 +151,11 @@ export function sanitizePublicRecord( }, inspection: { selection_grounded: inspectionGrounded, - required_parameters: array(record.inspection?.required_parameters), }, - parameters: policy.publish_parameters === true ? record.parameters : null, + parameterization: { + required_parameter_accuracy: scored.required_parameter_accuracy, + constraint_accuracy: scored.constraint_accuracy, + }, call, ...(record.error ? { error: publicError(record.error) } : {}), finished_at: record.finished_at, @@ -125,6 +181,13 @@ export function validatePolicy(policy) { ) { throw new Error('Publication policy needs approved_selected_tool_ids'); } + if ( + !Array.isArray(policy.approved_api_base_urls) || + policy.approved_api_base_urls.length === 0 || + policy.approved_api_base_urls.some((url) => !safePublicApiUrl(url)) + ) { + throw new Error('Publication policy needs approved public API base URLs'); + } if (!Array.isArray(policy.forbidden_fields)) { throw new Error('Publication policy needs forbidden_fields'); } @@ -147,6 +210,9 @@ export function validatePolicy(policy) { if (policy.unapproved_selected_tool_handling !== 'hash') { throw new Error('Publication policy must hash unapproved selected tools'); } + if (policy.publish_parameters !== false) { + throw new Error('Publication policy must not publish raw parameter values'); + } } export function validatePublicRecords(records, policy) { @@ -155,18 +221,57 @@ export function validatePublicRecords(records, policy) { const forbidden = forbiddenFieldSet(policy); for (const record of records) { assertNoForbiddenFields(record, forbidden); + assertObjectFields(record, PUBLIC_RECORD_FIELDS, '$'); + if ( + record.schema_version !== 1 || + record.benchmark_version !== 'v1' || + !nonemptyString(record.run_id) || + !nonemptyString(record.model) || + !nonemptyString(record.task_id) || + !Number.isInteger(record.trial) || + record.trial < 1 || + !['completed', 'failed'].includes(record.status) || + !validTimestamp(record.started_at) || + !validTimestamp(record.finished_at) + ) { + throw new Error('Public artifact has invalid record identity or lifecycle fields'); + } if (record.metadata?.publication_policy !== policy.policy_id) { throw new Error(`Public artifact is missing publication policy ${policy.policy_id}`); } - for (const field of Object.keys(record.metadata ?? {})) { - if (!PUBLIC_METADATA_FIELDS.has(field)) { - throw new Error(`Public artifact contains unsupported metadata: ${field}`); - } + assertObjectFields(record.metadata, PUBLIC_METADATA_FIELDS, '$.metadata'); + assertObjectFields(record.metadata.runtime, PUBLIC_RUNTIME_FIELDS, '$.metadata.runtime'); + assertObjectFields(record.discovery, PUBLIC_DISCOVERY_FIELDS, '$.discovery'); + assertObjectFields(record.selection, PUBLIC_SELECTION_FIELDS, '$.selection'); + assertObjectFields(record.inspection, PUBLIC_INSPECTION_FIELDS, '$.inspection'); + assertObjectFields( + record.parameterization, + PUBLIC_PARAMETERIZATION_FIELDS, + '$.parameterization', + ); + assertObjectFields(record.call, PUBLIC_CALL_FIELDS, '$.call'); + if (record.error !== undefined) { + assertObjectFields(record.error, PUBLIC_ERROR_FIELDS, '$.error'); } const selectedToolId = record.selection?.tool_id; + if (!policy.approved_api_base_urls.includes(record.metadata.api_base_url)) { + throw new Error(`Public artifact contains an unapproved API base URL`); + } + if ( + !isSha256(record.metadata.task_set_sha256) || + !isSha256(record.metadata.catalog_observation_sha256) || + !Number.isInteger(record.metadata.discovery_limit) || + record.metadata.discovery_limit < 1 || + typeof record.metadata.execute !== 'boolean' + ) { + throw new Error('Public artifact has invalid reproducibility metadata'); + } if (selectedToolId && !approvedToolIds.has(selectedToolId)) { throw new Error(`Public artifact contains an unapproved selected tool: ${selectedToolId}`); } + if (selectedToolId !== null && !nonemptyString(selectedToolId)) { + throw new Error('Public artifact selected tool must be a string or null'); + } if ( record.selection?.tool_id_sha256 !== undefined && (!isSha256(record.selection.tool_id_sha256) || selectedToolId) @@ -175,6 +280,7 @@ export function validatePublicRecords(records, policy) { } if ( !Number.isInteger(record.discovery?.result_count) || + record.discovery.result_count < 0 || !isSha256(record.discovery?.snapshot_sha256) ) { throw new Error('Public artifact discovery needs a count and SHA-256 snapshot'); @@ -182,17 +288,70 @@ export function validatePublicRecords(records, policy) { if (!isSha256(record.source_record_sha256)) { throw new Error('Public artifact needs source_record_sha256'); } - if (record.error) { - const errorFields = Object.keys(record.error); - if (errorFields.some((field) => !['stage', 'reason_code'].includes(field))) { - throw new Error('Public artifact error contains unsupported fields'); - } + if ( + !metric(record.parameterization?.required_parameter_accuracy) || + !metric(record.parameterization?.constraint_accuracy) + ) { + throw new Error('Public artifact parameterization metrics must be between 0 and 1'); + } + if ( + typeof record.discovery?.selection_grounded !== 'boolean' || + typeof record.inspection?.selection_grounded !== 'boolean' + ) { + throw new Error('Public artifact grounding attestations must be booleans'); + } + if ( + typeof record.call?.attempted !== 'boolean' || + !booleanOrNull(record.call?.success) || + (record.call?.result_nonempty !== undefined && + typeof record.call.result_nonempty !== 'boolean') + ) { + throw new Error('Public artifact call fields have invalid types'); + } + if ( + record.error !== undefined && + (safeCode(record.error.stage) !== record.error.stage || + safeCode(record.error.reason_code) !== record.error.reason_code) + ) { + throw new Error('Public artifact error codes are invalid'); + } + } +} + +export function validateOfficialPublicRun( + records, + { taskSetSha256, minTrialsPerTask = 3 } = {}, +) { + if (!isSha256(taskSetSha256)) throw new Error('Official public run needs the task-set SHA-256'); + if (!Number.isInteger(minTrialsPerTask) || minTrialsPerTask < 1) { + throw new Error('minTrialsPerTask must be a positive integer'); + } + const trials = new Map(); + for (const record of records) { + if (record.metadata?.task_set_sha256 !== taskSetSha256) { + throw new Error(`Public artifact task-set digest does not match ${record.task_id}`); + } + if (record.metadata?.execute !== true) { + throw new Error(`Official public artifact must execute every trial: ${record.task_id}`); + } + const key = `${record.model}\0${record.task_id}`; + trials.set(key, (trials.get(key) ?? 0) + 1); + } + for (const [key, count] of trials) { + if (count < minTrialsPerTask) { + const taskId = key.slice(key.indexOf('\0') + 1); + throw new Error( + `Official public artifact needs at least ${minTrialsPerTask} trials for ${taskId}`, + ); } } } function publicMetadata(metadata = {}, policy, catalogObservationSha256) { const lane = policy.legacy_lane_rewrites?.[metadata.lane] ?? metadata.lane ?? 'model'; + if (!policy.approved_api_base_urls.includes(metadata.api_base_url)) { + throw new Error('Raw benchmark record uses an unapproved public API base URL'); + } return { lane, model_revision: metadata.model_revision ?? 'unreported', @@ -236,6 +395,15 @@ function assertNoForbiddenFields(value, forbidden, path = '$') { } } +function assertObjectFields(value, allowed, path) { + if (!plainObject(value)) throw new Error(`Public artifact field ${path} must be an object`); + for (const field of Object.keys(value)) { + if (!allowed.has(field)) { + throw new Error(`Public artifact contains unsupported field at ${path}.${field}`); + } + } +} + function publicError(error) { return { stage: safeCode(error?.stage), @@ -263,6 +431,58 @@ function isSha256(value) { return typeof value === 'string' && /^[a-f0-9]{64}$/.test(value); } +function metric(value) { + return typeof value === 'number' && Number.isFinite(value) && value >= 0 && value <= 1; +} + +function booleanOrNull(value) { + return typeof value === 'boolean' || value === null; +} + +function nonemptyString(value) { + return typeof value === 'string' && value.length > 0; +} + +function validTimestamp(value) { + return ( + typeof value === 'string' && + /^\d{4}-\d{2}-\d{2}T/.test(value) && + Number.isFinite(Date.parse(value)) + ); +} + +function safePublicApiUrl(value) { + if (typeof value !== 'string') return false; + try { + const parsed = new URL(value); + return ( + parsed.protocol === 'https:' && + PUBLIC_API_HOSTS.has(parsed.hostname) && + !parsed.username && + !parsed.password && + !parsed.search && + !parsed.hash && + parsed.pathname.replace(/\/+$/, '') === '/api/v1' + ); + } catch { + return false; + } +} + +function taskMap(tasks) { + if (!Array.isArray(tasks) || tasks.length === 0) { + throw new Error('Public artifact publication needs benchmark tasks'); + } + const result = new Map(); + for (const task of tasks) { + if (!plainObject(task) || typeof task.id !== 'string' || !task.id || result.has(task.id)) { + throw new Error('Public artifact publication needs unique task ids'); + } + result.set(task.id, task); + } + return result; +} + function array(value) { return Array.isArray(value) ? value : []; } diff --git a/benchmarks/discover-call/src/publish.mjs b/benchmarks/discover-call/src/publish.mjs index 6a5a90c..7661d91 100644 --- a/benchmarks/discover-call/src/publish.mjs +++ b/benchmarks/discover-call/src/publish.mjs @@ -1,26 +1,40 @@ #!/usr/bin/env node +import { createHash } from 'node:crypto'; import { mkdir, readFile, writeFile } from 'node:fs/promises'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { readJsonLines, writeJsonLines } from './io.mjs'; -import { sanitizePublicRecords } from './publication.mjs'; +import { + sanitizePublicRecords, + validateOfficialPublicRun, + validatePublicRecords, +} from './publication.mjs'; import { scoreRecords } from './scoring.mjs'; const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..'); async function main(argv = process.argv.slice(2)) { const options = parseArgs(argv); - const [tasks, records, policy] = await Promise.all([ - readJsonLines(resolve(options.tasks)), + const taskPath = resolve(options.tasks); + const [taskBytes, tasks, records, policy] = await Promise.all([ + readFile(taskPath), + readJsonLines(taskPath), readJsonLines(resolve(options.runs)), readFile(resolve(options.policy), 'utf8').then(JSON.parse), ]); - const publicRecords = sanitizePublicRecords(records, policy); + const publicRecords = sanitizePublicRecords(records, policy, tasks); + validatePublicRecords(publicRecords, policy); + validateOfficialPublicRun(publicRecords, { + taskSetSha256: createHash('sha256').update(taskBytes).digest('hex'), + }); const summary = scoreRecords(tasks, publicRecords); const outputRuns = resolve(options.outputRuns); const outputSummary = resolve(options.outputSummary); + if (outputRuns === outputSummary) { + throw new Error('--output-runs and --output-summary must use different files'); + } await Promise.all([mkdir(dirname(outputRuns), { recursive: true }), mkdir(dirname(outputSummary), { recursive: true })]); await writeJsonLines(outputRuns, publicRecords); await writeFile(outputSummary, JSON.stringify(summary, null, 2) + '\n', { encoding: 'utf8', mode: 0o600 }); diff --git a/benchmarks/discover-call/src/scoring.mjs b/benchmarks/discover-call/src/scoring.mjs index 2adef01..4aae3e8 100644 --- a/benchmarks/discover-call/src/scoring.mjs +++ b/benchmarks/discover-call/src/scoring.mjs @@ -10,12 +10,16 @@ export function scoreRecords(tasks, records) { } const scored = records.map((record) => scoreRecord(taskById.get(record.task_id), record)); const byModel = new Map(); - for (const result of scored) { + const rawByModel = new Map(); + for (const [index, result] of scored.entries()) { const group = byModel.get(result.model) || []; group.push(result); byModel.set(result.model, group); + const rawGroup = rawByModel.get(result.model) || []; + rawGroup.push(records[index]); + rawByModel.set(result.model, rawGroup); } - for (const [model, results] of byModel) validateCompleteModelRun(model, tasks, results); + for (const [model] of byModel) validateCompleteModelRun(model, tasks, rawByModel.get(model)); return { schema_version: 1, @@ -52,6 +56,10 @@ function validateCompleteModelRun(model, tasks, records) { // Never aggregate records collected under different benchmark conditions. // Undefined is a valid shared value for old/synthetic fixtures, but mixing // it with runner metadata is rejected. + for (const field of ['schema_version', 'benchmark_version']) { + const values = new Set(records.map((record) => JSON.stringify(record[field]))); + if (values.size > 1) throw new Error(`Model ${model} mixes different ${field} values`); + } const comparableMetadata = [ 'lane', 'model_revision', @@ -62,6 +70,7 @@ function validateCompleteModelRun(model, tasks, records) { 'api_revision', 'catalog_revision', 'catalog_observation_sha256', + 'runtime', 'discovery_limit', 'execute', ]; @@ -83,6 +92,7 @@ export function scoreRecord(task, record) { const inspected = array(record.inspection?.returned_tool_ids); const required = array(record.inspection?.required_parameters); const parameters = plainObject(record.parameters) ? record.parameters : {}; + const parameterization = parameterizationMetrics(record.parameterization); const completed = record.status === 'completed'; const selectionGrounded = typeof record.discovery?.selection_grounded === 'boolean' @@ -97,8 +107,11 @@ export function scoreRecord(task, record) { ? inspected.includes(selected) : false; const requiredParameterAccuracy = - required.length === 0 ? 1 : mean(required.map((name) => hasValue(parameters[name]))); - const constraintAccuracy = mean(task.constraints.map((constraint) => constraintSatisfied(parameters, constraint))); + parameterization?.required_parameter_accuracy ?? + (required.length === 0 ? 1 : mean(required.map((name) => hasValue(parameters[name])))); + const constraintAccuracy = + parameterization?.constraint_accuracy ?? + mean(task.constraints.map((constraint) => constraintSatisfied(parameters, constraint))); const executed = record.call?.attempted === true; const callSuccess = executed ? record.call?.success === true : null; const resultNonempty = @@ -187,13 +200,13 @@ function constraintSatisfied(parameters, constraint) { for (const alias of constraint.aliases) { if (!hasValue(parameters[alias])) continue; const actual = parameters[alias]; - const expectedValues = [constraint.value, ...array(constraint.alias_values?.[alias])]; - for (const expected of expectedValues) { - if (constraint.match === 'contains') { - if (normalize(actual, constraint).includes(normalize(expected, constraint))) return 1; - } else if (normalize(actual, constraint) === normalize(expected, constraint)) { - return 1; - } + if (constraint.match === 'contains') { + if (normalize(actual, constraint).includes(normalize(constraint.value, constraint))) return 1; + } else if (normalize(actual, constraint) === normalize(constraint.value, constraint)) { + return 1; + } + for (const expected of array(constraint.alias_values?.[alias])) { + if (normalize(actual, constraint) === normalize(expected, constraint)) return 1; } } for (const alias of array(constraint.composite_aliases)) { @@ -285,3 +298,19 @@ function plainObject(value) { function booleanOrNull(value) { return typeof value === 'boolean' ? value : null; } + +function parameterizationMetrics(value) { + if (value === undefined) return null; + if ( + !plainObject(value) || + !metric(value.required_parameter_accuracy) || + !metric(value.constraint_accuracy) + ) { + throw new Error('Run record parameterization metrics must be between 0 and 1'); + } + return value; +} + +function metric(value) { + return typeof value === 'number' && Number.isFinite(value) && value >= 0 && value <= 1; +} diff --git a/benchmarks/discover-call/src/validate.mjs b/benchmarks/discover-call/src/validate.mjs index c03fa62..0221048 100644 --- a/benchmarks/discover-call/src/validate.mjs +++ b/benchmarks/discover-call/src/validate.mjs @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { readFile, readdir } from 'node:fs/promises'; import { resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -5,7 +6,11 @@ import { isDeepStrictEqual } from 'node:util'; import { validateTask } from './harness.mjs'; import { readJsonLines } from './io.mjs'; -import { validatePolicy, validatePublicRecords } from './publication.mjs'; +import { + validateOfficialPublicRun, + validatePolicy, + validatePublicRecords, +} from './publication.mjs'; import { scoreRecords } from './scoring.mjs'; const root = resolve(fileURLToPath(new URL('..', import.meta.url))); @@ -41,9 +46,13 @@ const resultFiles = (await readdir(resolve(root, 'results'))) for (const file of resultFiles) { const version = file.match(/-v(\d+)\.runs\.jsonl$/)?.[1]; if (!version) throw new Error(`${file}: result filename must identify its task-set version`); - const tasks = await readJsonLines(resolve(root, `tasks/v${version}.jsonl`)); + const taskPath = resolve(root, `tasks/v${version}.jsonl`); + const taskBytes = await readFile(taskPath); + const taskSetSha256 = createHash('sha256').update(taskBytes).digest('hex'); + const tasks = await readJsonLines(taskPath); const records = await readJsonLines(resolve(root, 'results', file)); validatePublicRecords(records, policy); + validateOfficialPublicRun(records, { taskSetSha256 }); const generatedSummary = scoreRecords(tasks, records); const summaryFile = file.replace(/\.runs\.jsonl$/, '.summary.json'); const publishedSummary = JSON.parse(await readFile(resolve(root, 'results', summaryFile), 'utf8')); diff --git a/benchmarks/discover-call/test/adapter.test.mjs b/benchmarks/discover-call/test/adapter.test.mjs index fc4b10d..fee0537 100644 --- a/benchmarks/discover-call/test/adapter.test.mjs +++ b/benchmarks/discover-call/test/adapter.test.mjs @@ -11,6 +11,7 @@ import { buildCodexInvocation, CODEX_REASONING_EFFORT, parseCodexEvents, + runCodex, } from '../adapters/codex-cli.mjs'; const adapterPath = resolve(fileURLToPath(new URL('../adapters/first-result.mjs', import.meta.url))); @@ -285,6 +286,23 @@ test('Codex adapter extracts the final structured message and rejects tool use', ); }); +test('Codex adapter force-kills a child that ignores the output-limit signal', async () => { + await assert.rejects( + runCodex( + { + command: process.execPath, + args: [ + '-e', + "process.on('SIGTERM', () => {}); process.stdout.write('x'.repeat(100)); setInterval(() => {}, 1000)", + ], + stdin: '', + }, + { outputLimit: 10, forceKillAfterMs: 20 }, + ), + (error) => error.adapterCode === 'output_limit', + ); +}); + function processExists(pid) { try { process.kill(pid, 0); diff --git a/benchmarks/discover-call/test/publication.test.mjs b/benchmarks/discover-call/test/publication.test.mjs index 28b4f73..c841380 100644 --- a/benchmarks/discover-call/test/publication.test.mjs +++ b/benchmarks/discover-call/test/publication.test.mjs @@ -3,17 +3,26 @@ import test from 'node:test'; import { sanitizePublicRecords, + validateOfficialPublicRun, validatePolicy, validatePublicRecords, } from '../src/publication.mjs'; +const taskSetSha256 = 'a'.repeat(64); +const task = { + id: 'weather', + prompt: 'Weather in Shanghai', + constraints: [{ id: 'city', aliases: ['city'], value: 'Shanghai' }], +}; +const tasks = [task]; const policy = { schema_version: 1, policy_id: 'test-public-v1', catalog_visibility_default: 'private', - publish_parameters: true, + publish_parameters: false, unapproved_selected_tool_handling: 'hash', legacy_lane_rewrites: { 'pinned-model': 'configured-model' }, + approved_api_base_urls: ['https://qveris.ai/api/v1'], approved_selected_tool_ids: ['weather.tool'], forbidden_fields: [ 'execution_id', @@ -29,41 +38,38 @@ const policy = { 'authorization', 'client_secret', 'password', + 'parameters', + 'required_parameters', ], }; -test('publishes grounded attestations and hashes instead of operational identifiers or catalog rows', () => { +test('publishes score attestations and hashes instead of raw parameters or catalog rows', () => { const [record] = sanitizePublicRecords( [ - { - schema_version: 1, - benchmark_version: 'v4', - run_id: 'run-1', - model: 'model-a', + rawRecord({ metadata: { lane: 'configured-model', api_revision: '2026-07-22.1', + task_set_sha256: taskSetSha256, + api_base_url: 'https://qveris.ai/api/v1', + discovery_limit: 10, + execute: true, internal_note: 'must-not-publish', }, - task_id: 'weather', - trial: 1, - status: 'completed', discovery: { search_id: 'search-1', result_tool_ids: ['other.tool', 'weather.tool'], }, - selection: { tool_id: 'weather.tool' }, - inspection: { returned_tool_ids: ['weather.tool'], required_parameters: ['city'] }, - parameters: { city: 'Shanghai' }, call: { attempted: true, success: true, execution_id: 'execution-1', result_nonempty: true, }, - }, + }), ], policy, + tasks, ); assert.equal(record.discovery.result_count, 2); @@ -71,6 +77,11 @@ test('publishes grounded attestations and hashes instead of operational identifi assert.equal(record.discovery.selection_grounded, true); assert.equal(record.inspection.selection_grounded, true); assert.equal(record.call.result_nonempty, true); + assert.deepEqual(record.parameterization, { + required_parameter_accuracy: 1, + constraint_accuracy: 1, + }); + assert.equal(JSON.stringify(record).includes('Shanghai'), false); assert.equal(record.metadata.publication_policy, policy.policy_id); assert.equal(record.metadata.model_revision, 'unreported'); assert.equal(record.metadata.catalog_revision, 'unreported'); @@ -85,37 +96,70 @@ test('publishes grounded attestations and hashes instead of operational identifi test('hashes an unapproved selected tool instead of expanding the public catalog', () => { const [record] = sanitizePublicRecords( [ - { - selection: { tool_id: 'private.tool' }, + rawRecord({ discovery: { result_tool_ids: ['private.tool'] }, - }, + selection: { tool_id: 'private.tool' }, + inspection: { returned_tool_ids: ['private.tool'], required_parameters: [] }, + parameters: {}, + }), ], policy, + tasks, ); assert.equal(record.selection.tool_id, null); assert.equal(record.selection.tool_id_sha256.length, 64); assert.equal(record.discovery.selection_grounded, true); }); -test('projects safe error fields and rejects credential-shaped parameter keys', () => { +test('recomputes parameterization scores instead of trusting raw attestations', () => { + const [record] = sanitizePublicRecords( + [ + rawRecord({ + parameters: { city: 'Wrong city' }, + parameterization: { + required_parameter_accuracy: 1, + constraint_accuracy: 1, + }, + }), + ], + policy, + tasks, + ); + assert.deepEqual(record.parameterization, { + required_parameter_accuracy: 1, + constraint_accuracy: 0, + }); +}); + +test('projects safe errors and rejects unsupported public fields at every boundary', () => { const [record] = sanitizePublicRecords( [ - { + rawRecord({ + status: 'failed', selection: { tool_id: null }, + inspection: { returned_tool_ids: [], required_parameters: [] }, + parameters: { query: 'private customer prompt' }, + call: { attempted: false, success: null }, error: { stage: 'adapter', reason_code: 'tool_use_rejected', message: 'raw provider body with a secret', provider_response: 'private', }, - }, + }), ], policy, + tasks, ); assert.deepEqual(record.error, { stage: 'adapter', reason_code: 'tool_use_rejected', }); + assert.equal(JSON.stringify(record).includes('private customer prompt'), false); + assert.throws( + () => validatePublicRecords([{ ...record, provider_response: 'private' }], policy), + /unsupported field at \$\.provider_response/, + ); assert.throws( () => validatePublicRecords( @@ -127,7 +171,20 @@ test('projects safe error fields and rejects credential-shaped parameter keys', ], policy, ), - /unsupported metadata/, + /unsupported field at \$\.metadata\.internal_note/, + ); + assert.throws( + () => + validatePublicRecords( + [ + { + ...record, + call: { ...record.call, provider_response: 'private' }, + }, + ], + policy, + ), + /unsupported field at \$\.call\.provider_response/, ); assert.throws( () => @@ -140,25 +197,28 @@ test('projects safe error fields and rejects credential-shaped parameter keys', ], policy, ), - /unsupported fields/, + /unsupported field at \$\.error\.message/, ); - assert.throws( () => - sanitizePublicRecords( + validatePublicRecords( [ { - selection: { tool_id: null }, - parameters: { apiKey: 'must-not-publish' }, + ...record, + call: { ...record.call, success: 'raw provider value' }, }, ], policy, ), - /Forbidden public artifact field/, + /call fields have invalid types/, ); }); -test('publication policy cannot omit a required protected field', () => { +test('publication policy cannot expose parameters or omit a required protected field', () => { + assert.throws( + () => validatePolicy({ ...policy, publish_parameters: true }), + /must not publish raw parameter values/, + ); assert.throws( () => validatePolicy({ @@ -167,4 +227,78 @@ test('publication policy cannot omit a required protected field', () => { }), /must forbid protected fields: api_key/, ); + assert.throws( + () => + sanitizePublicRecords( + [ + rawRecord({ + metadata: { + ...rawRecord().metadata, + api_base_url: 'https://internal.example/api/v1', + }, + }), + ], + policy, + tasks, + ), + /unapproved public API base URL/, + ); +}); + +test('official public runs require the exact task digest, execution, and three trials', () => { + const records = sanitizePublicRecords( + [1, 2, 3].map((trial) => rawRecord({ run_id: `run-${trial}`, trial })), + policy, + tasks, + ); + assert.doesNotThrow(() => validateOfficialPublicRun(records, { taskSetSha256 })); + assert.throws( + () => validateOfficialPublicRun(records, { taskSetSha256: 'b'.repeat(64) }), + /task-set digest does not match/, + ); + assert.throws( + () => validateOfficialPublicRun(records.slice(0, 2), { taskSetSha256 }), + /at least 3 trials/, + ); + assert.throws( + () => + validateOfficialPublicRun( + [ + { + ...records[0], + metadata: { ...records[0].metadata, execute: false }, + }, + ...records.slice(1), + ], + { taskSetSha256 }, + ), + /must execute every trial/, + ); }); + +function rawRecord(overrides = {}) { + return { + schema_version: 1, + benchmark_version: 'v1', + run_id: 'run-1', + model: 'model-a', + metadata: { + lane: 'configured-model', + task_set_sha256: taskSetSha256, + api_base_url: 'https://qveris.ai/api/v1', + discovery_limit: 10, + execute: true, + }, + task_id: task.id, + trial: 1, + status: 'completed', + started_at: '2026-07-23T00:00:00.000Z', + discovery: { result_tool_ids: ['weather.tool'] }, + selection: { tool_id: 'weather.tool' }, + inspection: { returned_tool_ids: ['weather.tool'], required_parameters: ['city'] }, + parameters: { city: 'Shanghai' }, + call: { attempted: true, success: true, result_nonempty: true }, + finished_at: '2026-07-23T00:00:01.000Z', + ...overrides, + }; +} diff --git a/benchmarks/discover-call/test/scoring.test.mjs b/benchmarks/discover-call/test/scoring.test.mjs index 204727a..d47ceac 100644 --- a/benchmarks/discover-call/test/scoring.test.mjs +++ b/benchmarks/discover-call/test/scoring.test.mjs @@ -213,6 +213,7 @@ test('scores aliases with task-versioned accepted values', () => { id: 'symbol', aliases: ['symbol', 'id'], value: 'BTC', + match: 'contains', alias_values: { id: ['1'] }, }, ], @@ -234,6 +235,38 @@ test('scores aliases with task-versioned accepted values', () => { assert.equal(result.inspection_grounded, true); assert.equal(result.constraint_accuracy, 1); assert.equal(result.workflow_success, true); + assert.equal( + scoreRecord(btcTask, { + ...resultRecord({ taskId: btcTask.id }), + parameters: { id: 10 }, + }).constraint_accuracy, + 0, + ); +}); + +test('scores sanitized parameterization attestations without publishing parameter values', () => { + const result = scoreRecord(task, { + ...resultRecord({ taskId: task.id }), + parameterization: { + required_parameter_accuracy: 1, + constraint_accuracy: 1, + }, + }); + + assert.equal(result.required_parameter_accuracy, 1); + assert.equal(result.constraint_accuracy, 1); + assert.equal(result.workflow_success, true); + assert.throws( + () => + scoreRecord(task, { + ...resultRecord({ taskId: task.id }), + parameterization: { + required_parameter_accuracy: 2, + constraint_accuracy: 1, + }, + }), + /parameterization metrics/, + ); }); test('rejects run records for unknown tasks', () => { @@ -243,6 +276,20 @@ test('rejects run records for unknown tasks', () => { ); }); +function resultRecord({ taskId }) { + return { + run_id: `run-${taskId}`, + task_id: taskId, + model: 'model-a', + trial: 1, + status: 'completed', + discovery: { selection_grounded: true }, + selection: { tool_id: 'tool' }, + inspection: { selection_grounded: true, required_parameters: [] }, + call: { attempted: true, success: true, result_nonempty: true }, + }; +} + test('rejects missing tasks and duplicate trial numbers', () => { const completeRecord = { run_id: 'run-1', @@ -290,4 +337,36 @@ test('rejects duplicate run ids and mixed benchmark conditions', () => { ), /adapter_revision/, ); + assert.throws( + () => + scoreRecords( + [task], + [ + { + ...record, + schema_version: 1, + metadata: { ...record.metadata, runtime: { node: 'v22', platform: 'linux', arch: 'x64' } }, + }, + { + ...record, + run_id: 'run-2', + trial: 2, + schema_version: 1, + metadata: { ...record.metadata, runtime: { node: 'v24', platform: 'linux', arch: 'x64' } }, + }, + ], + ), + /runtime/, + ); + assert.throws( + () => + scoreRecords( + [task], + [ + { ...record, schema_version: 1 }, + { ...record, run_id: 'run-2', trial: 2, schema_version: 2 }, + ], + ), + /schema_version/, + ); }); diff --git a/docs/en-US/discover-call-benchmark.md b/docs/en-US/discover-call-benchmark.md index 14785f1..d801139 100644 --- a/docs/en-US/discover-call-benchmark.md +++ b/docs/en-US/discover-call-benchmark.md @@ -53,9 +53,11 @@ revision, catalog revision when reported, catalog-observation digest, endpoint, and discovery limit. Only sanitized JSONL is committed. Public artifacts omit execution, search, and -connection identifiers and the ordered discovery catalog. Approved selected -tool IDs may remain visible; other selected tools are represented only by a -digest. See the +connection identifiers, raw parameter values, and the ordered discovery +catalog. Approved selected tool IDs may remain visible; other selected tools +are represented only by a digest. Parameter quality is published only as +required-parameter and constraint-accuracy attestations; inspected parameter +names are omitted so hashed tools do not leak schema details. See the [publication policy](../../benchmarks/discover-call/PUBLICATION_POLICY.md). ## Published results diff --git a/docs/zh-CN/discover-call-benchmark.md b/docs/zh-CN/discover-call-benchmark.md index 4484ac5..304403e 100644 --- a/docs/zh-CN/discover-call-benchmark.md +++ b/docs/zh-CN/discover-call-benchmark.md @@ -38,8 +38,10 @@ reference 与模型的严格工作流成功率之差称为 **strict benchmark ga `unreported`）、adapter 与 toolkit revision、任务集摘要、runtime、API revision、服务端有返回时 的 catalog revision、catalog observation 摘要、endpoint 和 discovery limit。 -仓库只提交脱敏后的 JSONL。公开 artifact 不包含 execution、search、connection 标识，也不包含完整 -有序 discovery 目录。已批准的 selected tool ID 可以保留；其他 selected tool 只保留摘要。详见 +仓库只提交脱敏后的 JSONL。公开 artifact 不包含 execution、search、connection 标识、原始参数值， +也不包含完整有序 discovery 目录。已批准的 selected tool ID 可以保留；其他 selected tool 只保留 +摘要。参数质量仅以必填参数完整率和任务约束准确率证明的形式公开；inspect 返回的参数名也会移除， +避免已哈希工具泄露 schema 细节。详见 [公开策略](../../benchmarks/discover-call/PUBLICATION_POLICY.md)。 ## 已发布结果 From 137417c6458f9801ea6ed32618fc8ccdd9265686 Mon Sep 17 00:00:00 2001 From: Linfang Wang Date: Thu, 23 Jul 2026 21:45:32 +0800 Subject: [PATCH 12/19] fix(benchmark): gate execution and verify provenance --- benchmarks/discover-call/README.md | 13 ++- .../discover-call/adapters/codex-cli.mjs | 21 ++-- .../discover-call/adapters/reference.mjs | 20 ++-- benchmarks/discover-call/src/api.mjs | 95 ++++++++------- benchmarks/discover-call/src/harness.mjs | 39 ++++--- benchmarks/discover-call/src/publication.mjs | 108 ++++++++---------- benchmarks/discover-call/src/publish.mjs | 80 +++++++++---- benchmarks/discover-call/src/run.mjs | 18 +-- .../discover-call/test/adapter.test.mjs | 59 ++++++---- benchmarks/discover-call/test/api.test.mjs | 58 +++++++++- .../discover-call/test/harness.test.mjs | 66 +++++++++-- .../discover-call/test/publication.test.mjs | 74 ++++++++++-- .../discover-call/test/publish.test.mjs | 53 +++++++++ docs/en-US/discover-call-benchmark.md | 3 + docs/zh-CN/discover-call-benchmark.md | 2 + 15 files changed, 500 insertions(+), 209 deletions(-) create mode 100644 benchmarks/discover-call/test/publish.test.mjs diff --git a/benchmarks/discover-call/README.md b/benchmarks/discover-call/README.md index 6eb8304..58bfd2e 100644 --- a/benchmarks/discover-call/README.md +++ b/benchmarks/discover-call/README.md @@ -27,6 +27,11 @@ Each task runs the same sequence: 4. Ask the adapter to construct parameters from the inspected schema. 5. Optionally execute the billed `call`. +The runner treats both grounding checks as execution gates: it stops before +`inspect` when selection is outside the discovery results, and stops before +parameterization or `call` when inspection does not return the exact selected +tool. + The scorer reports these metrics separately: - **selection grounded**: the selected tool was returned by `discover`; @@ -176,9 +181,11 @@ npm run validate The harness invokes the adapter once per decision stage. It writes one JSON object to stdin and expects exactly one JSON object on stdout. Each request -contains canonical `messages` and a `response_schema`; model adapters must send -both unchanged so model comparisons use the same prompt and output contract. -Ground-truth scoring constraints are deliberately not exposed to adapters. +contains the immutable `task_set_sha256`, canonical `messages`, and a +`response_schema`; model adapters must send the messages and schema unchanged +so model comparisons use the same prompt and output contract. The bundled +reference adapter rejects a task-file digest mismatch. Ground-truth scoring +constraints are deliberately not exposed to model adapters. Selection input uses `stage: "select"`; `input` contains the user prompt and the complete discovery response. Return: diff --git a/benchmarks/discover-call/adapters/codex-cli.mjs b/benchmarks/discover-call/adapters/codex-cli.mjs index 046f437..4c3aede 100644 --- a/benchmarks/discover-call/adapters/codex-cli.mjs +++ b/benchmarks/discover-call/adapters/codex-cli.mjs @@ -68,6 +68,7 @@ export function parseCodexEvents(stdout) { if (lines.length === 0) throw adapterFailure('Codex CLI returned no events', 'invalid_events'); let finalMessage; + let turnCompleted = false; for (const line of lines) { let event; try { @@ -79,6 +80,7 @@ export function parseCodexEvents(stdout) { if (event.type === 'error' || event.type === 'turn.failed') { throw adapterFailure('Codex CLI reported an unsuccessful result', 'model_failed'); } + if (event.type === 'turn.completed') turnCompleted = true; if ((event.type === 'item.started' || event.type === 'item.completed') && isObject(event.item)) { if (!ALLOWED_ITEM_TYPES.has(event.item.type)) { @@ -96,6 +98,9 @@ export function parseCodexEvents(stdout) { if (typeof finalMessage !== 'string') { throw adapterFailure('Codex CLI returned no structured output', 'invalid_output'); } + if (!turnCompleted) { + throw adapterFailure('Codex CLI returned an incomplete event stream', 'invalid_events'); + } try { const result = JSON.parse(finalMessage); if (isObject(result)) return result; @@ -118,10 +123,7 @@ export async function invokeCodex(payload) { } } -export function runCodex( - invocation, - { outputLimit = 1_000_000, forceKillAfterMs = 1_000 } = {}, -) { +export function runCodex(invocation, { outputLimit = 1_000_000, forceKillAfterMs = 1_000 } = {}) { if (!Number.isInteger(outputLimit) || outputLimit < 1) { throw new Error('outputLimit must be a positive integer'); } @@ -162,7 +164,9 @@ export function runCodex( const forwardSignal = (signal) => { if (forwardedSignal) return; forwardedSignal = signal; - terminate(adapterFailure('Codex CLI interrupted', 'interrupted'), signal); + const error = adapterFailure('Codex CLI interrupted', 'interrupted'); + error.adapterSignal = signal; + terminate(error, signal); }; const forwardSigint = () => forwardSignal('SIGINT'); const forwardSigterm = () => forwardSignal('SIGTERM'); @@ -186,8 +190,7 @@ export function runCodex( cleanup(); if (settled) return; if (forwardedSignal) { - process.kill(process.pid, forwardedSignal); - return; + return rejectOnce(terminationError); } if (terminationError) return rejectOnce(terminationError); settled = true; @@ -217,6 +220,10 @@ async function main() { if (process.argv[1] === fileURLToPath(import.meta.url)) { main().catch((error) => { + if (error?.adapterSignal) { + process.kill(process.pid, error.adapterSignal); + return; + } const code = typeof error?.adapterCode === 'string' ? error.adapterCode : 'invalid_output'; process.stderr.write(`QVERIS_BENCHMARK_ADAPTER_ERROR=${code}\n`); process.exitCode = 1; diff --git a/benchmarks/discover-call/adapters/reference.mjs b/benchmarks/discover-call/adapters/reference.mjs index 3ac5e4b..bdc2a0a 100644 --- a/benchmarks/discover-call/adapters/reference.mjs +++ b/benchmarks/discover-call/adapters/reference.mjs @@ -1,15 +1,17 @@ #!/usr/bin/env node +import { createHash } from 'node:crypto'; import { readFile } from 'node:fs/promises'; try { const tasksPath = parseTasksPath(process.argv.slice(2)); - const tasks = await readTasks(tasksPath); + const { tasks, sha256 } = await readTasks(tasksPath); const taskById = new Map(tasks.map((task) => [task.id, task])); let input = ''; for await (const chunk of process.stdin) input += chunk; const payload = JSON.parse(input); + if (payload.task_set_sha256 !== sha256) throw referenceFailure('task_set_mismatch'); const task = taskById.get(payload.input?.task_id); const reference = task?.reference ?? task?.oracle; if (!reference || !Array.isArray(reference.candidates)) throw referenceFailure('missing_reference'); @@ -36,12 +38,16 @@ try { } async function readTasks(path) { - const text = await readFile(path, 'utf8'); - return text - .split(/\r?\n/) - .map((line) => line.trim()) - .filter((line) => line && !line.startsWith('#')) - .map((line) => JSON.parse(line)); + const bytes = await readFile(path); + return { + sha256: createHash('sha256').update(bytes).digest('hex'), + tasks: bytes + .toString('utf8') + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith('#')) + .map((line) => JSON.parse(line)), + }; } function parseTasksPath(argv) { diff --git a/benchmarks/discover-call/src/api.mjs b/benchmarks/discover-call/src/api.mjs index 53bf9f9..5403d10 100644 --- a/benchmarks/discover-call/src/api.mjs +++ b/benchmarks/discover-call/src/api.mjs @@ -17,58 +17,67 @@ export function createApiClient({ if (!Number.isInteger(maxRetries) || maxRetries < 0 || maxRetries > 10) { throw new Error('maxRetries must be an integer from 0 to 10'); } + if (!Number.isInteger(timeoutMs) || timeoutMs < 1) { + throw new Error('timeoutMs must be a positive integer'); + } async function request(path, body) { - let response; for (let attempt = 0; ; attempt++) { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), timeoutMs); + let retryDelay; try { - response = await fetchImpl(`${resolvedBaseUrl}${path}`, { + const response = await fetchImpl(`${resolvedBaseUrl}${path}`, { method: 'POST', headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' }, body: JSON.stringify(body), signal: controller.signal, + redirect: 'error', }); - } catch { - if (attempt >= maxRetries) throw apiError('Network request failed'); - await sleep(retryDelayMs(null, attempt)); - continue; + if (RETRYABLE_STATUS.has(response.status) && attempt < maxRetries) { + await Promise.resolve(response.body?.cancel?.()).catch(() => undefined); + retryDelay = retryDelayMs(response.headers.get('retry-after'), attempt); + } else { + observeRevision(response.headers, 'x-qveris-api-version', observedApiRevisions); + observeRevision(response.headers, 'x-qveris-catalog-version', observedCatalogRevisions); + if (!response.ok) { + await Promise.resolve(response.body?.cancel?.()).catch(() => undefined); + throw apiError(`HTTP ${response.status}`); + } + let payload; + try { + payload = await response.json(); + } catch { + throw apiError(controller.signal.aborted ? 'API response timed out' : 'Invalid API response'); + } + if ( + payload && + typeof payload === 'object' && + 'data' in payload && + ('status' in payload || 'status_code' in payload || 'message' in payload) + ) { + const status = payload.status ?? payload.status_code; + if ( + (typeof status === 'string' && ['failure', 'failed', 'error'].includes(status.toLowerCase())) || + (typeof status === 'number' && status >= 400) + ) { + throw apiError('API returned a failure envelope'); + } + return payload.data; + } + return payload; + } + } catch (error) { + if (error?.benchmarkStage === 'api') throw error; + if (attempt >= maxRetries) { + throw apiError(controller.signal.aborted ? 'API request timed out' : 'Network request failed'); + } + retryDelay = retryDelayMs(null, attempt); } finally { clearTimeout(timeout); } - if (!RETRYABLE_STATUS.has(response.status) || attempt >= maxRetries) break; - await Promise.resolve(response.body?.cancel?.()).catch(() => undefined); - await sleep(retryDelayMs(response.headers.get('retry-after'), attempt)); - } - observeRevision(response.headers, 'x-qveris-api-version', observedApiRevisions); - observeRevision(response.headers, 'x-qveris-catalog-version', observedCatalogRevisions); - if (!response.ok) { - await Promise.resolve(response.body?.cancel?.()).catch(() => undefined); - throw apiError(`HTTP ${response.status}`); + await sleep(retryDelay); } - let payload; - try { - payload = await response.json(); - } catch { - throw apiError('Invalid API response'); - } - if ( - payload && - typeof payload === 'object' && - 'data' in payload && - ('status' in payload || 'status_code' in payload || 'message' in payload) - ) { - const status = payload.status ?? payload.status_code; - if ( - (typeof status === 'string' && ['failure', 'failed', 'error'].includes(status.toLowerCase())) || - (typeof status === 'number' && status >= 400) - ) { - throw apiError('API returned a failure envelope'); - } - return payload.data; - } - return payload; } return { @@ -103,9 +112,15 @@ function singleRevision(revisions) { } function retryDelayMs(retryAfter, attempt) { - const seconds = Number(retryAfter); - if (retryAfter !== null && Number.isFinite(seconds) && seconds >= 0) { - return Math.min(seconds * 1000, 60_000); + if (retryAfter !== null) { + const value = String(retryAfter).trim(); + if (/^\d+(?:\.\d+)?$/.test(value)) { + return Math.min(Number(value) * 1000, 60_000); + } + if (/[a-z]/i.test(value)) { + const date = Date.parse(value); + if (Number.isFinite(date)) return Math.min(Math.max(0, date - Date.now()), 60_000); + } } return Math.min(500 * 2 ** attempt, 60_000); } diff --git a/benchmarks/discover-call/src/harness.mjs b/benchmarks/discover-call/src/harness.mjs index 2324e68..5750b5f 100644 --- a/benchmarks/discover-call/src/harness.mjs +++ b/benchmarks/discover-call/src/harness.mjs @@ -72,11 +72,7 @@ export function validateTask(task) { throw new Error(`Task ${task.id}: an empty reference needs unavailable_reason`); } for (const candidate of reference.candidates) { - if ( - typeof candidate?.tool_id !== 'string' || - !candidate.tool_id || - !isPlainObject(candidate.parameters) - ) { + if (typeof candidate?.tool_id !== 'string' || !candidate.tool_id || !isPlainObject(candidate.parameters)) { throw new Error(`Task ${task.id}: every reference candidate needs tool_id and parameters`); } } @@ -144,16 +140,29 @@ async function runTrial({ task, model, trial, execute, limit, api, invokeAdapter record.discovery.result_tool_ids = results.map(toolId).filter(Boolean); const selection = await invokeAdapter( - adapterPayload({ stage: 'select', model, task, discovery: discovered, trial }), + adapterPayload({ + stage: 'select', + model, + task, + discovery: discovered, + trial, + taskSetSha256: metadata.task_set_sha256, + }), ); const selectedToolId = toolId(selection); if (!selectedToolId) throw stageError('select', 'Adapter did not return a non-empty tool_id'); record.selection.tool_id = selectedToolId; + if (!record.discovery.result_tool_ids.includes(selectedToolId)) { + throw stageError('select', 'Adapter selected a tool outside the discovery results', 'ungrounded_tool_id'); + } const inspected = await api.inspect({ toolIds: [selectedToolId], discoveryId: discovered?.search_id }); const inspectedResults = array(inspected?.results); record.inspection.returned_tool_ids = inspectedResults.map(toolId).filter(Boolean); - const selectedTool = inspectedResults.find((item) => toolId(item) === selectedToolId) || inspectedResults[0]; + const selectedTool = inspectedResults.find((item) => toolId(item) === selectedToolId); + if (!selectedTool) { + throw stageError('inspect', 'Inspect did not return the selected tool', 'selected_tool_not_returned'); + } record.inspection.required_parameters = requiredParameterNames(selectedTool); const parameterized = await invokeAdapter( @@ -164,6 +173,7 @@ async function runTrial({ task, model, trial, execute, limit, api, invokeAdapter selectedTool, discoveryId: discovered?.search_id ?? null, trial, + taskSetSha256: metadata.task_set_sha256, }), ); if (!isPlainObject(parameterized?.parameters)) { @@ -223,7 +233,7 @@ export function parameterResponseSchema(tool) { }; } -function adapterPayload({ stage, model, task, discovery, selectedTool, discoveryId, trial }) { +function adapterPayload({ stage, model, task, discovery, selectedTool, discoveryId, trial, taskSetSha256 }) { const input = stage === 'select' ? { task_id: task.id, prompt: task.prompt, discovery } @@ -246,6 +256,7 @@ function adapterPayload({ stage, model, task, discovery, selectedTool, discovery stage, model, trial, + task_set_sha256: taskSetSha256 ?? 'unreported', input, messages: [ { role: 'system', content: instruction }, @@ -278,12 +289,8 @@ function strictParameterSchema(parameter, forceRequired = parameter?.required == ); } else if (type === 'object') { const nestedProperties = {}; - for (const [name, definition] of Object.entries( - isPlainObject(parameter?.properties) ? parameter.properties : {}, - )) { - nestedProperties[name] = strictParameterSchema( - isPlainObject(definition) ? definition : { type: 'string' }, - ); + for (const [name, definition] of Object.entries(isPlainObject(parameter?.properties) ? parameter.properties : {})) { + nestedProperties[name] = strictParameterSchema(isPlainObject(definition) ? definition : { type: 'string' }); } schema.additionalProperties = false; schema.required = Object.keys(nestedProperties); @@ -346,10 +353,10 @@ function isPlainObject(value) { return value !== null && typeof value === 'object' && !Array.isArray(value); } -function stageError(stage, message) { +function stageError(stage, message, reason) { const error = new Error(message); error.benchmarkStage = stage; - error.benchmarkReason = stage === 'select' ? 'missing_tool_id' : 'invalid_parameters'; + error.benchmarkReason = reason ?? (stage === 'select' ? 'missing_tool_id' : 'invalid_parameters'); return error; } diff --git a/benchmarks/discover-call/src/publication.mjs b/benchmarks/discover-call/src/publication.mjs index c120589..8570ead 100644 --- a/benchmarks/discover-call/src/publication.mjs +++ b/benchmarks/discover-call/src/publication.mjs @@ -41,17 +41,10 @@ const PUBLIC_METADATA_FIELDS = new Set([ ]); const PUBLIC_RUNTIME_FIELDS = new Set(['node', 'platform', 'arch']); -const PUBLIC_DISCOVERY_FIELDS = new Set([ - 'result_count', - 'snapshot_sha256', - 'selection_grounded', -]); +const PUBLIC_DISCOVERY_FIELDS = new Set(['result_count', 'snapshot_sha256', 'selection_grounded']); const PUBLIC_SELECTION_FIELDS = new Set(['tool_id', 'tool_id_sha256']); const PUBLIC_INSPECTION_FIELDS = new Set(['selection_grounded']); -const PUBLIC_PARAMETERIZATION_FIELDS = new Set([ - 'required_parameter_accuracy', - 'constraint_accuracy', -]); +const PUBLIC_PARAMETERIZATION_FIELDS = new Set(['required_parameter_accuracy', 'constraint_accuracy']); const PUBLIC_CALL_FIELDS = new Set(['attempted', 'success', 'result_nonempty']); const PUBLIC_ERROR_FIELDS = new Set(['stage', 'reason_code']); @@ -77,26 +70,29 @@ export function sanitizePublicRecords(records, policy, tasks) { validatePolicy(policy); const taskById = taskMap(tasks); const approvedToolIds = new Set(policy.approved_selected_tool_ids); - const catalogObservationSha256 = - commonMetadataValue(records, 'catalog_observation_sha256') ?? - sha256( - JSON.stringify( - records.map((record) => ({ - task_id: record.task_id, - trial: record.trial, - result_tool_ids: array(record.discovery?.result_tool_ids), - })), - ), - ); - return records.map((record) => - sanitizePublicRecord( - record, - taskById.get(record.task_id), - policy, - approvedToolIds, - catalogObservationSha256, + const catalogObservationSha256 = sha256( + JSON.stringify( + records.map((record) => ({ + task_id: record.task_id, + trial: record.trial, + result_tool_ids: array(record.discovery?.result_tool_ids), + })), ), ); + const declaredCatalogDigests = new Set( + records + .map((record) => record.metadata?.catalog_observation_sha256) + .filter((value) => value !== undefined && value !== 'pending'), + ); + if ( + declaredCatalogDigests.size > 1 || + (declaredCatalogDigests.size === 1 && !declaredCatalogDigests.has(catalogObservationSha256)) + ) { + throw new Error('Raw benchmark catalog observation digest does not match its records'); + } + return records.map((record) => + sanitizePublicRecord(record, taskById.get(record.task_id), policy, approvedToolIds, catalogObservationSha256), + ); } function sanitizePublicRecord( @@ -145,9 +141,7 @@ function sanitizePublicRecord( }, selection: { tool_id: selectedToolApproved ? selectedToolId : null, - ...(selectedToolId && !selectedToolApproved - ? { tool_id_sha256: sha256(selectedToolId) } - : {}), + ...(selectedToolId && !selectedToolApproved ? { tool_id_sha256: sha256(selectedToolId) } : {}), }, inspection: { selection_grounded: inspectionGrounded, @@ -192,13 +186,9 @@ export function validatePolicy(policy) { throw new Error('Publication policy needs forbidden_fields'); } const forbidden = forbiddenFieldSet(policy); - const missingForbidden = REQUIRED_FORBIDDEN_FIELDS.filter( - (field) => !forbidden.has(normalizeFieldName(field)), - ); + const missingForbidden = REQUIRED_FORBIDDEN_FIELDS.filter((field) => !forbidden.has(normalizeFieldName(field))); if (missingForbidden.length > 0) { - throw new Error( - `Publication policy must forbid protected fields: ${missingForbidden.join(', ')}`, - ); + throw new Error(`Publication policy must forbid protected fields: ${missingForbidden.join(', ')}`); } if ( policy.legacy_lane_rewrites !== undefined && @@ -236,6 +226,12 @@ export function validatePublicRecords(records, policy) { ) { throw new Error('Public artifact has invalid record identity or lifecycle fields'); } + if ( + Date.parse(record.finished_at) < Date.parse(record.started_at) || + (record.status === 'failed') !== (record.error !== undefined) + ) { + throw new Error('Public artifact has inconsistent lifecycle fields'); + } if (record.metadata?.publication_policy !== policy.policy_id) { throw new Error(`Public artifact is missing publication policy ${policy.policy_id}`); } @@ -244,11 +240,7 @@ export function validatePublicRecords(records, policy) { assertObjectFields(record.discovery, PUBLIC_DISCOVERY_FIELDS, '$.discovery'); assertObjectFields(record.selection, PUBLIC_SELECTION_FIELDS, '$.selection'); assertObjectFields(record.inspection, PUBLIC_INSPECTION_FIELDS, '$.inspection'); - assertObjectFields( - record.parameterization, - PUBLIC_PARAMETERIZATION_FIELDS, - '$.parameterization', - ); + assertObjectFields(record.parameterization, PUBLIC_PARAMETERIZATION_FIELDS, '$.parameterization'); assertObjectFields(record.call, PUBLIC_CALL_FIELDS, '$.call'); if (record.error !== undefined) { assertObjectFields(record.error, PUBLIC_ERROR_FIELDS, '$.error'); @@ -303,11 +295,18 @@ export function validatePublicRecords(records, policy) { if ( typeof record.call?.attempted !== 'boolean' || !booleanOrNull(record.call?.success) || - (record.call?.result_nonempty !== undefined && - typeof record.call.result_nonempty !== 'boolean') + (record.call?.result_nonempty !== undefined && typeof record.call.result_nonempty !== 'boolean') ) { throw new Error('Public artifact call fields have invalid types'); } + if ( + (!record.call.attempted && (record.call.success !== null || record.call.result_nonempty !== undefined)) || + (record.call.result_nonempty === true && record.call.success !== true) || + (record.call.attempted && record.metadata.execute !== true) || + (record.status === 'completed' && record.metadata.execute === true && record.call.attempted !== true) + ) { + throw new Error('Public artifact has inconsistent call lifecycle'); + } if ( record.error !== undefined && (safeCode(record.error.stage) !== record.error.stage || @@ -318,10 +317,7 @@ export function validatePublicRecords(records, policy) { } } -export function validateOfficialPublicRun( - records, - { taskSetSha256, minTrialsPerTask = 3 } = {}, -) { +export function validateOfficialPublicRun(records, { taskSetSha256, minTrialsPerTask = 3 } = {}) { if (!isSha256(taskSetSha256)) throw new Error('Official public run needs the task-set SHA-256'); if (!Number.isInteger(minTrialsPerTask) || minTrialsPerTask < 1) { throw new Error('minTrialsPerTask must be a positive integer'); @@ -340,9 +336,7 @@ export function validateOfficialPublicRun( for (const [key, count] of trials) { if (count < minTrialsPerTask) { const taskId = key.slice(key.indexOf('\0') + 1); - throw new Error( - `Official public artifact needs at least ${minTrialsPerTask} trials for ${taskId}`, - ); + throw new Error(`Official public artifact needs at least ${minTrialsPerTask} trials for ${taskId}`); } } } @@ -373,14 +367,6 @@ function publicMetadata(metadata = {}, policy, catalogObservationSha256) { }; } -function commonMetadataValue(records, field) { - if (records.length === 0) return null; - const values = records.map((record) => record.metadata?.[field]); - if (values.some((value) => !isSha256(value))) return null; - const unique = new Set(values); - return unique.size === 1 ? values[0] : null; -} - function assertNoForbiddenFields(value, forbidden, path = '$') { if (Array.isArray(value)) { value.forEach((item, index) => assertNoForbiddenFields(item, forbidden, `${path}[${index}]`)); @@ -444,11 +430,7 @@ function nonemptyString(value) { } function validTimestamp(value) { - return ( - typeof value === 'string' && - /^\d{4}-\d{2}-\d{2}T/.test(value) && - Number.isFinite(Date.parse(value)) - ); + return typeof value === 'string' && /^\d{4}-\d{2}-\d{2}T/.test(value) && Number.isFinite(Date.parse(value)); } function safePublicApiUrl(value) { diff --git a/benchmarks/discover-call/src/publish.mjs b/benchmarks/discover-call/src/publish.mjs index 7661d91..0b17ccc 100644 --- a/benchmarks/discover-call/src/publish.mjs +++ b/benchmarks/discover-call/src/publish.mjs @@ -1,28 +1,32 @@ #!/usr/bin/env node import { createHash } from 'node:crypto'; -import { mkdir, readFile, writeFile } from 'node:fs/promises'; -import { dirname, resolve } from 'node:path'; +import { mkdir, readFile, realpath, stat, writeFile } from 'node:fs/promises'; +import { basename, dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { readJsonLines, writeJsonLines } from './io.mjs'; -import { - sanitizePublicRecords, - validateOfficialPublicRun, - validatePublicRecords, -} from './publication.mjs'; +import { sanitizePublicRecords, validateOfficialPublicRun, validatePublicRecords } from './publication.mjs'; import { scoreRecords } from './scoring.mjs'; const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..'); -async function main(argv = process.argv.slice(2)) { +export async function main(argv = process.argv.slice(2)) { const options = parseArgs(argv); const taskPath = resolve(options.tasks); + const runsPath = resolve(options.runs); + const policyPath = resolve(options.policy); + const outputRuns = resolve(options.outputRuns); + const outputSummary = resolve(options.outputSummary); + await validatePublicationPaths({ + inputs: [taskPath, runsPath, policyPath], + outputs: [outputRuns, outputSummary], + }); const [taskBytes, tasks, records, policy] = await Promise.all([ readFile(taskPath), readJsonLines(taskPath), - readJsonLines(resolve(options.runs)), - readFile(resolve(options.policy), 'utf8').then(JSON.parse), + readJsonLines(runsPath), + readFile(policyPath, 'utf8').then(JSON.parse), ]); const publicRecords = sanitizePublicRecords(records, policy, tasks); validatePublicRecords(publicRecords, policy); @@ -30,17 +34,51 @@ async function main(argv = process.argv.slice(2)) { taskSetSha256: createHash('sha256').update(taskBytes).digest('hex'), }); const summary = scoreRecords(tasks, publicRecords); - const outputRuns = resolve(options.outputRuns); - const outputSummary = resolve(options.outputSummary); - if (outputRuns === outputSummary) { - throw new Error('--output-runs and --output-summary must use different files'); - } - await Promise.all([mkdir(dirname(outputRuns), { recursive: true }), mkdir(dirname(outputSummary), { recursive: true })]); + await Promise.all([ + mkdir(dirname(outputRuns), { recursive: true }), + mkdir(dirname(outputSummary), { recursive: true }), + ]); await writeJsonLines(outputRuns, publicRecords); await writeFile(outputSummary, JSON.stringify(summary, null, 2) + '\n', { encoding: 'utf8', mode: 0o600 }); process.stdout.write(`Wrote ${publicRecords.length} sanitized records and summary\n`); } +export async function validatePublicationPaths({ inputs, outputs }) { + const inputIdentities = await Promise.all(inputs.map(fileIdentities)); + const outputIdentities = await Promise.all(outputs.map(fileIdentities)); + if (identitiesOverlap(outputIdentities[0], outputIdentities[1])) { + throw new Error('--output-runs and --output-summary must use different files'); + } + for (const output of outputIdentities) { + if (inputIdentities.some((input) => identitiesOverlap(input, output))) { + throw new Error('Publication output files must not overwrite task, run, or policy inputs'); + } + } +} + +async function fileIdentities(path) { + const absolute = resolve(path); + const identities = new Set([`path:${absolute}`]); + try { + const canonical = await realpath(absolute); + const info = await stat(absolute); + identities.add(`path:${canonical}`); + identities.add(`inode:${info.dev}:${info.ino}`); + } catch (error) { + if (error?.code !== 'ENOENT') throw error; + try { + identities.add(`path:${join(await realpath(dirname(absolute)), basename(absolute))}`); + } catch (parentError) { + if (parentError?.code !== 'ENOENT') throw parentError; + } + } + return identities; +} + +function identitiesOverlap(left, right) { + return [...left].some((identity) => right.has(identity)); +} + function parseArgs(argv) { const options = { tasks: resolve(ROOT, 'tasks/v1.jsonl'), @@ -70,7 +108,9 @@ function camelToKebab(value) { return value.replace(/[A-Z]/g, (character) => `-${character.toLowerCase()}`); } -main().catch((error) => { - process.stderr.write(`${error.message}\n`); - process.exitCode = 1; -}); +if (process.argv[1] === fileURLToPath(import.meta.url)) { + main().catch((error) => { + process.stderr.write(`${error.message}\n`); + process.exitCode = 1; + }); +} diff --git a/benchmarks/discover-call/src/run.mjs b/benchmarks/discover-call/src/run.mjs index be31b79..c0cefe9 100644 --- a/benchmarks/discover-call/src/run.mjs +++ b/benchmarks/discover-call/src/run.mjs @@ -61,21 +61,13 @@ export async function main(argv = process.argv.slice(2)) { invokeAdapter, metadata, }); - Object.assign( - metadata, - api.observedRevisions?.() ?? { api_revision: 'unreported', catalog_revision: 'unreported' }, - ); + Object.assign(metadata, api.observedRevisions?.() ?? { api_revision: 'unreported', catalog_revision: 'unreported' }); metadata.catalog_observation_sha256 = catalogObservationSha256(records); await writeJsonLines(output, records); process.stdout.write(`Wrote ${records.length} benchmark records to ${output}\n`); } -export function createProcessAdapter({ - command, - args = [], - timeoutMs = 120_000, - forceKillAfterMs = 1_000, -}) { +export function createProcessAdapter({ command, args = [], timeoutMs = 120_000, forceKillAfterMs = 1_000 }) { if (typeof command !== 'string' || !command) throw new Error('--adapter is required'); if (!Number.isInteger(forceKillAfterMs) || forceKillAfterMs < 1 || forceKillAfterMs > 10_000) { throw new Error('forceKillAfterMs must be an integer from 1 to 10000'); @@ -207,9 +199,7 @@ function integer(value, flag) { function lane(value) { if (!['model', 'oracle', 'reference', 'configured-model', 'pinned-model', 'current-model'].includes(value)) { - throw new Error( - '--lane must be model, oracle, reference, configured-model, pinned-model, or current-model', - ); + throw new Error('--lane must be model, oracle, reference, configured-model, pinned-model, or current-model'); } return value; } @@ -232,7 +222,7 @@ function adapterError(message, reason) { function adapterReasonFromStderr(stderr) { const match = stderr.match( - /QVERIS_BENCHMARK_ADAPTER_ERROR=(tool_use_rejected|model_failed|invalid_events|invalid_output|cli_failed|start_failed|output_limit|missing_reference|missing_reference_candidate|missing_oracle|missing_oracle_candidate|unsupported_stage)/, + /QVERIS_BENCHMARK_ADAPTER_ERROR=(tool_use_rejected|model_failed|invalid_events|invalid_output|cli_failed|start_failed|output_limit|missing_reference|missing_reference_candidate|missing_oracle|missing_oracle_candidate|task_set_mismatch|unsupported_stage)/, ); return match?.[1] ?? null; } diff --git a/benchmarks/discover-call/test/adapter.test.mjs b/benchmarks/discover-call/test/adapter.test.mjs index fee0537..1b9b2b3 100644 --- a/benchmarks/discover-call/test/adapter.test.mjs +++ b/benchmarks/discover-call/test/adapter.test.mjs @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import test from 'node:test'; +import { createHash } from 'node:crypto'; import { mkdtemp, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; @@ -7,12 +8,7 @@ import { fileURLToPath } from 'node:url'; import { createProcessAdapter, main } from '../src/run.mjs'; import { buildClaudeInvocation, parseClaudeEnvelope } from '../adapters/claude-cli.mjs'; -import { - buildCodexInvocation, - CODEX_REASONING_EFFORT, - parseCodexEvents, - runCodex, -} from '../adapters/codex-cli.mjs'; +import { buildCodexInvocation, CODEX_REASONING_EFFORT, parseCodexEvents, runCodex } from '../adapters/codex-cli.mjs'; const adapterPath = resolve(fileURLToPath(new URL('../adapters/first-result.mjs', import.meta.url))); const referenceAdapterPath = resolve(fileURLToPath(new URL('../adapters/reference.mjs', import.meta.url))); @@ -86,7 +82,7 @@ test('process adapter records safe failure reasons without provider stderr', asy test('process adapter classifies timeouts', async () => { const invoke = createProcessAdapter({ command: process.execPath, - args: ['-e', "process.stdin.resume(); setInterval(() => {}, 1000)"], + args: ['-e', 'process.stdin.resume(); setInterval(() => {}, 1000)'], timeoutMs: 25, }); await assert.rejects(invoke({ stage: 'select' }), (error) => { @@ -118,6 +114,9 @@ test('process adapter force-kills a child that ignores SIGTERM', async () => { }); test('reference adapter selects only configured candidates present in discovery', async () => { + const taskSetSha256 = createHash('sha256') + .update(await readFile(v4TasksPath)) + .digest('hex'); const invoke = createProcessAdapter({ command: process.execPath, args: [referenceAdapterPath, v4TasksPath], @@ -126,13 +125,11 @@ test('reference adapter selects only configured candidates present in discovery' assert.deepEqual( await invoke({ stage: 'select', + task_set_sha256: taskSetSha256, input: { task_id: 'weather-london', discovery: { - results: [ - { tool_id: 'unrelated.tool' }, - { tool_id: 'visualcrossing.timeline.retrieve.v1' }, - ], + results: [{ tool_id: 'unrelated.tool' }, { tool_id: 'visualcrossing.timeline.retrieve.v1' }], }, }, }), @@ -141,6 +138,7 @@ test('reference adapter selects only configured candidates present in discovery' assert.deepEqual( await invoke({ stage: 'parameterize', + task_set_sha256: taskSetSha256, input: { task_id: 'weather-london', selected_tool: { tool_id: 'visualcrossing.timeline.retrieve.v1' }, @@ -151,6 +149,7 @@ test('reference adapter selects only configured candidates present in discovery' assert.deepEqual( await invoke({ stage: 'select', + task_set_sha256: taskSetSha256, input: { task_id: 'timezone-tokyo', discovery: { results: [{ tool_id: 'api_sports.timezone.retrieve.v1.e993615d' }] }, @@ -158,6 +157,17 @@ test('reference adapter selects only configured candidates present in discovery' }), { tool_id: null }, ); + await assert.rejects( + invoke({ + stage: 'select', + task_set_sha256: '0'.repeat(64), + input: { + task_id: 'weather-london', + discovery: { results: [{ tool_id: 'visualcrossing.timeline.retrieve.v1' }] }, + }, + }), + (error) => error.benchmarkReason === 'task_set_mismatch', + ); }); test('Claude adapter preserves canonical messages and response schema', () => { @@ -230,14 +240,8 @@ test('Codex adapter preserves canonical messages and response schema', () => { assert.deepEqual(invocation.schema, payload.response_schema); assert.equal(invocation.args[invocation.args.indexOf('--model') + 1], payload.model); assert.equal(invocation.args[invocation.args.indexOf('--output-schema') + 1], '/tmp/response-schema.json'); - assert.equal( - configs.includes(`developer_instructions=${JSON.stringify(payload.messages[0].content)}`), - true, - ); - assert.equal( - configs.includes(`model_reasoning_effort=${JSON.stringify(CODEX_REASONING_EFFORT)}`), - true, - ); + assert.equal(configs.includes(`developer_instructions=${JSON.stringify(payload.messages[0].content)}`), true); + assert.equal(configs.includes(`model_reasoning_effort=${JSON.stringify(CODEX_REASONING_EFFORT)}`), true); assert.equal(invocation.args.includes('--ephemeral'), true); assert.equal(invocation.args.includes('--ignore-user-config'), true); assert.equal(invocation.args.includes('--ignore-rules'), true); @@ -274,15 +278,28 @@ test('Codex adapter extracts the final structured message and rejects tool use', () => parseCodexEvents(JSON.stringify({ type: 'turn.failed', error: { message: 'private' } })), (error) => error.adapterCode === 'model_failed' && /unsuccessful result/.test(error.message), ); + assert.throws( + () => + parseCodexEvents( + [ + JSON.stringify({ + type: 'item.completed', + item: { type: 'agent_message', text: 'not-json' }, + }), + JSON.stringify({ type: 'turn.completed' }), + ].join('\n'), + ), + /no structured output/, + ); assert.throws( () => parseCodexEvents( JSON.stringify({ type: 'item.completed', - item: { type: 'agent_message', text: 'not-json' }, + item: { type: 'agent_message', text: '{"tool_id":"weather.forecast"}' }, }), ), - /no structured output/, + (error) => error.adapterCode === 'invalid_events' && /incomplete event stream/.test(error.message), ); }); diff --git a/benchmarks/discover-call/test/api.test.mjs b/benchmarks/discover-call/test/api.test.mjs index 3728e48..3b60b0b 100644 --- a/benchmarks/discover-call/test/api.test.mjs +++ b/benchmarks/discover-call/test/api.test.mjs @@ -56,6 +56,28 @@ test('retries rate limits and transient unavailability before scoring a failure' assert.deepEqual(delays, [0, 0]); }); +test('honors HTTP-date Retry-After values and disables redirects', async () => { + const delays = []; + const redirects = []; + let attempts = 0; + const client = createApiClient({ + apiKey: 'test-key', + fetchImpl: async (_url, init) => { + redirects.push(init.redirect); + attempts++; + return new Response(JSON.stringify({ results: [] }), { + status: attempts === 1 ? 429 : 200, + headers: attempts === 1 ? { 'retry-after': 'Wed, 21 Oct 2015 07:28:00 GMT' } : {}, + }); + }, + sleep: async (ms) => delays.push(ms), + }); + + assert.deepEqual(await client.discover({ query: 'weather', limit: 5 }), { results: [] }); + assert.deepEqual(delays, [0]); + assert.deepEqual(redirects, ['error', 'error']); +}); + test('retries transient network failures before succeeding', async () => { let attempts = 0; const delays = []; @@ -74,6 +96,36 @@ test('retries transient network failures before succeeding', async () => { assert.deepEqual(delays, [500, 1000]); }); +test('request timeout covers a response body that stalls after headers', async () => { + let signal; + const client = createApiClient({ + apiKey: 'test-key', + timeoutMs: 10, + maxRetries: 0, + fetchImpl: async (_url, init) => { + signal = init.signal; + return { + ok: true, + status: 200, + headers: new Headers(), + json: () => + new Promise((resolve, reject) => { + signal.addEventListener( + 'abort', + () => reject(Object.assign(new Error('aborted'), { name: 'AbortError' })), + { once: true }, + ); + }), + }; + }, + }); + + await assert.rejects(client.discover({ query: 'weather', limit: 5 }), (error) => { + return error.benchmarkStage === 'api' && error.message === 'API response timed out'; + }); + assert.equal(signal.aborted, true); +}); + test('cancels unsuccessful response bodies before reporting API errors', async () => { let cancelled = false; const client = createApiClient({ @@ -82,7 +134,11 @@ test('cancels unsuccessful response bodies before reporting API errors', async ( ok: false, status: 400, headers: new Headers(), - body: { async cancel() { cancelled = true; } }, + body: { + async cancel() { + cancelled = true; + }, + }, }), }); diff --git a/benchmarks/discover-call/test/harness.test.mjs b/benchmarks/discover-call/test/harness.test.mjs index 8f790b4..4682006 100644 --- a/benchmarks/discover-call/test/harness.test.mjs +++ b/benchmarks/discover-call/test/harness.test.mjs @@ -149,6 +149,44 @@ test('records adapter failures without copying their error text', async () => { assert.equal(JSON.stringify(records[0]).includes('secret-token'), false); }); +test('does not inspect or execute a tool outside the discovery results', async () => { + const events = []; + const [record] = await runBenchmark({ + tasks: [task], + model: 'model-a', + trials: 1, + execute: true, + api: { + async discover() { + events.push('discover'); + return { results: [{ tool_id: 'weather.forecast' }] }; + }, + async inspect() { + events.push('inspect'); + throw new Error('not reached'); + }, + async call() { + events.push('call'); + throw new Error('not reached'); + }, + }, + async invokeAdapter() { + events.push('select'); + return { tool_id: 'unrelated.tool' }; + }, + }); + + assert.deepEqual(events, ['discover', 'select']); + assert.equal(record.status, 'failed'); + assert.equal(record.selection.tool_id, 'unrelated.tool'); + assert.equal(record.call.attempted, false); + assert.deepEqual(record.error, { + stage: 'select', + reason_code: 'ungrounded_tool_id', + message: 'Adapter selected a tool outside the discovery results', + }); +}); + test('validates task constraints and trial bounds', async () => { assert.throws(() => validateTask({ id: 'bad', prompt: 'x', constraints: [] }), /constraints/); assert.throws( @@ -183,32 +221,40 @@ test('validates task constraints and trial bounds', async () => { ); }); -test('keeps selected_tool explicit when inspection returns no tools', async () => { - let parameterizeInput; - await runBenchmark({ +test('does not parameterize or execute when inspection omits the selected tool', async () => { + const events = []; + const [record] = await runBenchmark({ tasks: [task], model: 'model-a', trials: 1, + execute: true, api: { async discover() { + events.push('discover'); return { results: [{ tool_id: 'weather.forecast' }] }; }, async inspect() { - return { results: [] }; + events.push('inspect'); + return { results: [{ tool_id: 'different.tool', params: [] }] }; }, async call() { + events.push('call'); throw new Error('not reached'); }, }, async invokeAdapter(payload) { - if (payload.stage === 'parameterize') { - parameterizeInput = payload.input; - return { parameters: {} }; - } + events.push(payload.stage); return { tool_id: 'weather.forecast' }; }, }); - assert.equal(Object.hasOwn(parameterizeInput, 'selected_tool'), true); - assert.equal(parameterizeInput.selected_tool, null); + assert.deepEqual(events, ['discover', 'select', 'inspect']); + assert.equal(record.status, 'failed'); + assert.deepEqual(record.inspection.returned_tool_ids, ['different.tool']); + assert.equal(record.call.attempted, false); + assert.deepEqual(record.error, { + stage: 'inspect', + reason_code: 'selected_tool_not_returned', + message: 'Inspect did not return the selected tool', + }); }); diff --git a/benchmarks/discover-call/test/publication.test.mjs b/benchmarks/discover-call/test/publication.test.mjs index c841380..9fd72c6 100644 --- a/benchmarks/discover-call/test/publication.test.mjs +++ b/benchmarks/discover-call/test/publication.test.mjs @@ -131,6 +131,29 @@ test('recomputes parameterization scores instead of trusting raw attestations', }); }); +test('recomputes and verifies the catalog observation digest', () => { + const record = rawRecord(); + const [published] = sanitizePublicRecords([record], policy, tasks); + assert.equal(published.metadata.catalog_observation_sha256.length, 64); + assert.notEqual(published.metadata.catalog_observation_sha256, 'b'.repeat(64)); + assert.throws( + () => + sanitizePublicRecords( + [ + rawRecord({ + metadata: { + ...record.metadata, + catalog_observation_sha256: 'b'.repeat(64), + }, + }), + ], + policy, + tasks, + ), + /catalog observation digest does not match/, + ); +}); + test('projects safe errors and rejects unsupported public fields at every boundary', () => { const [record] = sanitizePublicRecords( [ @@ -214,11 +237,51 @@ test('projects safe errors and rejects unsupported public fields at every bounda ); }); -test('publication policy cannot expose parameters or omit a required protected field', () => { +test('rejects contradictory public lifecycle attestations', () => { + const [record] = sanitizePublicRecords([rawRecord()], policy, tasks); + assert.throws( + () => + validatePublicRecords( + [ + { + ...record, + finished_at: '2026-07-22T23:59:59.000Z', + }, + ], + policy, + ), + /inconsistent lifecycle fields/, + ); assert.throws( - () => validatePolicy({ ...policy, publish_parameters: true }), - /must not publish raw parameter values/, + () => + validatePublicRecords( + [ + { + ...record, + status: 'failed', + }, + ], + policy, + ), + /inconsistent lifecycle fields/, + ); + assert.throws( + () => + validatePublicRecords( + [ + { + ...record, + call: { attempted: false, success: true, result_nonempty: true }, + }, + ], + policy, + ), + /inconsistent call lifecycle/, ); +}); + +test('publication policy cannot expose parameters or omit a required protected field', () => { + assert.throws(() => validatePolicy({ ...policy, publish_parameters: true }), /must not publish raw parameter values/); assert.throws( () => validatePolicy({ @@ -256,10 +319,7 @@ test('official public runs require the exact task digest, execution, and three t () => validateOfficialPublicRun(records, { taskSetSha256: 'b'.repeat(64) }), /task-set digest does not match/, ); - assert.throws( - () => validateOfficialPublicRun(records.slice(0, 2), { taskSetSha256 }), - /at least 3 trials/, - ); + assert.throws(() => validateOfficialPublicRun(records.slice(0, 2), { taskSetSha256 }), /at least 3 trials/); assert.throws( () => validateOfficialPublicRun( diff --git a/benchmarks/discover-call/test/publish.test.mjs b/benchmarks/discover-call/test/publish.test.mjs new file mode 100644 index 0000000..05d46ef --- /dev/null +++ b/benchmarks/discover-call/test/publish.test.mjs @@ -0,0 +1,53 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { mkdtemp, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { validatePublicationPaths } from '../src/publish.mjs'; + +test('publication outputs cannot overwrite inputs or each other', async () => { + const inputs = ['tasks.jsonl', 'private-runs.jsonl', 'policy.json']; + await assert.doesNotReject(() => + validatePublicationPaths({ + inputs, + outputs: ['public-runs.jsonl', 'public-summary.json'], + }), + ); + await assert.rejects( + () => + validatePublicationPaths({ + inputs, + outputs: ['private-runs.jsonl', 'public-summary.json'], + }), + /must not overwrite/, + ); + await assert.rejects( + () => + validatePublicationPaths({ + inputs, + outputs: [resolve('public-runs.jsonl'), './public-runs.jsonl'], + }), + /must use different files/, + ); +}); + +test('publication outputs cannot overwrite inputs through symlinks', async () => { + const directory = await mkdtemp(join(tmpdir(), 'qveris-publish-test-')); + const input = join(directory, 'private-runs.jsonl'); + const linkedOutput = join(directory, 'public-runs.jsonl'); + try { + await writeFile(input, '{}\n'); + await symlink(input, linkedOutput); + await assert.rejects( + () => + validatePublicationPaths({ + inputs: [input], + outputs: [linkedOutput, join(directory, 'summary.json')], + }), + /must not overwrite/, + ); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/docs/en-US/discover-call-benchmark.md b/docs/en-US/discover-call-benchmark.md index d801139..e8e5529 100644 --- a/docs/en-US/discover-call-benchmark.md +++ b/docs/en-US/discover-call-benchmark.md @@ -13,6 +13,9 @@ Long-horizon, judged domain evaluations are a separate instrument. For every task and trial, the harness runs `discover`, asks the adapter to select a returned capability, runs `inspect`, asks the adapter to construct parameters from the current schema, and performs a real `call`. +Selection outside the discovery results and inspection that omits the exact +selected capability are hard gates, so no later inspect, parameterization, or +billed call crosses an ungrounded step. The scorer reports grounded selection and inspection, required-parameter accuracy, task-constraint accuracy, call success, structural result diff --git a/docs/zh-CN/discover-call-benchmark.md b/docs/zh-CN/discover-call-benchmark.md index 304403e..4e39e89 100644 --- a/docs/zh-CN/discover-call-benchmark.md +++ b/docs/zh-CN/discover-call-benchmark.md @@ -10,6 +10,8 @@ QVeris 评测完整的 Agent 工作流，而不只对搜索相关性打分。公 每个任务和 trial 都按同一流程运行：调用 `discover`，让 adapter 从实际返回结果中选择能力，调用 `inspect`，让 adapter 根据当前 schema 构造参数，最后执行真实 `call`。 +选择结果不在 discovery 返回中，或 inspect 未返回完全一致的已选能力时，runner 会立即终止该 +trial；后续 inspect、参数构造或付费 call 不会越过未通过的 grounding 步骤。 评分分别报告选择与检查是否有真实返回依据、必填参数完整率、任务约束准确率、call 成功率、结果结构 是否非空，以及严格的端到端工作流成功率。“结果非空”不代表语义正确。Dry run 不计为工作流成功。 From 890f696769a6391b13725ee405a7e7be5ce0b4b6 Mon Sep 17 00:00:00 2001 From: Linfang Wang Date: Thu, 23 Jul 2026 22:19:32 +0800 Subject: [PATCH 13/19] fix(benchmark): harden v2 result integrity --- benchmarks/discover-call/README.md | 30 ++- .../discover-call/adapters/claude-cli.mjs | 49 +++-- .../discover-call/adapters/codex-cli.mjs | 1 + benchmarks/discover-call/results/README.md | 52 +++-- benchmarks/discover-call/src/api.mjs | 23 ++- benchmarks/discover-call/src/harness.mjs | 112 ++++++++-- benchmarks/discover-call/src/io.mjs | 22 +- benchmarks/discover-call/src/publication.mjs | 74 +++++-- benchmarks/discover-call/src/publish.mjs | 6 +- benchmarks/discover-call/src/run.mjs | 9 +- benchmarks/discover-call/src/score.mjs | 6 +- benchmarks/discover-call/src/scoring.mjs | 40 ++-- benchmarks/discover-call/src/validate.mjs | 19 +- .../discover-call/test/adapter.test.mjs | 46 ++++- benchmarks/discover-call/test/api.test.mjs | 46 +++++ .../discover-call/test/harness.test.mjs | 194 ++++++++++++++++++ benchmarks/discover-call/test/io.test.mjs | 28 +++ .../discover-call/test/publication.test.mjs | 187 +++++++++++++++++ .../discover-call/test/scoring.test.mjs | 46 +++++ 19 files changed, 862 insertions(+), 128 deletions(-) create mode 100644 benchmarks/discover-call/test/io.test.mjs diff --git a/benchmarks/discover-call/README.md b/benchmarks/discover-call/README.md index 58bfd2e..445f664 100644 --- a/benchmarks/discover-call/README.md +++ b/benchmarks/discover-call/README.md @@ -4,6 +4,11 @@ This directory contains the reproducible evaluation harness tracked by issue #163. It measures whether an agent can use the public QVeris workflow correctly, without assuming that one catalog tool ID is the only valid answer. +New runs use benchmark methodology `discover-call-v2`. Version 2 checks the +payload inside full execution-result wrappers and uses exact ordered composite +constraint matching. Historical v1 artifacts remain verifiable but must not be +mixed with v2 in one summary. + ## Scope boundary This harness deliberately stays at the contract level: deterministic scoring of @@ -30,7 +35,9 @@ Each task runs the same sequence: The runner treats both grounding checks as execution gates: it stops before `inspect` when selection is outside the discovery results, and stops before parameterization or `call` when inspection does not return the exact selected -tool. +tool. It validates the complete task set, including unique task and constraint +identifiers, before the first external request so malformed input cannot leave +a partial billed run. The scorer reports these metrics separately: @@ -40,14 +47,17 @@ The scorer reports these metrics separately: - **constraint accuracy**: fraction of task facts represented through accepted parameter aliases; - **call success**: successful real executions among attempted calls; - **result non-empty**: successful calls returned a structurally non-empty - result; this does not assert semantic correctness; + `result.data` (or non-empty truncated-content evidence); this does not assert + semantic correctness; - **workflow success**: all preceding checks equal 100%, the real call succeeds, and its result is non-empty. The transport retries `429` and `503` responses before recording an API-stage -failure, matching the public clients' transient-failure behavior. The summary -reports failures by stage; all exhausted API failures remain in the strict -workflow-success denominator. +failure, as well as transient network and response-body timeouts, matching the +public clients' transient-failure behavior. Calls explicitly request +`respond_with: "full"` so structural result scoring does not depend on a +changing server default. The summary reports failures by stage; all exhausted +API failures remain in the strict workflow-success denominator. The summary includes a deterministic 95% task-cluster bootstrap interval for workflow success. Tasks, rather than individual trials, are resampled so three @@ -206,7 +216,15 @@ inspected tool's parameter metadata. Every known parameter is represented; optional parameters are nullable because strict structured-output providers require every declared property to appear. The harness removes top-level null values before scoring and execution, so omitted optional parameters retain -their normal API semantics. +their normal API semantics. Because the public inspection contract does not +describe array items or object properties, an opaque optional array/object is +forced to `null`, while an opaque required array/object fails before model +parameterization or a billed call. + +When multiple constraints share a composite alias such as `symbol` or `pair`, +the scorer accepts only an exact, ordered composite representation (for +example, `USD/EUR`, `USD-EUR`, or `USDEUR`). Reversed pairs and substring +matches do not satisfy the constraints. Adapters may call any model provider, but must not alter canonical messages or print prompts, keys, tokens, or provider error bodies to stdout. A first-result diff --git a/benchmarks/discover-call/adapters/claude-cli.mjs b/benchmarks/discover-call/adapters/claude-cli.mjs index a7f156b..91b432b 100644 --- a/benchmarks/discover-call/adapters/claude-cli.mjs +++ b/benchmarks/discover-call/adapters/claude-cli.mjs @@ -71,28 +71,51 @@ export function parseClaudeEnvelope(stdout) { export async function invokeClaude(payload) { const invocation = buildClaudeInvocation(payload); - const stdout = await new Promise((resolve, reject) => { + return parseClaudeEnvelope(await runClaude(invocation)); +} + +export function runClaude(invocation, { outputLimit = 1_000_000, forceKillAfterMs = 1_000 } = {}) { + if (!Number.isInteger(outputLimit) || outputLimit < 1) { + throw new Error('outputLimit must be a positive integer'); + } + if (!Number.isInteger(forceKillAfterMs) || forceKillAfterMs < 1) { + throw new Error('forceKillAfterMs must be a positive integer'); + } + return new Promise((resolve, reject) => { + const env = { ...process.env }; + delete env.QVERIS_API_KEY; const child = spawn(invocation.command, invocation.args, { shell: false, stdio: ['pipe', 'pipe', 'pipe'], - env: process.env, + env, }); let output = ''; - let outputExceeded = false; let forwardedSignal; let forceKillTimer; + let terminationError; + let settled = false; const cleanup = () => { process.off('SIGINT', forwardSigint); process.off('SIGTERM', forwardSigterm); if (forceKillTimer) clearTimeout(forceKillTimer); }; + const rejectOnce = (error) => { + if (settled) return; + settled = true; + reject(error); + }; + const terminate = (error, signal = 'SIGTERM') => { + if (terminationError || settled) return; + terminationError = error; + child.kill(signal); + forceKillTimer = setTimeout(() => child.kill('SIGKILL'), forceKillAfterMs); + forceKillTimer.unref(); + }; const forwardSignal = (signal) => { if (forwardedSignal) return; forwardedSignal = signal; - child.kill(signal); - forceKillTimer = setTimeout(() => child.kill('SIGKILL'), 1_000); - forceKillTimer.unref(); + terminate(new Error('Claude CLI interrupted'), signal); }; const forwardSigint = () => forwardSignal('SIGINT'); const forwardSigterm = () => forwardSignal('SIGTERM'); @@ -101,30 +124,32 @@ export async function invokeClaude(payload) { child.stdin.on('error', () => {}); child.stdout.on('data', (chunk) => { + if (terminationError) return; output += chunk; - if (output.length > 1_000_000 && !outputExceeded) { - outputExceeded = true; - child.kill('SIGTERM'); + if (output.length > outputLimit) { + terminate(new Error('Claude CLI output exceeded the adapter limit')); } }); child.stderr.resume(); child.on('error', () => { + if (terminationError) return; cleanup(); - reject(new Error('Claude CLI could not be started')); + rejectOnce(new Error('Claude CLI could not be started')); }); child.on('close', (code) => { cleanup(); + if (settled) return; if (forwardedSignal) { process.kill(process.pid, forwardedSignal); return; } - if (outputExceeded) return reject(new Error('Claude CLI output exceeded the adapter limit')); + if (terminationError) return rejectOnce(terminationError); + settled = true; if (code === 0) resolve(output); else reject(new Error('Claude CLI invocation failed')); }); child.stdin.end(invocation.stdin); }); - return parseClaudeEnvelope(stdout); } function isObject(value) { diff --git a/benchmarks/discover-call/adapters/codex-cli.mjs b/benchmarks/discover-call/adapters/codex-cli.mjs index 4c3aede..d046163 100644 --- a/benchmarks/discover-call/adapters/codex-cli.mjs +++ b/benchmarks/discover-call/adapters/codex-cli.mjs @@ -183,6 +183,7 @@ export function runCodex(invocation, { outputLimit = 1_000_000, forceKillAfterMs }); child.stderr.resume(); child.on('error', () => { + if (terminationError) return; cleanup(); rejectOnce(adapterFailure('Codex CLI could not be started', 'start_failed')); }); diff --git a/benchmarks/discover-call/results/README.md b/benchmarks/discover-call/results/README.md index ed2c274..0c2dada 100644 --- a/benchmarks/discover-call/results/README.md +++ b/benchmarks/discover-call/results/README.md @@ -1,14 +1,24 @@ # Published benchmark results -## 2026-07-23 — v4 public quality baseline +## 2026-07-23 — v4 diagnostic baseline candidate -The v4 baseline compares the curated reference route and the configured model +This v4 run compares the curated reference route and the configured model over immutable `tasks/v4.jsonl`: 18 tasks, three trials per task, Top 10 discovery, and real calls. All 54 trials in each lane are retained. Public run records omit raw parameter values and retain only their scored required-parameter and constraint-accuracy attestations. Inspected parameter names are also omitted. +A later collector audit found that this run's `result_nonempty` implementation +tested the full `result` wrapper rather than its `result.data` payload. A +standard wrapper such as `{"data":{}}` was therefore classified as non-empty. +Raw result bodies were intentionally not retained, so the affected attestations +cannot be recomputed. The call-success and pre-result component metrics remain +diagnostic, but result non-emptiness, strict workflow success, its interval, and +the strict benchmark gap below are historical outputs rather than an official +quality baseline. Promotion requires a new run under a toolkit revision that +checks `result.data` and explicitly requests `respond_with: "full"`. + | Metric | Curated reference route | `gpt-5.6-sol` configured model | | --- | ---: | ---: | | Runs | 54 | 54 | @@ -18,12 +28,13 @@ names are also omitted. | Required-parameter accuracy | 100% | 100% | | Constraint accuracy | 94.44% | 88.89% | | Call success among attempted calls | 100% (51 / 51) | 88.24% (45 / 51) | -| Non-empty result among attempted calls | 100% (51 / 51) | 88.24% (45 / 51) | -| Strict workflow success | 94.44% (51 / 54) | 77.78% (42 / 54) | -| Workflow success, 95% task-cluster bootstrap | 83.33%–100% | 55.56%–94.44% | +| Historical wrapper-level non-empty observation | 100% (51 / 51) | 88.24% (45 / 51) | +| Historical strict workflow output | 94.44% (51 / 54) | 77.78% (42 / 54) | +| Historical workflow interval | 83.33%–100% | 55.56%–94.44% | -The v4 strict benchmark gap is **16.66 percentage points**. This is an -end-to-end benchmark difference, not a pure routing effect: the two sequential +The historical v4 strict benchmark gap output is **16.66 percentage points**. +It must not be presented as a current quality baseline. Independently, it is an +end-to-end difference rather than a pure routing effect: the two sequential lanes observed different live Top-10 catalog snapshots, as recorded by their different catalog-observation digests. @@ -52,11 +63,11 @@ Lane metadata: `e8a6ca09bbf6f24f2b0c3e6a4fffcf33d541f802cecb9ea37c543ba8ed04d054` The reference route's three failures are the expected `timezone-tokyo` -coverage misses; its 51 calls all succeeded and returned structurally non-empty -results. The configured model has 12 strict failures: three Tokyo constraint -misses, three IP lookup call failures, three company-profile call failures, and -three domain-intelligence `tool_use_rejected` adapter failures. All three -Bitcoin trials pass in v4, confirming that the task-versioned `id=1` mapping +coverage misses; its 51 attempted calls all reported success. The configured +model has 12 historical strict failures: three Tokyo constraint misses, three +IP lookup call failures, three company-profile call failures, and three +domain-intelligence `tool_use_rejected` adapter failures. All three Bitcoin +constraint checks pass in v4, confirming that the task-versioned `id=1` mapping removed the known v3 scoring false negative. Artifacts: @@ -72,7 +83,9 @@ This diagnostic run compares the curated reference route and a configured model over the same 18 tasks, three trials per task, Top 10 discovery limit, and real QVeris calls. No failed trial was removed or selectively rerun. It is not the official quality baseline because v3 has a known Bitcoin constraint-scoring -false negative. +false negative. It also predates the `result.data` collector correction +described above, so its result-non-empty and strict workflow outputs remain +historical diagnostics. | Metric | Curated reference route | `gpt-5.6-sol` configured model | | --- | ---: | ---: | @@ -83,9 +96,9 @@ false negative. | Required-parameter accuracy | 100% | 100% | | Constraint accuracy | 94.44% | 83.33% | | Call success among attempted calls | 100% (51 / 51) | 88.46% (46 / 52) | -| Non-empty result among attempted calls | 100% (51 / 51) | 88.46% (46 / 52) | -| Strict workflow success | 94.44% (51 / 54) | 72.22% (39 / 54) | -| Workflow success, 95% task-cluster bootstrap | 83.33%–100% | 50.00%–88.89% | +| Historical wrapper-level non-empty observation | 100% (51 / 51) | 88.46% (46 / 52) | +| Historical strict workflow output | 94.44% (51 / 54) | 72.22% (39 / 54) | +| Historical workflow interval | 83.33%–100% | 50.00%–88.89% | The v3 strict benchmark gap is **22.22 percentage points**: curated reference route 94.44% minus configured model 72.22%. It includes three Bitcoin false @@ -130,10 +143,9 @@ Artifacts: ### v3 diagnostic interpretation -The curated reference route's three strict failures are all `timezone-tokyo`: +The curated reference route's three historical strict failures are all `timezone-tokyo`: the fixed discovery query returned no Top 10 capability that accepts a city or -coordinates. All 51 reference-selected calls succeeded and returned non-empty -results. +coordinates. All 51 reference-selected calls reported success. The configured model also missed Tokyo semantically in all three trials: it selected a successful timezone-list capability with no Tokyo input. Six well-formed @@ -144,7 +156,7 @@ parameterization attempts were rejected because the isolated model session tried to use a disabled external tool; the remaining trial called the selected tool with `{}` and therefore missed the domain constraint. -Three cryptocurrency calls succeeded and returned non-empty Bitcoin results via +Three cryptocurrency calls succeeded and returned Bitcoin result wrappers via CoinMarketCap's provider-specific numeric `id=1`, but the deterministic task constraint expects a parameter value containing `BTC`; these remain strict constraint misses rather than being silently reclassified after the run. This diff --git a/benchmarks/discover-call/src/api.mjs b/benchmarks/discover-call/src/api.mjs index 5403d10..53c5bba 100644 --- a/benchmarks/discover-call/src/api.mjs +++ b/benchmarks/discover-call/src/api.mjs @@ -42,13 +42,18 @@ export function createApiClient({ observeRevision(response.headers, 'x-qveris-catalog-version', observedCatalogRevisions); if (!response.ok) { await Promise.resolve(response.body?.cancel?.()).catch(() => undefined); - throw apiError(`HTTP ${response.status}`); + throw apiError(`HTTP ${response.status}`, `http_${response.status}`); } let payload; try { payload = await response.json(); } catch { - throw apiError(controller.signal.aborted ? 'API response timed out' : 'Invalid API response'); + if (controller.signal.aborted) { + const error = new Error('response timeout'); + error.apiTimeoutPhase = 'response'; + throw error; + } + throw apiError('Invalid API response', 'invalid_response'); } if ( payload && @@ -61,7 +66,7 @@ export function createApiClient({ (typeof status === 'string' && ['failure', 'failed', 'error'].includes(status.toLowerCase())) || (typeof status === 'number' && status >= 400) ) { - throw apiError('API returned a failure envelope'); + throw apiError('API returned a failure envelope', 'failure_envelope'); } return payload.data; } @@ -70,7 +75,13 @@ export function createApiClient({ } catch (error) { if (error?.benchmarkStage === 'api') throw error; if (attempt >= maxRetries) { - throw apiError(controller.signal.aborted ? 'API request timed out' : 'Network request failed'); + if (controller.signal.aborted) { + throw apiError( + error?.apiTimeoutPhase === 'response' ? 'API response timed out' : 'API request timed out', + error?.apiTimeoutPhase === 'response' ? 'response_timeout' : 'request_timeout', + ); + } + throw apiError('Network request failed', 'network_failure'); } retryDelay = retryDelayMs(null, attempt); } finally { @@ -92,6 +103,7 @@ export function createApiClient({ request(`/tools/execute?tool_id=${encodeURIComponent(toolId)}`, { parameters, search_id: discoveryId ?? null, + respond_with: 'full', }), observedRevisions: () => ({ api_revision: singleRevision(observedApiRevisions), @@ -146,8 +158,9 @@ export function normalizeBaseUrl(value) { return parsed.toString().replace(/\/$/, ''); } -function apiError(message) { +function apiError(message, reason) { const error = new Error(message); error.benchmarkStage = 'api'; + error.benchmarkReason = reason; return error; } diff --git a/benchmarks/discover-call/src/harness.mjs b/benchmarks/discover-call/src/harness.mjs index 5750b5f..ef0a7bf 100644 --- a/benchmarks/discover-call/src/harness.mjs +++ b/benchmarks/discover-call/src/harness.mjs @@ -1,7 +1,20 @@ import { randomUUID } from 'node:crypto'; export const SCHEMA_VERSION = 1; -export const BENCHMARK_VERSION = 'v1'; +export const BENCHMARK_VERSION = 'v2'; + +export function validateTaskSet(tasks) { + if (!Array.isArray(tasks) || tasks.length === 0) { + throw new Error('At least one benchmark task is required'); + } + const taskIds = new Set(); + for (const task of tasks) { + validateTask(task); + if (taskIds.has(task.id)) throw new Error(`Duplicate benchmark task id: ${task.id}`); + taskIds.add(task.id); + } + return tasks; +} export function validateTask(task) { if (!task || typeof task !== 'object') throw new Error('Task must be an object'); @@ -12,10 +25,15 @@ export function validateTask(task) { if (!Array.isArray(task.constraints) || task.constraints.length === 0) { throw new Error(`Task ${task.id}: constraints must be a non-empty array`); } + const constraintIds = new Set(); for (const constraint of task.constraints) { if (typeof constraint.id !== 'string' || !constraint.id) { throw new Error(`Task ${task.id}: every constraint needs an id`); } + if (constraintIds.has(constraint.id)) { + throw new Error(`Task ${task.id}: duplicate constraint id ${constraint.id}`); + } + constraintIds.add(constraint.id); if ( !Array.isArray(constraint.aliases) || constraint.aliases.length === 0 || @@ -23,7 +41,12 @@ export function validateTask(task) { ) { throw new Error(`Task ${task.id}: constraint ${constraint.id} needs string aliases`); } - if (!('value' in constraint)) throw new Error(`Task ${task.id}: constraint ${constraint.id} needs a value`); + if ( + !['string', 'number', 'boolean'].includes(typeof constraint.value) || + (typeof constraint.value === 'number' && !Number.isFinite(constraint.value)) + ) { + throw new Error(`Task ${task.id}: constraint ${constraint.id} needs a primitive finite value`); + } if (constraint.match !== undefined && !['equals', 'contains'].includes(constraint.match)) { throw new Error(`Task ${task.id}: constraint ${constraint.id} has an unsupported match mode`); } @@ -50,7 +73,11 @@ export function validateTask(task) { !constraint.aliases.includes(alias) || !Array.isArray(values) || values.length === 0 || - values.some((value) => !['string', 'number', 'boolean'].includes(typeof value)) + values.some( + (value) => + !['string', 'number', 'boolean'].includes(typeof value) || + (typeof value === 'number' && !Number.isFinite(value)), + ) ) { throw new Error(`Task ${task.id}: constraint ${constraint.id} has invalid alias_values`); } @@ -92,7 +119,7 @@ export async function runBenchmark({ now = () => new Date().toISOString(), newRunId = randomUUID, }) { - if (!Array.isArray(tasks) || tasks.length === 0) throw new Error('At least one benchmark task is required'); + validateTaskSet(tasks); if (typeof model !== 'string' || !model.trim()) throw new Error('A model identifier is required'); if (!Number.isInteger(trials) || trials < 1 || trials > 100) throw new Error('trials must be an integer from 1 to 100'); @@ -107,8 +134,7 @@ export async function runBenchmark({ if (typeof invokeAdapter !== 'function') throw new Error('invokeAdapter must be a function'); const records = []; - for (const rawTask of tasks) { - const task = validateTask(rawTask); + for (const task of tasks) { for (let trial = 1; trial <= trials; trial++) { records.push(await runTrial({ task, model, trial, execute, limit, api, invokeAdapter, metadata, now, newRunId })); } @@ -213,8 +239,17 @@ function requiredParameterNames(tool) { export function parameterResponseSchema(tool) { const properties = {}; for (const parameter of array(tool?.params)) { - if (typeof parameter?.name !== 'string' || !parameter.name || Object.hasOwn(properties, parameter.name)) { - continue; + if ( + typeof parameter?.name !== 'string' || + !parameter.name.trim() || + typeof parameter.required !== 'boolean' || + Object.hasOwn(properties, parameter.name) + ) { + throw stageError( + 'parameterize', + 'Inspected tool contains invalid or duplicate parameter metadata', + 'unsupported_parameter_schema', + ); } properties[parameter.name] = strictParameterSchema(parameter); } @@ -275,7 +310,25 @@ function strictParameterSchema(parameter, forceRequired = parameter?.required == : `${parameter.description} Use null when this optional parameter is not needed.`; } + if (parameter?.enum !== undefined && !Array.isArray(parameter.enum)) { + throw stageError( + 'parameterize', + `Parameter ${parameter?.name ?? ''} has invalid enum metadata`, + 'unsupported_parameter_schema', + ); + } if (Array.isArray(parameter?.enum) && parameter.enum.length > 0) { + if ( + parameter.enum.some( + (value) => jsonValueType(value) === null || (typeof value === 'number' && !Number.isFinite(value)), + ) + ) { + throw stageError( + 'parameterize', + `Parameter ${parameter?.name ?? ''} has unsupported enum values`, + 'unsupported_parameter_schema', + ); + } schema.type = enumType(parameter.enum, forceRequired); schema.enum = forceRequired ? [...parameter.enum] : [...parameter.enum, null]; return schema; @@ -283,13 +336,31 @@ function strictParameterSchema(parameter, forceRequired = parameter?.required == schema.type = nullableType(type, forceRequired); if (type === 'array') { - schema.items = strictParameterSchema( - isPlainObject(parameter?.items) ? parameter.items : { type: 'string', required: true }, - true, - ); + if (!isPlainObject(parameter?.items)) { + if (forceRequired) { + throw stageError( + 'parameterize', + `Required array parameter ${parameter?.name ?? ''} has no item schema`, + 'unsupported_parameter_schema', + ); + } + return { ...schema, type: 'null' }; + } + schema.items = strictParameterSchema(parameter.items, true); } else if (type === 'object') { + const nestedDefinitions = isPlainObject(parameter?.properties) ? parameter.properties : {}; + if (Object.keys(nestedDefinitions).length === 0) { + if (forceRequired) { + throw stageError( + 'parameterize', + `Required object parameter ${parameter?.name ?? ''} has no property schema`, + 'unsupported_parameter_schema', + ); + } + return { ...schema, type: 'null' }; + } const nestedProperties = {}; - for (const [name, definition] of Object.entries(isPlainObject(parameter?.properties) ? parameter.properties : {})) { + for (const [name, definition] of Object.entries(nestedDefinitions)) { nestedProperties[name] = strictParameterSchema(isPlainObject(definition) ? definition : { type: 'string' }); } schema.additionalProperties = false; @@ -300,13 +371,17 @@ function strictParameterSchema(parameter, forceRequired = parameter?.required == } function normalizeParameterType(value) { - const type = typeof value === 'string' ? value.toLowerCase() : 'string'; + if (typeof value !== 'string') { + throw stageError('parameterize', 'Parameter type is missing', 'unsupported_parameter_schema'); + } + const type = value.toLowerCase(); if (['integer', 'int'].includes(type)) return 'integer'; if (['number', 'float', 'double'].includes(type)) return 'number'; if (['boolean', 'bool'].includes(type)) return 'boolean'; if (['array', 'list'].includes(type)) return 'array'; if (['object', 'map'].includes(type)) return 'object'; - return 'string'; + if (type === 'string') return 'string'; + throw stageError('parameterize', `Unsupported parameter type: ${value}`, 'unsupported_parameter_schema'); } function nullableType(type, required) { @@ -336,7 +411,12 @@ function hasNonemptyResult(result) { if (result === null || result === undefined) return false; if (typeof result === 'string') return result.trim().length > 0; if (Array.isArray(result)) return result.length > 0; - if (isPlainObject(result)) return Object.keys(result).length > 0; + if (isPlainObject(result)) { + if (Object.hasOwn(result, 'data')) return hasNonemptyResult(result.data); + if (typeof result.truncated_content === 'string' && result.truncated_content.trim()) return true; + if (typeof result.full_content_file_url === 'string' && result.full_content_file_url.trim()) return true; + return Object.keys(result).length > 0; + } return true; } diff --git a/benchmarks/discover-call/src/io.mjs b/benchmarks/discover-call/src/io.mjs index ba34e55..071cf60 100644 --- a/benchmarks/discover-call/src/io.mjs +++ b/benchmarks/discover-call/src/io.mjs @@ -1,4 +1,5 @@ -import { readFile, writeFile } from 'node:fs/promises'; +import { randomUUID } from 'node:crypto'; +import { open, readFile, rename, unlink } from 'node:fs/promises'; export async function readJsonLines(path) { const text = await readFile(path, 'utf8'); @@ -17,5 +18,22 @@ export async function readJsonLines(path) { export async function writeJsonLines(path, records) { const content = records.map((record) => JSON.stringify(record)).join('\n') + '\n'; - await writeFile(path, content, { encoding: 'utf8', mode: 0o600 }); + await writeTextAtomic(path, content); +} + +export async function writeTextAtomic(path, content) { + const temporaryPath = `${path}.${process.pid}.${randomUUID()}.tmp`; + let handle; + try { + handle = await open(temporaryPath, 'wx', 0o600); + await handle.writeFile(content, { encoding: 'utf8' }); + await handle.sync(); + await handle.close(); + handle = undefined; + await rename(temporaryPath, path); + } catch (error) { + await handle?.close().catch(() => {}); + await unlink(temporaryPath).catch(() => {}); + throw error; + } } diff --git a/benchmarks/discover-call/src/publication.mjs b/benchmarks/discover-call/src/publication.mjs index 8570ead..6f87061 100644 --- a/benchmarks/discover-call/src/publication.mjs +++ b/benchmarks/discover-call/src/publication.mjs @@ -1,8 +1,10 @@ import { createHash } from 'node:crypto'; +import { validateTaskSet } from './harness.mjs'; import { scoreRecord } from './scoring.mjs'; const PUBLIC_API_HOSTS = new Set(['qveris.ai', 'qveris.cn']); +const PUBLIC_LANES = new Set(['model', 'reference', 'configured-model', 'pinned-model', 'current-model']); const PUBLIC_RECORD_FIELDS = new Set([ 'schema_version', @@ -90,9 +92,11 @@ export function sanitizePublicRecords(records, policy, tasks) { ) { throw new Error('Raw benchmark catalog observation digest does not match its records'); } - return records.map((record) => + const sanitized = records.map((record) => sanitizePublicRecord(record, taskById.get(record.task_id), policy, approvedToolIds, catalogObservationSha256), ); + validatePublicRecords(sanitized, policy); + return sanitized; } function sanitizePublicRecord( @@ -163,7 +167,7 @@ export function validatePolicy(policy) { if (!policy || typeof policy !== 'object' || policy.schema_version !== 1) { throw new Error('Publication policy schema_version must be 1'); } - if (typeof policy.policy_id !== 'string' || !policy.policy_id) { + if (!safePublicString(policy.policy_id, 128)) { throw new Error('Publication policy needs a policy_id'); } if (policy.catalog_visibility_default !== 'private') { @@ -171,7 +175,7 @@ export function validatePolicy(policy) { } if ( !Array.isArray(policy.approved_selected_tool_ids) || - policy.approved_selected_tool_ids.some((toolId) => typeof toolId !== 'string' || !toolId) + policy.approved_selected_tool_ids.some((toolId) => !safePublicString(toolId, 512)) ) { throw new Error('Publication policy needs approved_selected_tool_ids'); } @@ -193,9 +197,9 @@ export function validatePolicy(policy) { if ( policy.legacy_lane_rewrites !== undefined && (!plainObject(policy.legacy_lane_rewrites) || - Object.values(policy.legacy_lane_rewrites).some((lane) => typeof lane !== 'string' || !lane)) + Object.values(policy.legacy_lane_rewrites).some((lane) => !PUBLIC_LANES.has(lane))) ) { - throw new Error('Publication policy legacy_lane_rewrites must map strings to strings'); + throw new Error('Publication policy legacy_lane_rewrites must map to public lanes'); } if (policy.unapproved_selected_tool_handling !== 'hash') { throw new Error('Publication policy must hash unapproved selected tools'); @@ -214,10 +218,10 @@ export function validatePublicRecords(records, policy) { assertObjectFields(record, PUBLIC_RECORD_FIELDS, '$'); if ( record.schema_version !== 1 || - record.benchmark_version !== 'v1' || - !nonemptyString(record.run_id) || - !nonemptyString(record.model) || - !nonemptyString(record.task_id) || + !['v1', 'v2'].includes(record.benchmark_version) || + !safePublicString(record.run_id, 256) || + !safePublicString(record.model, 256) || + !safePublicString(record.task_id, 256) || !Number.isInteger(record.trial) || record.trial < 1 || !['completed', 'failed'].includes(record.status) || @@ -258,6 +262,19 @@ export function validatePublicRecords(records, policy) { ) { throw new Error('Public artifact has invalid reproducibility metadata'); } + if ( + !PUBLIC_LANES.has(record.metadata.lane) || + !safePublicString(record.metadata.model_revision, 256) || + !safePublicString(record.metadata.adapter_revision, 256) || + !safePublicString(record.metadata.toolkit_revision, 256) || + !safePublicString(record.metadata.api_revision, 256) || + !safePublicString(record.metadata.catalog_revision, 256) || + !safePublicString(record.metadata.runtime.node, 128) || + !safePublicString(record.metadata.runtime.platform, 128) || + !safePublicString(record.metadata.runtime.arch, 128) + ) { + throw new Error('Public artifact has invalid public metadata values'); + } if (selectedToolId && !approvedToolIds.has(selectedToolId)) { throw new Error(`Public artifact contains an unapproved selected tool: ${selectedToolId}`); } @@ -292,6 +309,14 @@ export function validatePublicRecords(records, policy) { ) { throw new Error('Public artifact grounding attestations must be booleans'); } + const hasSelectionIdentity = Boolean(selectedToolId || record.selection?.tool_id_sha256); + if ( + (record.discovery.selection_grounded && (record.discovery.result_count === 0 || !hasSelectionIdentity)) || + (record.inspection.selection_grounded && (!record.discovery.selection_grounded || !hasSelectionIdentity)) || + (record.status === 'completed' && (!record.discovery.selection_grounded || !record.inspection.selection_grounded)) + ) { + throw new Error('Public artifact has inconsistent grounding attestations'); + } if ( typeof record.call?.attempted !== 'boolean' || !booleanOrNull(record.call?.success) || @@ -303,14 +328,17 @@ export function validatePublicRecords(records, policy) { (!record.call.attempted && (record.call.success !== null || record.call.result_nonempty !== undefined)) || (record.call.result_nonempty === true && record.call.success !== true) || (record.call.attempted && record.metadata.execute !== true) || - (record.status === 'completed' && record.metadata.execute === true && record.call.attempted !== true) + (record.status === 'completed' && record.metadata.execute === true && record.call.attempted !== true) || + (record.status === 'completed' && record.call.attempted && typeof record.call.success !== 'boolean') || + (record.status === 'failed' && (record.call.success !== null || record.call.result_nonempty !== undefined)) ) { throw new Error('Public artifact has inconsistent call lifecycle'); } if ( record.error !== undefined && - (safeCode(record.error.stage) !== record.error.stage || - safeCode(record.error.reason_code) !== record.error.reason_code) + (safeCode(record.error.stage) === null || + safeCode(record.error.stage) !== record.error.stage || + (record.error.reason_code !== null && safeCode(record.error.reason_code) !== record.error.reason_code)) ) { throw new Error('Public artifact error codes are invalid'); } @@ -393,7 +421,7 @@ function assertObjectFields(value, allowed, path) { function publicError(error) { return { stage: safeCode(error?.stage), - reason_code: safeCode(error?.reason_code), + reason_code: safeCode(error?.reason_code) ?? 'unclassified', }; } @@ -429,8 +457,19 @@ function nonemptyString(value) { return typeof value === 'string' && value.length > 0; } +function safePublicString(value, maxLength) { + return ( + nonemptyString(value) && value.trim().length > 0 && value.length <= maxLength && !/[\p{Cc}\p{Cf}]/u.test(value) + ); +} + function validTimestamp(value) { - return typeof value === 'string' && /^\d{4}-\d{2}-\d{2}T/.test(value) && Number.isFinite(Date.parse(value)); + if (typeof value !== 'string' || !Number.isFinite(Date.parse(value))) return false; + try { + return new Date(value).toISOString() === value; + } catch { + return false; + } } function safePublicApiUrl(value) { @@ -452,14 +491,9 @@ function safePublicApiUrl(value) { } function taskMap(tasks) { - if (!Array.isArray(tasks) || tasks.length === 0) { - throw new Error('Public artifact publication needs benchmark tasks'); - } + validateTaskSet(tasks); const result = new Map(); for (const task of tasks) { - if (!plainObject(task) || typeof task.id !== 'string' || !task.id || result.has(task.id)) { - throw new Error('Public artifact publication needs unique task ids'); - } result.set(task.id, task); } return result; diff --git a/benchmarks/discover-call/src/publish.mjs b/benchmarks/discover-call/src/publish.mjs index 0b17ccc..1909f0f 100644 --- a/benchmarks/discover-call/src/publish.mjs +++ b/benchmarks/discover-call/src/publish.mjs @@ -1,11 +1,11 @@ #!/usr/bin/env node import { createHash } from 'node:crypto'; -import { mkdir, readFile, realpath, stat, writeFile } from 'node:fs/promises'; +import { mkdir, readFile, realpath, stat } from 'node:fs/promises'; import { basename, dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { readJsonLines, writeJsonLines } from './io.mjs'; +import { readJsonLines, writeJsonLines, writeTextAtomic } from './io.mjs'; import { sanitizePublicRecords, validateOfficialPublicRun, validatePublicRecords } from './publication.mjs'; import { scoreRecords } from './scoring.mjs'; @@ -39,7 +39,7 @@ export async function main(argv = process.argv.slice(2)) { mkdir(dirname(outputSummary), { recursive: true }), ]); await writeJsonLines(outputRuns, publicRecords); - await writeFile(outputSummary, JSON.stringify(summary, null, 2) + '\n', { encoding: 'utf8', mode: 0o600 }); + await writeTextAtomic(outputSummary, JSON.stringify(summary, null, 2) + '\n'); process.stdout.write(`Wrote ${publicRecords.length} sanitized records and summary\n`); } diff --git a/benchmarks/discover-call/src/run.mjs b/benchmarks/discover-call/src/run.mjs index c0cefe9..1cac0a1 100644 --- a/benchmarks/discover-call/src/run.mjs +++ b/benchmarks/discover-call/src/run.mjs @@ -87,6 +87,7 @@ export function createProcessAdapter({ command, args = [], timeoutMs = 120_000, let stdoutExceeded = false; let stderrExceeded = false; let settled = false; + let terminationError; let forceKillTimer; const cleanup = () => { clearTimeout(timer); @@ -98,18 +99,19 @@ export function createProcessAdapter({ command, args = [], timeoutMs = 120_000, reject(error); }; const terminate = (error) => { - if (settled) return; + if (settled || terminationError) return; + terminationError = error; clearTimeout(timer); child.kill('SIGTERM'); forceKillTimer = setTimeout(() => child.kill('SIGKILL'), forceKillAfterMs); forceKillTimer.unref(); - rejectOnce(error); }; const timer = setTimeout(() => { terminate(adapterError('Adapter timed out', 'timeout')); }, timeoutMs); child.stdout.on('data', (chunk) => { + if (terminationError) return; stdout += chunk; if (stdout.length > 1_000_000 && !stdoutExceeded) { stdoutExceeded = true; @@ -117,6 +119,7 @@ export function createProcessAdapter({ command, args = [], timeoutMs = 120_000, } }); child.stderr.on('data', (chunk) => { + if (terminationError) return; stderr += chunk; if (stderr.length > 100_000 && !stderrExceeded) { stderrExceeded = true; @@ -124,12 +127,14 @@ export function createProcessAdapter({ command, args = [], timeoutMs = 120_000, } }); child.on('error', () => { + if (terminationError) return; cleanup(); rejectOnce(adapterError('Adapter could not be started', 'start_failed')); }); child.on('close', (code) => { cleanup(); if (settled) return; + if (terminationError) return rejectOnce(terminationError); settled = true; if (code !== 0) { return reject( diff --git a/benchmarks/discover-call/src/score.mjs b/benchmarks/discover-call/src/score.mjs index b2d0fd1..8e5f31c 100644 --- a/benchmarks/discover-call/src/score.mjs +++ b/benchmarks/discover-call/src/score.mjs @@ -1,10 +1,10 @@ #!/usr/bin/env node -import { mkdir, writeFile } from 'node:fs/promises'; +import { mkdir } from 'node:fs/promises'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { readJsonLines } from './io.mjs'; +import { readJsonLines, writeTextAtomic } from './io.mjs'; import { scoreRecords } from './scoring.mjs'; const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..'); @@ -20,7 +20,7 @@ export async function main(argv = process.argv.slice(2)) { if (options.output) { const output = resolve(options.output); await mkdir(dirname(output), { recursive: true }); - await writeFile(output, json, { encoding: 'utf8', mode: 0o600 }); + await writeTextAtomic(output, json); process.stdout.write(`Wrote benchmark summary to ${output}\n`); } else { process.stdout.write(json); diff --git a/benchmarks/discover-call/src/scoring.mjs b/benchmarks/discover-call/src/scoring.mjs index 4aae3e8..aa422b9 100644 --- a/benchmarks/discover-call/src/scoring.mjs +++ b/benchmarks/discover-call/src/scoring.mjs @@ -1,6 +1,16 @@ +import { validateTaskSet } from './harness.mjs'; + export function scoreRecords(tasks, records) { - if (!Array.isArray(tasks) || tasks.length === 0) throw new Error('At least one benchmark task is required'); + validateTaskSet(tasks); if (!Array.isArray(records) || records.length === 0) throw new Error('At least one run record is required'); + const benchmarkVersions = new Set(records.map((record) => record?.benchmark_version ?? 'v1')); + if (benchmarkVersions.size !== 1) { + throw new Error('Benchmark summary cannot mix benchmark versions'); + } + const benchmarkVersion = [...benchmarkVersions][0]; + if (!['v1', 'v2'].includes(benchmarkVersion)) { + throw new Error(`Unsupported benchmark version: ${benchmarkVersion}`); + } const taskById = new Map(tasks.map((task) => [task.id, task])); const runIds = new Set(); for (const record of records) { @@ -23,7 +33,7 @@ export function scoreRecords(tasks, records) { return { schema_version: 1, - methodology: 'discover-call-v1', + methodology: `discover-call-${benchmarkVersion}`, generated_at: new Date().toISOString(), models: [...byModel.entries()].map(([model, results]) => aggregateModel(model, results)), records: scored, @@ -111,7 +121,7 @@ export function scoreRecord(task, record) { (required.length === 0 ? 1 : mean(required.map((name) => hasValue(parameters[name])))); const constraintAccuracy = parameterization?.constraint_accuracy ?? - mean(task.constraints.map((constraint) => constraintSatisfied(parameters, constraint))); + mean(task.constraints.map((constraint) => constraintSatisfied(parameters, constraint, task.constraints))); const executed = record.call?.attempted === true; const callSuccess = executed ? record.call?.success === true : null; const resultNonempty = @@ -183,9 +193,7 @@ function aggregateModel(model, records) { constraint_accuracy: round(mean(records.map((record) => record.constraint_accuracy))), call_success_rate: executed.length ? round(mean(executed.map((record) => record.call_success))) : null, result_nonempty_rate: - executed.length && resultEvidenceComplete - ? round(mean(executed.map((record) => record.result_nonempty))) - : null, + executed.length && resultEvidenceComplete ? round(mean(executed.map((record) => record.result_nonempty))) : null, pre_result_gate_workflow_success_rate: round( mean(records.map((record) => record.pre_result_gate_workflow_success)), ), @@ -196,7 +204,7 @@ function aggregateModel(model, records) { }; } -function constraintSatisfied(parameters, constraint) { +function constraintSatisfied(parameters, constraint, constraints) { for (const alias of constraint.aliases) { if (!hasValue(parameters[alias])) continue; const actual = parameters[alias]; @@ -211,7 +219,17 @@ function constraintSatisfied(parameters, constraint) { } for (const alias of array(constraint.composite_aliases)) { if (!hasValue(parameters[alias])) continue; - if (normalize(parameters[alias], constraint).includes(normalize(constraint.value, constraint))) return 1; + const compositeConstraints = constraints.filter((candidate) => array(candidate.composite_aliases).includes(alias)); + const values = compositeConstraints.map((candidate) => normalize(candidate.value, candidate)); + const representations = new Set([ + values.join(''), + values.join('/'), + values.join('-'), + values.join('_'), + values.join(':'), + values.join(' '), + ]); + if (representations.has(normalize(parameters[alias], constraint))) return 1; } return 0; } @@ -301,11 +319,7 @@ function booleanOrNull(value) { function parameterizationMetrics(value) { if (value === undefined) return null; - if ( - !plainObject(value) || - !metric(value.required_parameter_accuracy) || - !metric(value.constraint_accuracy) - ) { + if (!plainObject(value) || !metric(value.required_parameter_accuracy) || !metric(value.constraint_accuracy)) { throw new Error('Run record parameterization metrics must be between 0 and 1'); } return value; diff --git a/benchmarks/discover-call/src/validate.mjs b/benchmarks/discover-call/src/validate.mjs index 0221048..996ef1a 100644 --- a/benchmarks/discover-call/src/validate.mjs +++ b/benchmarks/discover-call/src/validate.mjs @@ -4,13 +4,9 @@ import { resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { isDeepStrictEqual } from 'node:util'; -import { validateTask } from './harness.mjs'; +import { validateTaskSet } from './harness.mjs'; import { readJsonLines } from './io.mjs'; -import { - validateOfficialPublicRun, - validatePolicy, - validatePublicRecords, -} from './publication.mjs'; +import { validateOfficialPublicRun, validatePolicy, validatePublicRecords } from './publication.mjs'; import { scoreRecords } from './scoring.mjs'; const root = resolve(fileURLToPath(new URL('..', import.meta.url))); @@ -24,12 +20,7 @@ if (taskFiles.length === 0) throw new Error('No versioned task sets found under const counts = []; for (const file of taskFiles) { const tasks = await readJsonLines(resolve(root, 'tasks', file)); - const ids = new Set(); - for (const task of tasks) { - validateTask(task); - if (ids.has(task.id)) throw new Error(`${file}: duplicate task id: ${task.id}`); - ids.add(task.id); - } + validateTaskSet(tasks); counts.push(`${file}: ${tasks.length} tasks`); } @@ -40,9 +31,7 @@ scoreRecords(v1, fixture); const policy = JSON.parse(await readFile(resolve(root, 'publication-policy.json'), 'utf8')); validatePolicy(policy); -const resultFiles = (await readdir(resolve(root, 'results'))) - .filter((name) => /\.runs\.jsonl$/.test(name)) - .sort(); +const resultFiles = (await readdir(resolve(root, 'results'))).filter((name) => /\.runs\.jsonl$/.test(name)).sort(); for (const file of resultFiles) { const version = file.match(/-v(\d+)\.runs\.jsonl$/)?.[1]; if (!version) throw new Error(`${file}: result filename must identify its task-set version`); diff --git a/benchmarks/discover-call/test/adapter.test.mjs b/benchmarks/discover-call/test/adapter.test.mjs index 1b9b2b3..2f85c33 100644 --- a/benchmarks/discover-call/test/adapter.test.mjs +++ b/benchmarks/discover-call/test/adapter.test.mjs @@ -7,7 +7,7 @@ import { join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { createProcessAdapter, main } from '../src/run.mjs'; -import { buildClaudeInvocation, parseClaudeEnvelope } from '../adapters/claude-cli.mjs'; +import { buildClaudeInvocation, parseClaudeEnvelope, runClaude } from '../adapters/claude-cli.mjs'; import { buildCodexInvocation, CODEX_REASONING_EFFORT, parseCodexEvents, runCodex } from '../adapters/codex-cli.mjs'; const adapterPath = resolve(fileURLToPath(new URL('../adapters/first-result.mjs', import.meta.url))); @@ -107,7 +107,7 @@ test('process adapter force-kills a child that ignores SIGTERM', async () => { }); await assert.rejects(invoke({ stage: 'select' }), /timed out/); const pid = Number(await readFile(pidPath, 'utf8')); - assert.equal(await waitForProcessExit(pid), true); + assert.equal(processExists(pid), false); } finally { await rm(directory, { recursive: true, force: true }); } @@ -213,6 +213,39 @@ test('Claude adapter extracts only structured model output', () => { assert.throws(() => parseClaudeEnvelope(JSON.stringify({ result: 'not-json' })), /no structured output/); }); +test('Claude adapter removes the QVeris key from its model subprocess', async () => { + const previous = process.env.QVERIS_API_KEY; + process.env.QVERIS_API_KEY = 'must-not-reach-model'; + try { + const output = await runClaude({ + command: process.execPath, + args: ['-e', 'process.stdout.write(String(Boolean(process.env.QVERIS_API_KEY)))'], + stdin: '', + }); + assert.equal(output, 'false'); + } finally { + if (previous === undefined) delete process.env.QVERIS_API_KEY; + else process.env.QVERIS_API_KEY = previous; + } +}); + +test('Claude adapter force-kills a child that ignores the output-limit signal', async () => { + await assert.rejects( + runClaude( + { + command: process.execPath, + args: [ + '-e', + "process.on('SIGTERM', () => {}); process.stdout.write('x'.repeat(100)); setInterval(() => {}, 1000)", + ], + stdin: '', + }, + { outputLimit: 10, forceKillAfterMs: 20 }, + ), + /output exceeded/, + ); +}); + test('Codex adapter preserves canonical messages and response schema', () => { const payload = { model: 'gpt-5.6-sol', @@ -329,12 +362,3 @@ function processExists(pid) { throw error; } } - -async function waitForProcessExit(pid, timeoutMs = 1_000) { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - if (!processExists(pid)) return true; - await new Promise((resolvePromise) => setTimeout(resolvePromise, 20)); - } - return !processExists(pid); -} diff --git a/benchmarks/discover-call/test/api.test.mjs b/benchmarks/discover-call/test/api.test.mjs index 3b60b0b..8cdd1d5 100644 --- a/benchmarks/discover-call/test/api.test.mjs +++ b/benchmarks/discover-call/test/api.test.mjs @@ -126,6 +126,52 @@ test('request timeout covers a response body that stalls after headers', async ( assert.equal(signal.aborted, true); }); +test('retries a response-body timeout before recording an API failure', async () => { + let attempts = 0; + const delays = []; + const client = createApiClient({ + apiKey: 'test-key', + timeoutMs: 10, + maxRetries: 1, + sleep: async (ms) => delays.push(ms), + fetchImpl: async (_url, init) => { + attempts++; + if (attempts === 2) return new Response(JSON.stringify({ results: [] })); + return { + ok: true, + status: 200, + headers: new Headers(), + json: () => + new Promise((resolve, reject) => { + init.signal.addEventListener( + 'abort', + () => reject(Object.assign(new Error('aborted'), { name: 'AbortError' })), + { once: true }, + ); + }), + }; + }, + }); + + assert.deepEqual(await client.discover({ query: 'weather', limit: 5 }), { results: [] }); + assert.equal(attempts, 2); + assert.deepEqual(delays, [500]); +}); + +test('pins full execution results for structural non-empty scoring', async () => { + let requestBody; + const client = createApiClient({ + apiKey: 'test-key', + fetchImpl: async (_url, init) => { + requestBody = JSON.parse(init.body); + return new Response(JSON.stringify({ success: true, result: { data: { value: 1 } } })); + }, + }); + + await client.call({ toolId: 'weather.tool', discoveryId: 'search-1', parameters: { city: 'London' } }); + assert.equal(requestBody.respond_with, 'full'); +}); + test('cancels unsuccessful response bodies before reporting API errors', async () => { let cancelled = false; const client = createApiClient({ diff --git a/benchmarks/discover-call/test/harness.test.mjs b/benchmarks/discover-call/test/harness.test.mjs index 4682006..220a022 100644 --- a/benchmarks/discover-call/test/harness.test.mjs +++ b/benchmarks/discover-call/test/harness.test.mjs @@ -61,6 +61,7 @@ test('orchestrates discover, select, inspect, parameterize, and call', async () ['discover', 'select', 'inspect', 'parameterize', 'call'], ); assert.equal(records[0].status, 'completed'); + assert.equal(records[0].benchmark_version, 'v2'); assert.deepEqual(records[0].inspection.required_parameters, ['city']); assert.deepEqual(records[0].parameters, { city: 'London' }); assert.equal(records[0].call.success, true); @@ -118,6 +119,118 @@ test('builds a strict provider-neutral schema from inspected parameters', () => ); }); +test('rejects required object parameters without a usable property schema', () => { + assert.throws( + () => + parameterResponseSchema({ + params: [{ name: 'payload', type: 'object', required: true }], + }), + (error) => + error.benchmarkStage === 'parameterize' && + error.benchmarkReason === 'unsupported_parameter_schema' && + /payload/.test(error.message), + ); + assert.throws( + () => + parameterResponseSchema({ + params: [{ name: 'items', type: 'array', required: true }], + }), + (error) => + error.benchmarkStage === 'parameterize' && + error.benchmarkReason === 'unsupported_parameter_schema' && + /items/.test(error.message), + ); + assert.deepEqual( + parameterResponseSchema({ + params: [ + { name: 'payload', type: 'object', required: false }, + { name: 'items', type: 'array', required: false }, + ], + }).properties.parameters.properties, + { + payload: { type: 'null' }, + items: { type: 'null' }, + }, + ); +}); + +test('rejects ambiguous inspected parameter metadata instead of guessing', () => { + assert.throws( + () => + parameterResponseSchema({ + params: [ + { name: 'query', type: 'string', required: true }, + { name: 'query', type: 'number', required: false }, + ], + }), + (error) => error.benchmarkReason === 'unsupported_parameter_schema', + ); + assert.throws( + () => + parameterResponseSchema({ + params: [{ name: 'query', type: 'unknown', required: true }], + }), + (error) => error.benchmarkReason === 'unsupported_parameter_schema', + ); + assert.throws( + () => + parameterResponseSchema({ + params: [{ name: 'query', type: 'string' }], + }), + (error) => error.benchmarkReason === 'unsupported_parameter_schema', + ); + assert.throws( + () => + parameterResponseSchema({ + params: [{ name: 'query', type: 'string', required: true, enum: { unsafe: true } }], + }), + (error) => error.benchmarkReason === 'unsupported_parameter_schema', + ); +}); + +test('does not invoke parameterization or a billed call for an opaque required parameter', async () => { + const events = []; + const [record] = await runBenchmark({ + tasks: [task], + model: 'model-a', + trials: 1, + execute: true, + api: { + async discover() { + events.push('discover'); + return { search_id: 'search-1', results: [{ tool_id: 'weather.forecast' }] }; + }, + async inspect() { + events.push('inspect'); + return { + results: [ + { + tool_id: 'weather.forecast', + params: [{ name: 'payload', type: 'object', required: true }], + }, + ], + }; + }, + async call() { + events.push('call'); + throw new Error('not reached'); + }, + }, + async invokeAdapter(payload) { + events.push(payload.stage); + return { tool_id: 'weather.forecast' }; + }, + }); + + assert.deepEqual(events, ['discover', 'select', 'inspect']); + assert.equal(record.call.attempted, false); + assert.deepEqual(record.error, { + stage: 'parameterize', + reason_code: 'unsupported_parameter_schema', + message: 'Required object parameter payload has no property schema', + }); +}); + test('records adapter failures without copying their error text', async () => { const records = await runBenchmark({ tasks: [task], @@ -149,6 +262,40 @@ test('records adapter failures without copying their error text', async () => { assert.equal(JSON.stringify(records[0]).includes('secret-token'), false); }); +test('checks the full result data instead of treating its wrapper as non-empty', async () => { + const [record] = await runBenchmark({ + tasks: [task], + model: 'model-a', + trials: 1, + execute: true, + api: { + async discover() { + return { search_id: 'search-1', results: [{ tool_id: 'weather.forecast' }] }; + }, + async inspect() { + return { + results: [ + { + tool_id: 'weather.forecast', + params: [{ name: 'city', type: 'string', required: true }], + }, + ], + }; + }, + async call() { + return { success: true, result: { data: {} } }; + }, + }, + async invokeAdapter(payload) { + return payload.stage === 'select' ? { tool_id: 'weather.forecast' } : { parameters: { city: 'London' } }; + }, + }); + + assert.equal(record.status, 'completed'); + assert.equal(record.call.success, true); + assert.equal(record.call.result_nonempty, false); +}); + test('does not inspect or execute a tool outside the discovery results', async () => { const events = []; const [record] = await runBenchmark({ @@ -215,12 +362,59 @@ test('validates task constraints and trial bounds', async () => { }), /alias_values/, ); + assert.throws( + () => + validateTask({ + ...task, + constraints: [task.constraints[0], { ...task.constraints[0] }], + }), + /duplicate constraint id/, + ); + assert.throws( + () => + validateTask({ + ...task, + constraints: [{ ...task.constraints[0], value: Number.POSITIVE_INFINITY }], + }), + /primitive finite value/, + ); await assert.rejects( runBenchmark({ tasks: [task], model: 'model-a', trials: 0, api: {}, invokeAdapter() {} }), /trials/, ); }); +test('validates the complete task set before any external call', async () => { + let externalCalls = 0; + const api = { + async discover() { + externalCalls++; + return { results: [] }; + }, + async inspect() { + externalCalls++; + return { results: [] }; + }, + async call() { + externalCalls++; + return { success: false }; + }, + }; + await assert.rejects( + runBenchmark({ + tasks: [task, { ...task }], + model: 'model-a', + api, + async invokeAdapter() { + externalCalls++; + return {}; + }, + }), + /Duplicate benchmark task id/, + ); + assert.equal(externalCalls, 0); +}); + test('does not parameterize or execute when inspection omits the selected tool', async () => { const events = []; const [record] = await runBenchmark({ diff --git a/benchmarks/discover-call/test/io.test.mjs b/benchmarks/discover-call/test/io.test.mjs new file mode 100644 index 0000000..9096d63 --- /dev/null +++ b/benchmarks/discover-call/test/io.test.mjs @@ -0,0 +1,28 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { mkdtemp, readFile, readdir, rm, stat } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { writeJsonLines, writeTextAtomic } from '../src/io.mjs'; + +test('atomically replaces private benchmark output files', async () => { + const directory = await mkdtemp(join(tmpdir(), 'qveris-benchmark-io-')); + const textPath = join(directory, 'summary.json'); + const linesPath = join(directory, 'runs.jsonl'); + try { + await writeTextAtomic(textPath, 'first\n'); + await writeTextAtomic(textPath, 'second\n'); + await writeJsonLines(linesPath, [{ run_id: 'run-1' }, { run_id: 'run-2' }]); + + assert.equal(await readFile(textPath, 'utf8'), 'second\n'); + assert.equal(await readFile(linesPath, 'utf8'), '{"run_id":"run-1"}\n{"run_id":"run-2"}\n'); + if (process.platform !== 'win32') { + assert.equal((await stat(textPath)).mode & 0o777, 0o600); + assert.equal((await stat(linesPath)).mode & 0o777, 0o600); + } + assert.deepEqual((await readdir(directory)).sort(), ['runs.jsonl', 'summary.json']); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/benchmarks/discover-call/test/publication.test.mjs b/benchmarks/discover-call/test/publication.test.mjs index 9fd72c6..8042ecc 100644 --- a/benchmarks/discover-call/test/publication.test.mjs +++ b/benchmarks/discover-call/test/publication.test.mjs @@ -278,6 +278,193 @@ test('rejects contradictory public lifecycle attestations', () => { ), /inconsistent call lifecycle/, ); + assert.throws( + () => + validatePublicRecords( + [ + { + ...record, + call: { attempted: true, success: null }, + }, + ], + policy, + ), + /inconsistent call lifecycle/, + ); + assert.throws( + () => + validatePublicRecords( + [ + { + ...record, + status: 'failed', + error: { stage: 'api', reason_code: 'request_failed' }, + call: { attempted: true, success: true, result_nonempty: true }, + }, + ], + policy, + ), + /inconsistent call lifecycle/, + ); +}); + +test('rejects unsafe metadata scalar types and control characters', () => { + const [record] = sanitizePublicRecords([rawRecord()], policy, tasks); + assert.throws( + () => + sanitizePublicRecords( + [ + rawRecord({ + metadata: { + ...rawRecord().metadata, + runtime: { node: { private: 'nested value' }, platform: 'darwin', arch: 'arm64' }, + }, + }), + ], + policy, + tasks, + ), + /invalid public metadata values/, + ); + assert.throws( + () => + validatePublicRecords( + [ + { + ...record, + metadata: { + ...record.metadata, + runtime: { ...record.metadata.runtime, node: { private: 'nested value' } }, + }, + }, + ], + policy, + ), + /invalid public metadata values/, + ); + assert.throws( + () => + validatePublicRecords( + [ + { + ...record, + metadata: { ...record.metadata, api_revision: 'revision\nprivate-value' }, + }, + ], + policy, + ), + /invalid public metadata values/, + ); + assert.throws( + () => + validatePublicRecords( + [ + { + ...record, + metadata: { ...record.metadata, api_revision: 'safe\u202Etxt' }, + }, + ], + policy, + ), + /invalid public metadata values/, + ); +}); + +test('rejects impossible public grounding attestations', () => { + const [record] = sanitizePublicRecords([rawRecord()], policy, tasks); + assert.throws( + () => + validatePublicRecords( + [ + { + ...record, + discovery: { ...record.discovery, result_count: 0 }, + }, + ], + policy, + ), + /inconsistent grounding attestations/, + ); + assert.throws( + () => + validatePublicRecords( + [ + { + ...record, + discovery: { ...record.discovery, selection_grounded: false }, + }, + ], + policy, + ), + /inconsistent grounding attestations/, + ); + assert.throws( + () => + validatePublicRecords( + [ + { + ...record, + selection: { tool_id: null }, + }, + ], + policy, + ), + /inconsistent grounding attestations/, + ); +}); + +test('requires a categorical error stage and safe optional reason code', () => { + const [record] = sanitizePublicRecords( + [ + rawRecord({ + status: 'failed', + selection: { tool_id: null }, + inspection: { returned_tool_ids: [], required_parameters: [] }, + parameters: null, + call: { attempted: false, success: null }, + error: { stage: 'adapter', reason_code: 'model_failed' }, + }), + ], + policy, + tasks, + ); + assert.throws( + () => + validatePublicRecords( + [ + { + ...record, + error: { stage: null, reason_code: null }, + }, + ], + policy, + ), + /error codes are invalid/, + ); + assert.doesNotThrow(() => + validatePublicRecords( + [ + { + ...record, + error: { stage: 'adapter', reason_code: null }, + }, + ], + policy, + ), + ); + assert.throws( + () => + validatePublicRecords( + [ + { + ...record, + error: { stage: 'adapter', reason_code: 'unsafe reason' }, + }, + ], + policy, + ), + /error codes are invalid/, + ); }); test('publication policy cannot expose parameters or omit a required protected field', () => { diff --git a/benchmarks/discover-call/test/scoring.test.mjs b/benchmarks/discover-call/test/scoring.test.mjs index d47ceac..4c62434 100644 --- a/benchmarks/discover-call/test/scoring.test.mjs +++ b/benchmarks/discover-call/test/scoring.test.mjs @@ -87,6 +87,33 @@ test('v3 constraints support composite parameters and explicit URL decoding', () .constraint_accuracy, 1, ); + assert.equal( + scoreRecord(exchange, { + ...baseRecord, + run_id: 'run-reversed', + task_id: exchange.id, + parameters: { symbol: 'EUR/USD' }, + }).constraint_accuracy, + 0, + ); + assert.equal( + scoreRecord(exchange, { + ...baseRecord, + run_id: 'run-substring', + task_id: exchange.id, + parameters: { symbol: 'USDT/EUR' }, + }).constraint_accuracy, + 0, + ); + assert.equal( + scoreRecord(exchange, { + ...baseRecord, + run_id: 'run-compact', + task_id: exchange.id, + parameters: { symbol: 'USDEUR' }, + }).constraint_accuracy, + 1, + ); assert.equal( scoreRecord(news, { ...baseRecord, @@ -276,6 +303,14 @@ test('rejects run records for unknown tasks', () => { ); }); +test('rejects duplicate task definitions before scoring', () => { + assert.throws( + () => + scoreRecords([task, { ...task, prompt: 'Different duplicate definition' }], [resultRecord({ taskId: task.id })]), + /Duplicate benchmark task id/, + ); +}); + function resultRecord({ taskId }) { return { run_id: `run-${taskId}`, @@ -369,4 +404,15 @@ test('rejects duplicate run ids and mixed benchmark conditions', () => { ), /schema_version/, ); + assert.throws( + () => + scoreRecords( + [task], + [ + { ...record, benchmark_version: 'v1' }, + { ...record, run_id: 'run-2', trial: 2, benchmark_version: 'v2' }, + ], + ), + /cannot mix benchmark versions/, + ); }); From 564615b3a4552c749ea25a30c8ec0b06102a914d Mon Sep 17 00:00:00 2001 From: Linfang Wang Date: Thu, 23 Jul 2026 23:58:45 +0800 Subject: [PATCH 14/19] fix(benchmark): close execution and publication gaps --- .../discover-call/PUBLICATION_POLICY.md | 2 +- benchmarks/discover-call/README.md | 49 +++- .../discover-call/adapters/claude-cli.mjs | 8 +- .../discover-call/adapters/codex-cli.mjs | 8 +- .../discover-call/publication-policy.json | 5 +- ...-23-gpt-5.6-sol-configured-v4.summary.json | 11 +- .../2026-07-23-gpt-5.6-sol-v2.summary.json | 9 +- .../2026-07-23-gpt-5.6-sol-v3.summary.json | 11 +- .../2026-07-23-oracle-v1-v3.summary.json | 7 +- .../2026-07-23-reference-v1-v4.summary.json | 7 +- benchmarks/discover-call/src/api.mjs | 38 ++- benchmarks/discover-call/src/harness.mjs | 244 ++++++++++++++++-- benchmarks/discover-call/src/io.mjs | 46 +++- benchmarks/discover-call/src/publication.mjs | 33 ++- benchmarks/discover-call/src/publish.mjs | 47 +--- benchmarks/discover-call/src/run.mjs | 68 +++-- benchmarks/discover-call/src/score.mjs | 13 +- benchmarks/discover-call/src/scoring.mjs | 49 +++- benchmarks/discover-call/src/validate.mjs | 2 +- .../discover-call/test/adapter.test.mjs | 97 ++++++- benchmarks/discover-call/test/api.test.mjs | 91 ++++++- .../discover-call/test/harness.test.mjs | 230 ++++++++++++++++- benchmarks/discover-call/test/io.test.mjs | 25 +- .../discover-call/test/publication.test.mjs | 112 +++++++- .../discover-call/test/scoring.test.mjs | 75 +++++- docs/en-US/discover-call-benchmark.md | 47 ++-- docs/zh-CN/discover-call-benchmark.md | 37 +-- 27 files changed, 1182 insertions(+), 189 deletions(-) diff --git a/benchmarks/discover-call/PUBLICATION_POLICY.md b/benchmarks/discover-call/PUBLICATION_POLICY.md index 37b6186..527aa54 100644 --- a/benchmarks/discover-call/PUBLICATION_POLICY.md +++ b/benchmarks/discover-call/PUBLICATION_POLICY.md @@ -31,7 +31,7 @@ and account-specific availability remain internal. Public artifacts must not contain API keys, OAuth tokens, connection/account identifiers, `search_id`, `execution_id`, credit balances, raw provider errors, -raw tool results, parameter values, private prompts, tenant-private tools, or +`session_id`, raw tool results, parameter values, private prompts, tenant-private tools, or the unfiltered ordered discovery catalog. Inspected parameter names are also omitted because they can disclose the input schema of an unapproved tool. diff --git a/benchmarks/discover-call/README.md b/benchmarks/discover-call/README.md index 445f664..1b2a022 100644 --- a/benchmarks/discover-call/README.md +++ b/benchmarks/discover-call/README.md @@ -52,12 +52,16 @@ The scorer reports these metrics separately: - **workflow success**: all preceding checks equal 100%, the real call succeeds, and its result is non-empty. -The transport retries `429` and `503` responses before recording an API-stage -failure, as well as transient network and response-body timeouts, matching the -public clients' transient-failure behavior. Calls explicitly request -`respond_with: "full"` so structural result scoring does not depend on a -changing server default. The summary reports failures by stage; all exhausted -API failures remain in the strict workflow-success denominator. +Discover and inspect retry `429` and `503` responses, transient network +failures, and response-body timeouts. Execute retries only an explicit `429`; +it does not retry `503`, network errors, or timeouts because the first request +may already have produced third-party side effects or a charge. Calls explicitly +request `respond_with: "full"` so structural result scoring does not depend on +a changing server default. The summary assigns each observed strict failure to +its earliest failed gate and reports normalized stage/reason counts; all +exhausted API failures remain in the strict workflow-success denominator. +Legacy successful calls without result-non-emptiness evidence remain +unreportable rather than being relabeled as failures. The summary includes a deterministic 95% task-cluster bootstrap interval for workflow success. Tasks, rather than individual trials, are resampled so three @@ -101,13 +105,18 @@ Use `--lane pinned-model` only when the provider exposes a verifiable immutable model revision and record it with `--model-revision`. Otherwise use `configured-model` or `current-model` and record `unreported`. The adapter command is spawned directly without shell parsing. -The harness removes `QVERIS_API_KEY` from the adapter environment; adapters -must use separately named credentials for their model provider. +The harness removes every `QVERIS_*` variable from the adapter environment; +the bundled adapters repeat that boundary before starting their model CLI. +Adapters must use separately named credentials for their model provider. +The runner requires an explicit immutable task file and comparison lane. It +records a commit-shaped toolkit revision and refuses to run from a checkout +with tracked changes, including when `--toolkit-revision` is supplied. ```bash node src/run.mjs \ --model provider/model-version \ --lane configured-model \ + --tasks tasks/v4.jsonl \ --adapter node \ --adapter-arg /absolute/path/to/model-adapter.mjs \ --adapter-revision adapter-git-sha-or-config-hash \ @@ -117,7 +126,15 @@ node src/run.mjs \ ``` `--execute` performs billed calls. Omit it while validating an adapter, but do -not publish dry-run records as benchmark results. +not publish dry-run records as benchmark results. Each trial atomically +checkpoints the private output file so an interrupted paid run retains all +records completed before the interruption; a partial checkpoint still fails +official publication completeness checks. + +Within a trial, Discover, Inspect, and Execute share an independent private +`session_id`, and Execute also records the model identifier in the API request. +The session id is not derived from the public run id. Neither operational field +is sent to the model adapter or retained in public artifacts. ### Comparison lanes @@ -136,7 +153,7 @@ Use the lanes together; none is a substitute for the others: - `current-model` measures the currently recommended model under the same task contract. - `model` is retained for backward-compatible records that predate explicit - lanes. + lanes; the v2 runner does not accept it for new runs. The strict benchmark gap is `curated reference route workflow success - model workflow success`. @@ -197,15 +214,19 @@ so model comparisons use the same prompt and output contract. The bundled reference adapter rejects a task-file digest mismatch. Ground-truth scoring constraints are deliberately not exposed to model adapters. -Selection input uses `stage: "select"`; `input` contains the user prompt and the -complete discovery response. Return: +Selection input uses `stage: "select"`; `input` contains the user prompt and +compact routing cards. The operational `search_id` and non-routing discovery +metadata are not sent to the model adapter. Return: ```json {"tool_id":"selected.tool.id"} ``` Parameterization input uses `stage: "parameterize"`; `input` contains the same -user prompt and the inspected `selected_tool`. Return: +user prompt and a least-privilege projection of the inspected `selected_tool` +(`tool_id`, descriptive fields, parameters, one-of requirements, and examples). +The discovery id, billing metadata, runtime statistics, and prior-execution +records are not sent to the model adapter. Return: ```json {"parameters":{"city":"London"}} @@ -245,6 +266,7 @@ alongside the adapter commit in `--adapter-revision`: ```bash node src/run.mjs \ --model gpt-5.6-sol \ + --lane configured-model \ --adapter node \ --adapter-arg "$PWD/adapters/codex-cli.mjs" \ --adapter-revision adapter-git-sha/codex-cli-0.144.1/medium \ @@ -270,6 +292,7 @@ messages. Use a full model identifier rather than an alias: ```bash node src/run.mjs \ --model claude-sonnet-5 \ + --lane configured-model \ --adapter node \ --adapter-arg "$PWD/adapters/claude-cli.mjs" \ --adapter-revision adapter-git-sha \ diff --git a/benchmarks/discover-call/adapters/claude-cli.mjs b/benchmarks/discover-call/adapters/claude-cli.mjs index 91b432b..90e21f1 100644 --- a/benchmarks/discover-call/adapters/claude-cli.mjs +++ b/benchmarks/discover-call/adapters/claude-cli.mjs @@ -83,7 +83,7 @@ export function runClaude(invocation, { outputLimit = 1_000_000, forceKillAfterM } return new Promise((resolve, reject) => { const env = { ...process.env }; - delete env.QVERIS_API_KEY; + stripQverisEnvironment(env); const child = spawn(invocation.command, invocation.args, { shell: false, stdio: ['pipe', 'pipe', 'pipe'], @@ -152,6 +152,12 @@ export function runClaude(invocation, { outputLimit = 1_000_000, forceKillAfterM }); } +function stripQverisEnvironment(env) { + for (const name of Object.keys(env)) { + if (name.startsWith('QVERIS_')) delete env[name]; + } +} + function isObject(value) { return value !== null && typeof value === 'object' && !Array.isArray(value); } diff --git a/benchmarks/discover-call/adapters/codex-cli.mjs b/benchmarks/discover-call/adapters/codex-cli.mjs index d046163..bfb7b7c 100644 --- a/benchmarks/discover-call/adapters/codex-cli.mjs +++ b/benchmarks/discover-call/adapters/codex-cli.mjs @@ -132,7 +132,7 @@ export function runCodex(invocation, { outputLimit = 1_000_000, forceKillAfterMs } return new Promise((resolve, reject) => { const env = { ...process.env }; - delete env.QVERIS_API_KEY; + stripQverisEnvironment(env); const child = spawn(invocation.command, invocation.args, { shell: false, stdio: ['pipe', 'pipe', 'pipe'], @@ -202,6 +202,12 @@ export function runCodex(invocation, { outputLimit = 1_000_000, forceKillAfterMs }); } +function stripQverisEnvironment(env) { + for (const name of Object.keys(env)) { + if (name.startsWith('QVERIS_')) delete env[name]; + } +} + function isObject(value) { return value !== null && typeof value === 'object' && !Array.isArray(value); } diff --git a/benchmarks/discover-call/publication-policy.json b/benchmarks/discover-call/publication-policy.json index 9f5eb13..709cb9e 100644 --- a/benchmarks/discover-call/publication-policy.json +++ b/benchmarks/discover-call/publication-policy.json @@ -8,9 +8,7 @@ "oracle": "reference", "pinned-model": "configured-model" }, - "approved_api_base_urls": [ - "https://qveris.ai/api/v1" - ], + "approved_api_base_urls": ["https://qveris.ai/api/v1"], "approved_selected_tool_ids": [ "amap_webservice.geocode.geo.retrieve.v3", "api-football.standings.list.v3.76f7703e", @@ -43,6 +41,7 @@ "forbidden_fields": [ "execution_id", "search_id", + "session_id", "connection_id", "remaining_credits", "result_tool_ids", diff --git a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.summary.json b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.summary.json index a741e3e..d6a1461 100644 --- a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.summary.json +++ b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-configured-v4.summary.json @@ -1,7 +1,7 @@ { "schema_version": 1, "methodology": "discover-call-v1", - "generated_at": "2026-07-23T12:14:22.053Z", + "generated_at": "2026-07-23T15:53:48.739Z", "models": [ { "model": "gpt-5.6-sol", @@ -19,14 +19,15 @@ "result_nonempty_rate": 0.8824, "pre_result_gate_workflow_success_rate": 0.7778, "workflow_success_rate": 0.7778, - "workflow_success_task_cluster_bootstrap_95": [ - 0.5556, - 0.9444 - ], + "workflow_success_task_cluster_bootstrap_95": [0.5556, 0.9444], "failures_by_stage": { + "parameterize": 3, + "call": 6, "adapter": 3 }, "failures_by_reason": { + "constraint_mismatch": 3, + "tool_returned_failure": 6, "tool_use_rejected": 3 } } diff --git a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.summary.json b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.summary.json index e6f0ef9..20c2099 100644 --- a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.summary.json +++ b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v2.summary.json @@ -1,7 +1,7 @@ { "schema_version": 1, "methodology": "discover-call-v1", - "generated_at": "2026-07-23T12:14:22.012Z", + "generated_at": "2026-07-23T15:53:48.695Z", "models": [ { "model": "gpt-5.6-sol", @@ -21,9 +21,14 @@ "workflow_success_rate": null, "workflow_success_task_cluster_bootstrap_95": null, "failures_by_stage": { + "parameterize": 9, + "call": 6, "adapter": 4 }, - "failures_by_reason": {} + "failures_by_reason": { + "constraint_mismatch": 9, + "tool_returned_failure": 6 + } } ], "records": [ diff --git a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.summary.json b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.summary.json index 8f3e7ea..743448c 100644 --- a/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.summary.json +++ b/benchmarks/discover-call/results/2026-07-23-gpt-5.6-sol-v3.summary.json @@ -1,7 +1,7 @@ { "schema_version": 1, "methodology": "discover-call-v1", - "generated_at": "2026-07-23T12:14:22.020Z", + "generated_at": "2026-07-23T15:53:48.716Z", "models": [ { "model": "gpt-5.6-sol", @@ -19,14 +19,15 @@ "result_nonempty_rate": 0.8846, "pre_result_gate_workflow_success_rate": 0.7222, "workflow_success_rate": 0.7222, - "workflow_success_task_cluster_bootstrap_95": [ - 0.5, - 0.8889 - ], + "workflow_success_task_cluster_bootstrap_95": [0.5, 0.8889], "failures_by_stage": { + "parameterize": 7, + "call": 6, "adapter": 2 }, "failures_by_reason": { + "constraint_mismatch": 7, + "tool_returned_failure": 6, "tool_use_rejected": 2 } } diff --git a/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.summary.json b/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.summary.json index b191d8b..9e8bb29 100644 --- a/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.summary.json +++ b/benchmarks/discover-call/results/2026-07-23-oracle-v1-v3.summary.json @@ -1,7 +1,7 @@ { "schema_version": 1, "methodology": "discover-call-v1", - "generated_at": "2026-07-23T12:14:22.036Z", + "generated_at": "2026-07-23T15:53:48.696Z", "models": [ { "model": "oracle-v1", @@ -19,10 +19,7 @@ "result_nonempty_rate": 1, "pre_result_gate_workflow_success_rate": 0.9444, "workflow_success_rate": 0.9444, - "workflow_success_task_cluster_bootstrap_95": [ - 0.8333, - 1 - ], + "workflow_success_task_cluster_bootstrap_95": [0.8333, 1], "failures_by_stage": { "select": 3 }, diff --git a/benchmarks/discover-call/results/2026-07-23-reference-v1-v4.summary.json b/benchmarks/discover-call/results/2026-07-23-reference-v1-v4.summary.json index 9ec48e1..a5f3806 100644 --- a/benchmarks/discover-call/results/2026-07-23-reference-v1-v4.summary.json +++ b/benchmarks/discover-call/results/2026-07-23-reference-v1-v4.summary.json @@ -1,7 +1,7 @@ { "schema_version": 1, "methodology": "discover-call-v1", - "generated_at": "2026-07-23T12:14:22.009Z", + "generated_at": "2026-07-23T15:53:48.692Z", "models": [ { "model": "reference-v1", @@ -19,10 +19,7 @@ "result_nonempty_rate": 1, "pre_result_gate_workflow_success_rate": 0.9444, "workflow_success_rate": 0.9444, - "workflow_success_task_cluster_bootstrap_95": [ - 0.8333, - 1 - ], + "workflow_success_task_cluster_bootstrap_95": [0.8333, 1], "failures_by_stage": { "select": 3 }, diff --git a/benchmarks/discover-call/src/api.mjs b/benchmarks/discover-call/src/api.mjs index 53c5bba..d5ab729 100644 --- a/benchmarks/discover-call/src/api.mjs +++ b/benchmarks/discover-call/src/api.mjs @@ -1,5 +1,6 @@ const DEFAULT_BASE_URL = 'https://qveris.ai/api/v1'; const RETRYABLE_STATUS = new Set([429, 503]); +const LOOPBACK_HOSTS = new Set(['localhost', '127.0.0.1', '[::1]']); export function createApiClient({ apiKey, @@ -21,7 +22,7 @@ export function createApiClient({ throw new Error('timeoutMs must be a positive integer'); } - async function request(path, body) { + async function request(path, body, { retryAmbiguousFailures = true, retryableStatuses = RETRYABLE_STATUS } = {}) { for (let attempt = 0; ; attempt++) { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), timeoutMs); @@ -34,7 +35,7 @@ export function createApiClient({ signal: controller.signal, redirect: 'error', }); - if (RETRYABLE_STATUS.has(response.status) && attempt < maxRetries) { + if (retryableStatuses.has(response.status) && attempt < maxRetries) { await Promise.resolve(response.body?.cancel?.()).catch(() => undefined); retryDelay = retryDelayMs(response.headers.get('retry-after'), attempt); } else { @@ -74,7 +75,7 @@ export function createApiClient({ } } catch (error) { if (error?.benchmarkStage === 'api') throw error; - if (attempt >= maxRetries) { + if (attempt >= maxRetries || !retryAmbiguousFailures) { if (controller.signal.aborted) { throw apiError( error?.apiTimeoutPhase === 'response' ? 'API response timed out' : 'API request timed out', @@ -93,18 +94,30 @@ export function createApiClient({ return { baseUrl: resolvedBaseUrl, - discover: ({ query, limit }) => request('/search', { query, limit }), - inspect: ({ toolIds, discoveryId }) => + discover: ({ query, limit, sessionId }) => + request('/search', { query, limit, session_id: sessionId, view: 'routing' }), + inspect: ({ toolIds, discoveryId, sessionId }) => request('/tools/by-ids', { tool_ids: toolIds, ...(discoveryId ? { search_id: discoveryId } : {}), + session_id: sessionId, + view: 'lean', }), - call: ({ toolId, discoveryId, parameters }) => - request(`/tools/execute?tool_id=${encodeURIComponent(toolId)}`, { - parameters, - search_id: discoveryId ?? null, - respond_with: 'full', - }), + call: ({ toolId, discoveryId, sessionId, model, parameters }) => + request( + `/tools/execute?tool_id=${encodeURIComponent(toolId)}`, + { + parameters, + search_id: discoveryId ?? null, + session_id: sessionId, + model, + respond_with: 'full', + }, + { + retryAmbiguousFailures: false, + retryableStatuses: new Set([429]), + }, + ), observedRevisions: () => ({ api_revision: singleRevision(observedApiRevisions), catalog_revision: singleRevision(observedCatalogRevisions), @@ -151,6 +164,9 @@ export function normalizeBaseUrl(value) { if (!['http:', 'https:'].includes(parsed.protocol) || !parsed.hostname || parsed.username || parsed.password) { throw new Error('QVERIS_BASE_URL must be a valid HTTP(S) URL without credentials'); } + if (parsed.protocol !== 'https:' && !LOOPBACK_HOSTS.has(parsed.hostname)) { + throw new Error('QVERIS_BASE_URL must use HTTPS except for loopback development'); + } if (candidate.includes('?') || candidate.includes('#')) { throw new Error('QVERIS_BASE_URL must not contain a query or fragment'); } diff --git a/benchmarks/discover-call/src/harness.mjs b/benchmarks/discover-call/src/harness.mjs index ef0a7bf..09efa5b 100644 --- a/benchmarks/discover-call/src/harness.mjs +++ b/benchmarks/discover-call/src/harness.mjs @@ -18,16 +18,21 @@ export function validateTaskSet(tasks) { export function validateTask(task) { if (!task || typeof task !== 'object') throw new Error('Task must be an object'); - if (typeof task.id !== 'string' || !task.id.trim()) throw new Error('Task id must be a non-empty string'); + if (typeof task.id !== 'string' || !task.id.trim() || task.id.length > 256 || /[\p{Cc}\p{Cf}]/u.test(task.id)) { + throw new Error('Task id must be a safe non-empty string'); + } if (typeof task.prompt !== 'string' || !task.prompt.trim()) { throw new Error(`Task ${task.id}: prompt must be a non-empty string`); } + if (task.discover_query !== undefined && (typeof task.discover_query !== 'string' || !task.discover_query.trim())) { + throw new Error(`Task ${task.id}: discover_query must be a non-empty string`); + } if (!Array.isArray(task.constraints) || task.constraints.length === 0) { throw new Error(`Task ${task.id}: constraints must be a non-empty array`); } const constraintIds = new Set(); for (const constraint of task.constraints) { - if (typeof constraint.id !== 'string' || !constraint.id) { + if (typeof constraint.id !== 'string' || !constraint.id.trim()) { throw new Error(`Task ${task.id}: every constraint needs an id`); } if (constraintIds.has(constraint.id)) { @@ -37,12 +42,14 @@ export function validateTask(task) { if ( !Array.isArray(constraint.aliases) || constraint.aliases.length === 0 || - constraint.aliases.some((alias) => typeof alias !== 'string' || !alias) + constraint.aliases.some((alias) => typeof alias !== 'string' || !alias.trim()) || + new Set(constraint.aliases).size !== constraint.aliases.length ) { throw new Error(`Task ${task.id}: constraint ${constraint.id} needs string aliases`); } if ( !['string', 'number', 'boolean'].includes(typeof constraint.value) || + (typeof constraint.value === 'string' && !constraint.value.trim()) || (typeof constraint.value === 'number' && !Number.isFinite(constraint.value)) ) { throw new Error(`Task ${task.id}: constraint ${constraint.id} needs a primitive finite value`); @@ -53,7 +60,8 @@ export function validateTask(task) { if ( constraint.composite_aliases !== undefined && (!Array.isArray(constraint.composite_aliases) || - constraint.composite_aliases.some((alias) => typeof alias !== 'string' || !alias)) + constraint.composite_aliases.some((alias) => typeof alias !== 'string' || !alias.trim()) || + new Set(constraint.composite_aliases).size !== constraint.composite_aliases.length) ) { throw new Error(`Task ${task.id}: constraint ${constraint.id} has invalid composite_aliases`); } @@ -76,6 +84,7 @@ export function validateTask(task) { values.some( (value) => !['string', 'number', 'boolean'].includes(typeof value) || + (typeof value === 'string' && !value.trim()) || (typeof value === 'number' && !Number.isFinite(value)), ) ) { @@ -99,10 +108,18 @@ export function validateTask(task) { throw new Error(`Task ${task.id}: an empty reference needs unavailable_reason`); } for (const candidate of reference.candidates) { - if (typeof candidate?.tool_id !== 'string' || !candidate.tool_id || !isPlainObject(candidate.parameters)) { + if ( + typeof candidate?.tool_id !== 'string' || + !candidate.tool_id.trim() || + !isPlainObject(candidate.parameters) || + !isJsonValue(candidate.parameters) + ) { throw new Error(`Task ${task.id}: every reference candidate needs tool_id and parameters`); } } + if (new Set(reference.candidates.map((candidate) => candidate.tool_id)).size !== reference.candidates.length) { + throw new Error(`Task ${task.id}: reference candidates need unique tool ids`); + } } return task; } @@ -118,11 +135,22 @@ export async function runBenchmark({ metadata = {}, now = () => new Date().toISOString(), newRunId = randomUUID, + newSessionId = randomUUID, + onRecord = async () => {}, }) { validateTaskSet(tasks); if (typeof model !== 'string' || !model.trim()) throw new Error('A model identifier is required'); if (!Number.isInteger(trials) || trials < 1 || trials > 100) throw new Error('trials must be an integer from 1 to 100'); + if (!Number.isInteger(limit) || limit < 1 || limit > 100) { + throw new Error('limit must be an integer from 1 to 100'); + } + if (typeof execute !== 'boolean') throw new Error('execute must be a boolean'); + if (!isPlainObject(metadata)) throw new Error('metadata must be an object'); + if (typeof now !== 'function') throw new Error('now must be a function'); + if (typeof newRunId !== 'function') throw new Error('newRunId must be a function'); + if (typeof newSessionId !== 'function') throw new Error('newSessionId must be a function'); + if (typeof onRecord !== 'function') throw new Error('onRecord must be a function'); if ( !api || typeof api.discover !== 'function' || @@ -133,20 +161,47 @@ export async function runBenchmark({ } if (typeof invokeAdapter !== 'function') throw new Error('invokeAdapter must be a function'); - const records = []; + const runPlan = []; + const runIds = new Set(); + const sessionIds = new Set(); for (const task of tasks) { for (let trial = 1; trial <= trials; trial++) { - records.push(await runTrial({ task, model, trial, execute, limit, api, invokeAdapter, metadata, now, newRunId })); + const runId = newRunId(); + if (typeof runId !== 'string' || !runId.trim() || runId.length > 256 || /[\p{Cc}\p{Cf}]/u.test(runId)) { + throw new Error('newRunId must return a safe non-empty string'); + } + if (runIds.has(runId)) throw new Error(`Duplicate benchmark run id: ${runId}`); + const sessionId = newSessionId(); + if ( + typeof sessionId !== 'string' || + !sessionId.trim() || + sessionId.length > 256 || + /[\p{Cc}\p{Cf}]/u.test(sessionId) + ) { + throw new Error('newSessionId must return a safe non-empty string'); + } + if (sessionIds.has(sessionId)) throw new Error('Duplicate benchmark session id'); + runIds.add(runId); + sessionIds.add(sessionId); + runPlan.push({ task, trial, runId, sessionId }); } } + + const records = []; + for (const { task, trial, runId, sessionId } of runPlan) { + records.push( + await runTrial({ task, model, trial, runId, sessionId, execute, limit, api, invokeAdapter, metadata, now }), + ); + await onRecord(records); + } return records; } -async function runTrial({ task, model, trial, execute, limit, api, invokeAdapter, metadata, now, newRunId }) { +async function runTrial({ task, model, trial, runId, sessionId, execute, limit, api, invokeAdapter, metadata, now }) { const record = { schema_version: SCHEMA_VERSION, benchmark_version: BENCHMARK_VERSION, - run_id: newRunId(), + run_id: runId, model, metadata, task_id: task.id, @@ -161,7 +216,12 @@ async function runTrial({ task, model, trial, execute, limit, api, invokeAdapter }; try { - const discovered = await api.discover({ query: task.discover_query || task.prompt, limit }); + const discovered = await api.discover({ query: task.discover_query || task.prompt, limit, sessionId }); + const discoveryId = + typeof discovered?.search_id === 'string' && discovered.search_id.trim() ? discovered.search_id.trim() : null; + if (!discoveryId) { + throw stageError('discover', 'Discover did not return a search_id', 'missing_search_id'); + } const results = array(discovered?.results); record.discovery.result_tool_ids = results.map(toolId).filter(Boolean); @@ -170,7 +230,7 @@ async function runTrial({ task, model, trial, execute, limit, api, invokeAdapter stage: 'select', model, task, - discovery: discovered, + discovery: adapterDiscovery(discovered), trial, taskSetSha256: metadata.task_set_sha256, }), @@ -182,7 +242,7 @@ async function runTrial({ task, model, trial, execute, limit, api, invokeAdapter throw stageError('select', 'Adapter selected a tool outside the discovery results', 'ungrounded_tool_id'); } - const inspected = await api.inspect({ toolIds: [selectedToolId], discoveryId: discovered?.search_id }); + const inspected = await api.inspect({ toolIds: [selectedToolId], discoveryId, sessionId }); const inspectedResults = array(inspected?.results); record.inspection.returned_tool_ids = inspectedResults.map(toolId).filter(Boolean); const selectedTool = inspectedResults.find((item) => toolId(item) === selectedToolId); @@ -196,8 +256,7 @@ async function runTrial({ task, model, trial, execute, limit, api, invokeAdapter stage: 'parameterize', model, task, - selectedTool, - discoveryId: discovered?.search_id ?? null, + selectedTool: adapterTool(selectedTool), trial, taskSetSha256: metadata.task_set_sha256, }), @@ -206,12 +265,15 @@ async function runTrial({ task, model, trial, execute, limit, api, invokeAdapter throw stageError('parameterize', 'Adapter did not return a parameters object'); } record.parameters = omitNullParameters(parameterized.parameters); + validateAdapterParameters(selectedTool, record.parameters); if (execute) { record.call.attempted = true; const response = await api.call({ toolId: selectedToolId, - discoveryId: discovered?.search_id, + discoveryId, + sessionId, + model, parameters: record.parameters, }); record.call.success = response?.success === true; @@ -231,9 +293,125 @@ async function runTrial({ task, model, trial, execute, limit, api, invokeAdapter } function requiredParameterNames(tool) { - return array(tool?.params) + const names = array(tool?.params) .filter((parameter) => parameter?.required === true && typeof parameter?.name === 'string') .map((parameter) => parameter.name); + return [...names, ...oneOfRequiredGroups(tool)]; +} + +function oneOfRequiredGroups(tool) { + const value = tool?.one_of_required; + if (value === undefined || value === null) return []; + if (Array.isArray(value) && value.length === 0) return []; + const groups = + Array.isArray(value) && value.every((name) => typeof name === 'string') + ? [value] + : Array.isArray(value) + ? value + : null; + const parameterNames = new Set(array(tool?.params).map((parameter) => parameter?.name)); + if ( + !groups || + groups.length === 0 || + groups.some( + (group) => + !Array.isArray(group) || + group.length === 0 || + group.some((name) => typeof name !== 'string' || !name.trim() || !parameterNames.has(name)), + ) + ) { + throw stageError( + 'parameterize', + 'Inspected tool contains invalid one_of_required metadata', + 'unsupported_parameter_schema', + ); + } + return groups.map((group) => [...new Set(group)]); +} + +function validateAdapterParameters(tool, parameters) { + const definitions = new Map(array(tool?.params).map((parameter) => [parameter.name, parameter])); + for (const [name, value] of Object.entries(parameters)) { + const definition = definitions.get(name); + if (!definition) { + throw stageError('parameterize', `Adapter returned unknown parameter ${name}`, 'invalid_parameter_values'); + } + validateParameterValue(value, definition); + } + for (const definition of definitions.values()) { + if (definition.required === true && !hasParameterValue(parameters[definition.name])) { + throw stageError( + 'parameterize', + `Adapter omitted required parameter ${definition.name}`, + 'invalid_parameter_values', + ); + } + } + for (const group of oneOfRequiredGroups(tool)) { + if (!group.some((name) => hasParameterValue(parameters[name]))) { + throw stageError( + 'parameterize', + `Adapter omitted one_of_required group ${group.join(',')}`, + 'invalid_parameter_values', + ); + } + } +} + +function validateParameterValue(value, parameter, forceRequired = parameter?.required === true) { + if (value === null) { + if (!forceRequired) return; + throw stageError('parameterize', 'Adapter returned null for a required parameter', 'invalid_parameter_values'); + } + if (Array.isArray(parameter?.enum) && parameter.enum.length > 0) { + if (!parameter.enum.includes(value)) { + throw stageError( + 'parameterize', + 'Adapter returned a value outside the parameter enum', + 'invalid_parameter_values', + ); + } + return; + } + const type = normalizeParameterType(parameter?.type); + const valid = + type === 'string' + ? typeof value === 'string' + : type === 'integer' + ? Number.isInteger(value) + : type === 'number' + ? typeof value === 'number' && Number.isFinite(value) + : type === 'boolean' + ? typeof value === 'boolean' + : type === 'array' + ? Array.isArray(value) + : isPlainObject(value); + if (!valid) { + throw stageError('parameterize', `Adapter returned an invalid ${type} value`, 'invalid_parameter_values'); + } + if (type === 'array') { + if (!isPlainObject(parameter?.items)) { + throw stageError('parameterize', 'Adapter returned an opaque array parameter', 'invalid_parameter_values'); + } + value.forEach((item) => validateParameterValue(item, parameter.items, true)); + } + if (type === 'object') { + const properties = isPlainObject(parameter?.properties) ? parameter.properties : {}; + if (Object.keys(properties).length === 0) { + throw stageError('parameterize', 'Adapter returned an opaque object parameter', 'invalid_parameter_values'); + } + for (const [name, child] of Object.entries(value)) { + if (!Object.hasOwn(properties, name)) { + throw stageError('parameterize', 'Adapter returned an unknown nested parameter', 'invalid_parameter_values'); + } + validateParameterValue(child, properties[name]); + } + for (const [name, child] of Object.entries(properties)) { + if (child?.required === true && !hasParameterValue(value[name])) { + throw stageError('parameterize', 'Adapter omitted a required nested parameter', 'invalid_parameter_values'); + } + } + } } export function parameterResponseSchema(tool) { @@ -268,11 +446,11 @@ export function parameterResponseSchema(tool) { }; } -function adapterPayload({ stage, model, task, discovery, selectedTool, discoveryId, trial, taskSetSha256 }) { +function adapterPayload({ stage, model, task, discovery, selectedTool, trial, taskSetSha256 }) { const input = stage === 'select' ? { task_id: task.id, prompt: task.prompt, discovery } - : { task_id: task.id, prompt: task.prompt, selected_tool: selectedTool ?? null, discovery_id: discoveryId }; + : { task_id: task.id, prompt: task.prompt, selected_tool: selectedTool ?? null }; const responseSchema = stage === 'select' ? { @@ -301,6 +479,24 @@ function adapterPayload({ stage, model, task, discovery, selectedTool, discovery }; } +function adapterDiscovery(discovery) { + return { + results: array(discovery?.results).map((tool) => + pickDefined(tool, ['tool_id', 'capability', 'cost_class', 'reliability', 'as_of_support']), + ), + }; +} + +function adapterTool(tool) { + return pickDefined(tool, ['tool_id', 'name', 'description', 'params', 'one_of_required', 'examples']); +} + +function pickDefined(value, fields) { + return Object.fromEntries( + fields.filter((field) => value?.[field] !== undefined).map((field) => [field, structuredClone(value[field])]), + ); +} + function strictParameterSchema(parameter, forceRequired = parameter?.required === true) { const type = normalizeParameterType(parameter?.type); const schema = {}; @@ -407,6 +603,10 @@ function omitNullParameters(parameters) { return Object.fromEntries(Object.entries(parameters).filter(([, value]) => value !== null)); } +function hasParameterValue(value) { + return value !== undefined && value !== null && value !== ''; +} + function hasNonemptyResult(result) { if (result === null || result === undefined) return false; if (typeof result === 'string') return result.trim().length > 0; @@ -433,6 +633,14 @@ function isPlainObject(value) { return value !== null && typeof value === 'object' && !Array.isArray(value); } +function isJsonValue(value) { + if (value === null || typeof value === 'string' || typeof value === 'boolean') return true; + if (typeof value === 'number') return Number.isFinite(value); + if (Array.isArray(value)) return value.every(isJsonValue); + if (isPlainObject(value)) return Object.values(value).every(isJsonValue); + return false; +} + function stageError(stage, message, reason) { const error = new Error(message); error.benchmarkStage = stage; diff --git a/benchmarks/discover-call/src/io.mjs b/benchmarks/discover-call/src/io.mjs index 071cf60..9ec97de 100644 --- a/benchmarks/discover-call/src/io.mjs +++ b/benchmarks/discover-call/src/io.mjs @@ -1,5 +1,6 @@ import { randomUUID } from 'node:crypto'; -import { open, readFile, rename, unlink } from 'node:fs/promises'; +import { open, readFile, realpath, rename, stat, unlink } from 'node:fs/promises'; +import { basename, dirname, join, resolve } from 'node:path'; export async function readJsonLines(path) { const text = await readFile(path, 'utf8'); @@ -37,3 +38,46 @@ export async function writeTextAtomic(path, content) { throw error; } } + +export async function validatePathSeparation({ inputs, outputs }) { + if (!Array.isArray(inputs) || !Array.isArray(outputs) || outputs.length === 0) { + throw new Error('Path validation needs input and output arrays'); + } + const inputIdentities = await Promise.all(inputs.map(fileIdentities)); + const outputIdentities = await Promise.all(outputs.map(fileIdentities)); + for (let left = 0; left < outputIdentities.length; left++) { + for (let right = left + 1; right < outputIdentities.length; right++) { + if (identitiesOverlap(outputIdentities[left], outputIdentities[right])) { + throw new Error('Output files must use different files'); + } + } + } + for (const output of outputIdentities) { + if (inputIdentities.some((input) => identitiesOverlap(input, output))) { + throw new Error('Output files must not overwrite input files'); + } + } +} + +async function fileIdentities(path) { + const absolute = resolve(path); + const identities = new Set([`path:${absolute}`]); + try { + const canonical = await realpath(absolute); + const info = await stat(absolute); + identities.add(`path:${canonical}`); + identities.add(`inode:${info.dev}:${info.ino}`); + } catch (error) { + if (error?.code !== 'ENOENT') throw error; + try { + identities.add(`path:${join(await realpath(dirname(absolute)), basename(absolute))}`); + } catch (parentError) { + if (parentError?.code !== 'ENOENT') throw parentError; + } + } + return identities; +} + +function identitiesOverlap(left, right) { + return [...left].some((identity) => right.has(identity)); +} diff --git a/benchmarks/discover-call/src/publication.mjs b/benchmarks/discover-call/src/publication.mjs index 6f87061..78ee3a8 100644 --- a/benchmarks/discover-call/src/publication.mjs +++ b/benchmarks/discover-call/src/publication.mjs @@ -53,6 +53,7 @@ const PUBLIC_ERROR_FIELDS = new Set(['stage', 'reason_code']); const REQUIRED_FORBIDDEN_FIELDS = [ 'execution_id', 'search_id', + 'session_id', 'connection_id', 'remaining_credits', 'result_tool_ids', @@ -224,6 +225,7 @@ export function validatePublicRecords(records, policy) { !safePublicString(record.task_id, 256) || !Number.isInteger(record.trial) || record.trial < 1 || + record.trial > 100 || !['completed', 'failed'].includes(record.status) || !validTimestamp(record.started_at) || !validTimestamp(record.finished_at) @@ -258,6 +260,7 @@ export function validatePublicRecords(records, policy) { !isSha256(record.metadata.catalog_observation_sha256) || !Number.isInteger(record.metadata.discovery_limit) || record.metadata.discovery_limit < 1 || + record.metadata.discovery_limit > 100 || typeof record.metadata.execute !== 'boolean' ) { throw new Error('Public artifact has invalid reproducibility metadata'); @@ -275,6 +278,9 @@ export function validatePublicRecords(records, policy) { ) { throw new Error('Public artifact has invalid public metadata values'); } + if (record.metadata.lane === 'pinned-model' && record.metadata.model_revision === 'unreported') { + throw new Error('Public pinned-model artifact needs an immutable model revision'); + } if (selectedToolId && !approvedToolIds.has(selectedToolId)) { throw new Error(`Public artifact contains an unapproved selected tool: ${selectedToolId}`); } @@ -290,6 +296,8 @@ export function validatePublicRecords(records, policy) { if ( !Number.isInteger(record.discovery?.result_count) || record.discovery.result_count < 0 || + record.discovery.result_count > record.metadata.discovery_limit || + (record.discovery.result_count === 0 && record.discovery.snapshot_sha256 !== sha256('[]')) || !isSha256(record.discovery?.snapshot_sha256) ) { throw new Error('Public artifact discovery needs a count and SHA-256 snapshot'); @@ -358,6 +366,23 @@ export function validateOfficialPublicRun(records, { taskSetSha256, minTrialsPer if (record.metadata?.execute !== true) { throw new Error(`Official public artifact must execute every trial: ${record.task_id}`); } + if (record.benchmark_version === 'v2') { + if (record.metadata?.lane === 'model') { + throw new Error('Official benchmark v2 artifacts need an explicit comparison lane'); + } + if ( + ['reference', 'pinned-model'].includes(record.metadata?.lane) && + record.metadata?.model_revision === 'unreported' + ) { + throw new Error(`Official ${record.metadata.lane} artifact needs an immutable model revision`); + } + if (!/^[a-f0-9]{40}(?:[a-f0-9]{24})?$/.test(record.metadata?.toolkit_revision ?? '')) { + throw new Error('Official benchmark v2 artifact needs a toolkit commit SHA'); + } + if (record.metadata?.adapter_revision === 'unreported') { + throw new Error('Official benchmark v2 artifact needs an adapter revision'); + } + } const key = `${record.model}\0${record.task_id}`; trials.set(key, (trials.get(key) ?? 0) + 1); } @@ -370,7 +395,13 @@ export function validateOfficialPublicRun(records, { taskSetSha256, minTrialsPer } function publicMetadata(metadata = {}, policy, catalogObservationSha256) { - const lane = policy.legacy_lane_rewrites?.[metadata.lane] ?? metadata.lane ?? 'model'; + const hasPinnedRevision = + metadata.lane === 'pinned-model' && + typeof metadata.model_revision === 'string' && + metadata.model_revision !== 'unreported'; + const lane = hasPinnedRevision + ? 'pinned-model' + : (policy.legacy_lane_rewrites?.[metadata.lane] ?? metadata.lane ?? 'model'); if (!policy.approved_api_base_urls.includes(metadata.api_base_url)) { throw new Error('Raw benchmark record uses an unapproved public API base URL'); } diff --git a/benchmarks/discover-call/src/publish.mjs b/benchmarks/discover-call/src/publish.mjs index 1909f0f..5b6a689 100644 --- a/benchmarks/discover-call/src/publish.mjs +++ b/benchmarks/discover-call/src/publish.mjs @@ -1,11 +1,11 @@ #!/usr/bin/env node import { createHash } from 'node:crypto'; -import { mkdir, readFile, realpath, stat } from 'node:fs/promises'; -import { basename, dirname, join, resolve } from 'node:path'; +import { mkdir, readFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { readJsonLines, writeJsonLines, writeTextAtomic } from './io.mjs'; +import { readJsonLines, validatePathSeparation, writeJsonLines, writeTextAtomic } from './io.mjs'; import { sanitizePublicRecords, validateOfficialPublicRun, validatePublicRecords } from './publication.mjs'; import { scoreRecords } from './scoring.mjs'; @@ -28,12 +28,13 @@ export async function main(argv = process.argv.slice(2)) { readJsonLines(runsPath), readFile(policyPath, 'utf8').then(JSON.parse), ]); + const taskSetSha256 = createHash('sha256').update(taskBytes).digest('hex'); const publicRecords = sanitizePublicRecords(records, policy, tasks); validatePublicRecords(publicRecords, policy); validateOfficialPublicRun(publicRecords, { - taskSetSha256: createHash('sha256').update(taskBytes).digest('hex'), + taskSetSha256, }); - const summary = scoreRecords(tasks, publicRecords); + const summary = scoreRecords(tasks, publicRecords, { taskSetSha256 }); await Promise.all([ mkdir(dirname(outputRuns), { recursive: true }), mkdir(dirname(outputSummary), { recursive: true }), @@ -44,39 +45,17 @@ export async function main(argv = process.argv.slice(2)) { } export async function validatePublicationPaths({ inputs, outputs }) { - const inputIdentities = await Promise.all(inputs.map(fileIdentities)); - const outputIdentities = await Promise.all(outputs.map(fileIdentities)); - if (identitiesOverlap(outputIdentities[0], outputIdentities[1])) { - throw new Error('--output-runs and --output-summary must use different files'); - } - for (const output of outputIdentities) { - if (inputIdentities.some((input) => identitiesOverlap(input, output))) { - throw new Error('Publication output files must not overwrite task, run, or policy inputs'); - } - } -} - -async function fileIdentities(path) { - const absolute = resolve(path); - const identities = new Set([`path:${absolute}`]); try { - const canonical = await realpath(absolute); - const info = await stat(absolute); - identities.add(`path:${canonical}`); - identities.add(`inode:${info.dev}:${info.ino}`); + await validatePathSeparation({ inputs, outputs }); } catch (error) { - if (error?.code !== 'ENOENT') throw error; - try { - identities.add(`path:${join(await realpath(dirname(absolute)), basename(absolute))}`); - } catch (parentError) { - if (parentError?.code !== 'ENOENT') throw parentError; + if (error?.message === 'Output files must use different files') { + throw new Error('--output-runs and --output-summary must use different files'); + } + if (error?.message === 'Output files must not overwrite input files') { + throw new Error('Publication output files must not overwrite task, run, or policy inputs'); } + throw error; } - return identities; -} - -function identitiesOverlap(left, right) { - return [...left].some((identity) => right.has(identity)); } function parseArgs(argv) { diff --git a/benchmarks/discover-call/src/run.mjs b/benchmarks/discover-call/src/run.mjs index 1cac0a1..9b10f7e 100644 --- a/benchmarks/discover-call/src/run.mjs +++ b/benchmarks/discover-call/src/run.mjs @@ -8,7 +8,7 @@ import { fileURLToPath } from 'node:url'; import { createApiClient } from './api.mjs'; import { runBenchmark } from './harness.mjs'; -import { readJsonLines, writeJsonLines } from './io.mjs'; +import { readJsonLines, validatePathSeparation, writeJsonLines } from './io.mjs'; const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..'); @@ -20,6 +20,8 @@ export async function main(argv = process.argv.slice(2)) { } const tasksPath = resolve(options.tasks); + const output = resolve(options.output); + await validatePathSeparation({ inputs: [tasksPath], outputs: [output] }); const tasks = await readJsonLines(tasksPath); const taskSetSha256 = createHash('sha256') .update(await readFile(tasksPath)) @@ -31,7 +33,6 @@ export async function main(argv = process.argv.slice(2)) { args: options.adapterArgs, timeoutMs: options.adapterTimeoutMs, }); - const output = resolve(options.output); await mkdir(dirname(output), { recursive: true }); const metadata = { lane: options.lane, @@ -60,6 +61,7 @@ export async function main(argv = process.argv.slice(2)) { api, invokeAdapter, metadata, + onRecord: (records) => writeJsonLines(output, records), }); Object.assign(metadata, api.observedRevisions?.() ?? { api_revision: 'unreported', catalog_revision: 'unreported' }); metadata.catalog_observation_sha256 = catalogObservationSha256(records); @@ -73,7 +75,7 @@ export function createProcessAdapter({ command, args = [], timeoutMs = 120_000, throw new Error('forceKillAfterMs must be an integer from 1 to 10000'); } const adapterEnv = { ...process.env }; - delete adapterEnv.QVERIS_API_KEY; + stripQverisEnvironment(adapterEnv); return (payload) => new Promise((resolvePromise, reject) => { const child = spawn(command, args, { @@ -151,14 +153,18 @@ export function createProcessAdapter({ command, args = [], timeoutMs = 120_000, }); } +function stripQverisEnvironment(env) { + for (const name of Object.keys(env)) { + if (name.startsWith('QVERIS_')) delete env[name]; + } +} + function parseArgs(argv) { const options = { - tasks: resolve(ROOT, 'tasks/v1.jsonl'), output: resolve(process.cwd(), 'benchmark-runs.jsonl'), trials: 3, execute: false, limit: 10, - lane: 'model', modelRevision: 'unreported', adapterArgs: [], adapterTimeoutMs: 120_000, @@ -184,8 +190,10 @@ function parseArgs(argv) { if (!options.help && !options.model) throw new Error('--model is required'); if (!options.help && !options.adapter) throw new Error('--adapter is required'); if (!options.help && !options.adapterRevision) throw new Error('--adapter-revision is required'); - if (!options.help && options.lane === 'pinned-model' && options.modelRevision === 'unreported') { - throw new Error('--model-revision is required for --lane pinned-model'); + if (!options.help && !options.tasks) throw new Error('--tasks is required'); + if (!options.help && !options.lane) throw new Error('--lane is required'); + if (!options.help && ['reference', 'pinned-model'].includes(options.lane) && options.modelRevision === 'unreported') { + throw new Error(`--model-revision is required for --lane ${options.lane}`); } return options; } @@ -203,8 +211,8 @@ function integer(value, flag) { } function lane(value) { - if (!['model', 'oracle', 'reference', 'configured-model', 'pinned-model', 'current-model'].includes(value)) { - throw new Error('--lane must be model, oracle, reference, configured-model, pinned-model, or current-model'); + if (!['reference', 'configured-model', 'pinned-model', 'current-model'].includes(value)) { + throw new Error('--lane must be reference, configured-model, pinned-model, or current-model'); } return value; } @@ -232,18 +240,48 @@ function adapterReasonFromStderr(stderr) { return match?.[1] ?? null; } -function resolveToolkitRevision(explicit) { - if (explicit) return explicit; - if (process.env.GITHUB_SHA) return process.env.GITHUB_SHA; +export function resolveToolkitRevision( + explicit, + { githubSha = process.env.GITHUB_SHA, execFile = execFileSync, root = ROOT } = {}, +) { + const declaredRevision = explicit + ? commitSha(explicit, '--toolkit-revision') + : githubSha + ? commitSha(githubSha, 'GITHUB_SHA') + : null; + let dirty; + try { + dirty = execFile('git', ['status', '--porcelain', '--untracked-files=no'], { + cwd: root, + encoding: 'utf8', + }).trim(); + } catch { + if (declaredRevision) return declaredRevision; + throw new Error('--toolkit-revision is required outside a clean git checkout'); + } + if (dirty) { + throw new Error('Working tree has tracked changes; commit them before running the benchmark'); + } + if (declaredRevision) return declaredRevision; + let revision; try { - return execFileSync('git', ['rev-parse', 'HEAD'], { cwd: ROOT, encoding: 'utf8' }).trim(); + revision = execFile('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).trim(); } catch { - throw new Error('--toolkit-revision is required outside a git checkout'); + throw new Error('--toolkit-revision is required outside a clean git checkout'); + } + return commitSha(revision, 'git HEAD'); +} + +function commitSha(value, source) { + const revision = typeof value === 'string' ? value.trim().toLowerCase() : ''; + if (!/^[a-f0-9]{40}(?:[a-f0-9]{24})?$/.test(revision)) { + throw new Error(`${source} must be a 40- or 64-character commit SHA`); } + return revision; } function helpText() { - return `QVeris discover→call benchmark\n\nUsage:\n node src/run.mjs --model MODEL --adapter COMMAND --adapter-revision REVISION [options]\n\nOptions:\n --adapter-arg VALUE Repeatable adapter argument (no shell parsing)\n --adapter-revision VALUE Immutable adapter source/config revision (required)\n --toolkit-revision VALUE Toolkit revision (auto-detected in git/GitHub Actions)\n --model-revision VALUE Provider snapshot/backend revision; required for pinned-model\n --tasks PATH Task-set JSONL (default: tasks/v1.jsonl)\n --output PATH Run-record JSONL (default: benchmark-runs.jsonl)\n --trials N Trials per task (default: 3)\n --limit N Discover result limit (default: 10)\n --lane VALUE reference, configured-model, pinned-model, current-model, or legacy value\n --execute Perform billed call requests (required for workflow success)\n --adapter-timeout-ms N Per-stage adapter timeout (default: 120000)\n`; + return `QVeris discover→call benchmark\n\nUsage:\n node src/run.mjs --model MODEL --lane LANE --tasks PATH --adapter COMMAND --adapter-revision REVISION [options]\n\nOptions:\n --adapter-arg VALUE Repeatable adapter argument (no shell parsing)\n --adapter-revision VALUE Immutable adapter source/config revision (required)\n --toolkit-revision VALUE Toolkit commit SHA (auto-detected only in a clean checkout)\n --model-revision VALUE Provider snapshot/backend revision; required for reference and pinned-model\n --tasks PATH Immutable task-set JSONL (required)\n --output PATH Private run-record JSONL with per-trial checkpoints (default: benchmark-runs.jsonl)\n --trials N Trials per task (default: 3)\n --limit N Discover result limit (default: 10)\n --lane VALUE reference, configured-model, pinned-model, or current-model (required)\n --execute Perform billed call requests (required for workflow success)\n --adapter-timeout-ms N Per-stage adapter timeout (default: 120000)\n`; } if (process.argv[1] === fileURLToPath(import.meta.url)) { diff --git a/benchmarks/discover-call/src/score.mjs b/benchmarks/discover-call/src/score.mjs index 8e5f31c..f46e3b0 100644 --- a/benchmarks/discover-call/src/score.mjs +++ b/benchmarks/discover-call/src/score.mjs @@ -1,6 +1,7 @@ #!/usr/bin/env node -import { mkdir } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import { mkdir, readFile } from 'node:fs/promises'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -11,11 +12,15 @@ const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..'); export async function main(argv = process.argv.slice(2)) { const options = parseArgs(argv); - const [tasks, records] = await Promise.all([ - readJsonLines(resolve(options.tasks)), + const tasksPath = resolve(options.tasks); + const [taskBytes, tasks, records] = await Promise.all([ + readFile(tasksPath), + readJsonLines(tasksPath), readJsonLines(resolve(options.runs)), ]); - const summary = scoreRecords(tasks, records); + const summary = scoreRecords(tasks, records, { + taskSetSha256: createHash('sha256').update(taskBytes).digest('hex'), + }); const json = JSON.stringify(summary, null, 2) + '\n'; if (options.output) { const output = resolve(options.output); diff --git a/benchmarks/discover-call/src/scoring.mjs b/benchmarks/discover-call/src/scoring.mjs index aa422b9..54c24ad 100644 --- a/benchmarks/discover-call/src/scoring.mjs +++ b/benchmarks/discover-call/src/scoring.mjs @@ -1,8 +1,22 @@ import { validateTaskSet } from './harness.mjs'; -export function scoreRecords(tasks, records) { +export function scoreRecords(tasks, records, { taskSetSha256 } = {}) { validateTaskSet(tasks); if (!Array.isArray(records) || records.length === 0) throw new Error('At least one run record is required'); + if (taskSetSha256 !== undefined) { + if (typeof taskSetSha256 !== 'string' || !/^[a-f0-9]{64}$/.test(taskSetSha256)) { + throw new Error('Scoring taskSetSha256 must be a SHA-256 digest'); + } + for (const record of records) { + const declaredTaskSetSha256 = record?.metadata?.task_set_sha256; + if ( + (declaredTaskSetSha256 !== undefined && declaredTaskSetSha256 !== taskSetSha256) || + (record?.benchmark_version === 'v2' && declaredTaskSetSha256 !== taskSetSha256) + ) { + throw new Error(`Run record task-set digest does not match ${record?.task_id ?? ''}`); + } + } + } const benchmarkVersions = new Set(records.map((record) => record?.benchmark_version ?? 'v1')); if (benchmarkVersions.size !== 1) { throw new Error('Benchmark summary cannot mix benchmark versions'); @@ -102,7 +116,7 @@ export function scoreRecord(task, record) { const inspected = array(record.inspection?.returned_tool_ids); const required = array(record.inspection?.required_parameters); const parameters = plainObject(record.parameters) ? record.parameters : {}; - const parameterization = parameterizationMetrics(record.parameterization); + const parameterization = record.parameters === undefined ? parameterizationMetrics(record.parameterization) : null; const completed = record.status === 'completed'; const selectionGrounded = typeof record.discovery?.selection_grounded === 'boolean' @@ -118,7 +132,15 @@ export function scoreRecord(task, record) { : false; const requiredParameterAccuracy = parameterization?.required_parameter_accuracy ?? - (required.length === 0 ? 1 : mean(required.map((name) => hasValue(parameters[name])))); + (required.length === 0 + ? 1 + : mean( + required.map((requirement) => + Array.isArray(requirement) + ? requirement.some((name) => hasValue(parameters[name])) + : hasValue(parameters[requirement]), + ), + )); const constraintAccuracy = parameterization?.constraint_accuracy ?? mean(task.constraints.map((constraint) => constraintSatisfied(parameters, constraint, task.constraints))); @@ -174,10 +196,9 @@ function aggregateModel(model, records) { const failuresByStage = {}; const failuresByReason = {}; for (const record of records) { - if (record.error_stage) failuresByStage[record.error_stage] = (failuresByStage[record.error_stage] || 0) + 1; - if (record.error_reason) { - failuresByReason[record.error_reason] = (failuresByReason[record.error_reason] || 0) + 1; - } + const failure = strictFailure(record); + if (failure?.stage) failuresByStage[failure.stage] = (failuresByStage[failure.stage] || 0) + 1; + if (failure?.reason) failuresByReason[failure.reason] = (failuresByReason[failure.reason] || 0) + 1; } return { model, @@ -204,6 +225,20 @@ function aggregateModel(model, records) { }; } +function strictFailure(record) { + if (record.error_stage) return { stage: record.error_stage, reason: record.error_reason }; + if (!record.completed) return { stage: 'unknown', reason: 'incomplete_trial' }; + if (!record.selection_grounded) return { stage: 'select', reason: 'ungrounded_selection' }; + if (!record.inspection_grounded) return { stage: 'inspect', reason: 'ungrounded_inspection' }; + if (record.required_parameter_accuracy < 1) { + return { stage: 'parameterize', reason: 'incomplete_required_parameters' }; + } + if (record.constraint_accuracy < 1) return { stage: 'parameterize', reason: 'constraint_mismatch' }; + if (record.call_success === false) return { stage: 'call', reason: 'tool_returned_failure' }; + if (record.result_nonempty === false) return { stage: 'result', reason: 'empty_result' }; + return null; +} + function constraintSatisfied(parameters, constraint, constraints) { for (const alias of constraint.aliases) { if (!hasValue(parameters[alias])) continue; diff --git a/benchmarks/discover-call/src/validate.mjs b/benchmarks/discover-call/src/validate.mjs index 996ef1a..c2a7d2c 100644 --- a/benchmarks/discover-call/src/validate.mjs +++ b/benchmarks/discover-call/src/validate.mjs @@ -42,7 +42,7 @@ for (const file of resultFiles) { const records = await readJsonLines(resolve(root, 'results', file)); validatePublicRecords(records, policy); validateOfficialPublicRun(records, { taskSetSha256 }); - const generatedSummary = scoreRecords(tasks, records); + const generatedSummary = scoreRecords(tasks, records, { taskSetSha256 }); const summaryFile = file.replace(/\.runs\.jsonl$/, '.summary.json'); const publishedSummary = JSON.parse(await readFile(resolve(root, 'results', summaryFile), 'utf8')); if (!isDeepStrictEqual(comparableSummary(generatedSummary), comparableSummary(publishedSummary))) { diff --git a/benchmarks/discover-call/test/adapter.test.mjs b/benchmarks/discover-call/test/adapter.test.mjs index 2f85c33..30089b9 100644 --- a/benchmarks/discover-call/test/adapter.test.mjs +++ b/benchmarks/discover-call/test/adapter.test.mjs @@ -6,7 +6,7 @@ import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { createProcessAdapter, main } from '../src/run.mjs'; +import { createProcessAdapter, main, resolveToolkitRevision } from '../src/run.mjs'; import { buildClaudeInvocation, parseClaudeEnvelope, runClaude } from '../adapters/claude-cli.mjs'; import { buildCodexInvocation, CODEX_REASONING_EFFORT, parseCodexEvents, runCodex } from '../adapters/codex-cli.mjs'; @@ -29,27 +29,69 @@ test('process adapter exchanges one JSON object without shell parsing', async () assert.deepEqual(parameterized, { parameters: {} }); }); -test('process adapter cannot read the QVeris API key', async () => { - const previous = process.env.QVERIS_API_KEY; +test('process adapter cannot read QVeris environment values', async () => { + const previousKey = process.env.QVERIS_API_KEY; + const previousToken = process.env.QVERIS_MCP_HTTP_AUTH_TOKEN; process.env.QVERIS_API_KEY = 'must-not-reach-adapter'; + process.env.QVERIS_MCP_HTTP_AUTH_TOKEN = 'must-also-not-reach-adapter'; try { const invoke = createProcessAdapter({ command: process.execPath, args: [ '-e', - "process.stdin.resume(); process.stdin.on('end', () => process.stdout.write(JSON.stringify({visible: Boolean(process.env.QVERIS_API_KEY)})))", + "process.stdin.resume(); process.stdin.on('end', () => process.stdout.write(JSON.stringify({visible: Object.keys(process.env).some((name) => name.startsWith('QVERIS_'))})))", ], timeoutMs: 5_000, }); assert.deepEqual(await invoke({ stage: 'select' }), { visible: false }); } finally { - if (previous === undefined) delete process.env.QVERIS_API_KEY; - else process.env.QVERIS_API_KEY = previous; + if (previousKey === undefined) delete process.env.QVERIS_API_KEY; + else process.env.QVERIS_API_KEY = previousKey; + if (previousToken === undefined) delete process.env.QVERIS_MCP_HTTP_AUTH_TOKEN; + else process.env.QVERIS_MCP_HTTP_AUTH_TOKEN = previousToken; } }); test('runner requires an immutable adapter revision before API setup', async () => { await assert.rejects(main(['--model', 'model-a', '--adapter', process.execPath]), /adapter-revision/); + await assert.rejects( + main([ + '--model', + 'reference-v1', + '--lane', + 'reference', + '--adapter', + process.execPath, + '--adapter-revision', + 'adapter-sha', + '--tasks', + v4TasksPath, + ]), + /model-revision.*reference/, + ); +}); + +test('runner records only commit-shaped toolkit revisions from a clean checkout', () => { + const sha = 'a'.repeat(40); + const cleanGit = { + githubSha: '', + execFile(_command, args) { + return args[0] === 'status' ? '' : `${sha}\n`; + }, + }; + assert.equal(resolveToolkitRevision(sha.toUpperCase(), cleanGit), sha); + assert.throws(() => resolveToolkitRevision('main'), /commit SHA/); + assert.equal(resolveToolkitRevision(undefined, cleanGit), sha); + assert.throws( + () => + resolveToolkitRevision(sha, { + githubSha: '', + execFile(_command, args) { + return args[0] === 'status' ? ' M benchmarks/discover-call/src/run.mjs\n' : `${sha}\n`; + }, + }), + /tracked changes/, + ); }); test('process adapter reports startup failures without an unhandled stdin error', async () => { @@ -213,19 +255,23 @@ test('Claude adapter extracts only structured model output', () => { assert.throws(() => parseClaudeEnvelope(JSON.stringify({ result: 'not-json' })), /no structured output/); }); -test('Claude adapter removes the QVeris key from its model subprocess', async () => { - const previous = process.env.QVERIS_API_KEY; +test('Claude adapter removes QVeris environment values from its model subprocess', async () => { + const previousKey = process.env.QVERIS_API_KEY; + const previousToken = process.env.QVERIS_MCP_HTTP_AUTH_TOKEN; process.env.QVERIS_API_KEY = 'must-not-reach-model'; + process.env.QVERIS_MCP_HTTP_AUTH_TOKEN = 'must-also-not-reach-model'; try { const output = await runClaude({ command: process.execPath, - args: ['-e', 'process.stdout.write(String(Boolean(process.env.QVERIS_API_KEY)))'], + args: ['-e', "process.stdout.write(String(Object.keys(process.env).some((name) => name.startsWith('QVERIS_'))))"], stdin: '', }); assert.equal(output, 'false'); } finally { - if (previous === undefined) delete process.env.QVERIS_API_KEY; - else process.env.QVERIS_API_KEY = previous; + if (previousKey === undefined) delete process.env.QVERIS_API_KEY; + else process.env.QVERIS_API_KEY = previousKey; + if (previousToken === undefined) delete process.env.QVERIS_MCP_HTTP_AUTH_TOKEN; + else process.env.QVERIS_MCP_HTTP_AUTH_TOKEN = previousToken; } }); @@ -336,6 +382,35 @@ test('Codex adapter extracts the final structured message and rejects tool use', ); }); +test('Codex adapter removes QVeris environment values from its model subprocess', async () => { + const previousKey = process.env.QVERIS_API_KEY; + const previousToken = process.env.QVERIS_MCP_HTTP_AUTH_TOKEN; + process.env.QVERIS_API_KEY = 'must-not-reach-model'; + process.env.QVERIS_MCP_HTTP_AUTH_TOKEN = 'must-also-not-reach-model'; + try { + const result = parseCodexEvents( + await runCodex({ + command: process.execPath, + args: [ + '-e', + `const visible = Object.keys(process.env).some((name) => name.startsWith('QVERIS_')); + process.stdout.write([ + JSON.stringify({type:'item.completed',item:{type:'agent_message',text:JSON.stringify({visible})}}), + JSON.stringify({type:'turn.completed'}) + ].join('\\n'));`, + ], + stdin: '', + }), + ); + assert.deepEqual(result, { visible: false }); + } finally { + if (previousKey === undefined) delete process.env.QVERIS_API_KEY; + else process.env.QVERIS_API_KEY = previousKey; + if (previousToken === undefined) delete process.env.QVERIS_MCP_HTTP_AUTH_TOKEN; + else process.env.QVERIS_MCP_HTTP_AUTH_TOKEN = previousToken; + } +}); + test('Codex adapter force-kills a child that ignores the output-limit signal', async () => { await assert.rejects( runCodex( diff --git a/benchmarks/discover-call/test/api.test.mjs b/benchmarks/discover-call/test/api.test.mjs index 8cdd1d5..1f65ff7 100644 --- a/benchmarks/discover-call/test/api.test.mjs +++ b/benchmarks/discover-call/test/api.test.mjs @@ -8,11 +8,13 @@ test('normalizes safe API base URLs and rejects unsafe values', () => { for (const value of [ '', 'ftp://qveris.ai/api/v1', + 'http://qveris.ai/api/v1', 'https://user:pass@qveris.ai/api/v1', 'https://qveris.ai/api/v1?x=1', ]) { assert.throws(() => normalizeBaseUrl(value), /QVERIS_BASE_URL/); } + assert.equal(normalizeBaseUrl('http://127.0.0.1:8000/api/v1/'), 'http://127.0.0.1:8000/api/v1'); }); test('sends bearer authentication without exposing it in API errors', async () => { @@ -158,6 +160,58 @@ test('retries a response-body timeout before recording an API failure', async () assert.deepEqual(delays, [500]); }); +test('does not retry an ambiguous execute timeout that could duplicate a billed call', async () => { + let attempts = 0; + const delays = []; + const client = createApiClient({ + apiKey: 'test-key', + timeoutMs: 10, + maxRetries: 3, + sleep: async (ms) => delays.push(ms), + fetchImpl: async (_url, init) => { + attempts++; + return { + ok: true, + status: 200, + headers: new Headers(), + json: () => + new Promise((resolve, reject) => { + init.signal.addEventListener( + 'abort', + () => reject(Object.assign(new Error('aborted'), { name: 'AbortError' })), + { once: true }, + ); + }), + }; + }, + }); + + await assert.rejects( + client.call({ toolId: 'weather.tool', discoveryId: 'search-1', parameters: {} }), + (error) => error.benchmarkReason === 'response_timeout', + ); + assert.equal(attempts, 1); + assert.deepEqual(delays, []); +}); + +test('does not retry an execute 503 that could follow an upstream side effect', async () => { + let attempts = 0; + const client = createApiClient({ + apiKey: 'test-key', + maxRetries: 3, + fetchImpl: async () => { + attempts++; + return new Response('', { status: 503 }); + }, + }); + + await assert.rejects( + client.call({ toolId: 'weather.tool', discoveryId: 'search-1', parameters: {} }), + (error) => error.benchmarkReason === 'http_503', + ); + assert.equal(attempts, 1); +}); + test('pins full execution results for structural non-empty scoring', async () => { let requestBody; const client = createApiClient({ @@ -168,8 +222,41 @@ test('pins full execution results for structural non-empty scoring', async () => }, }); - await client.call({ toolId: 'weather.tool', discoveryId: 'search-1', parameters: { city: 'London' } }); - assert.equal(requestBody.respond_with, 'full'); + await client.call({ + toolId: 'weather.tool', + discoveryId: 'search-1', + sessionId: 'run-1', + model: 'model-a', + parameters: { city: 'London' }, + }); + assert.deepEqual(requestBody, { + parameters: { city: 'London' }, + search_id: 'search-1', + session_id: 'run-1', + model: 'model-a', + respond_with: 'full', + }); +}); + +test('pins compact discovery and inspection projections', async () => { + const bodies = []; + const client = createApiClient({ + apiKey: 'test-key', + fetchImpl: async (_url, init) => { + bodies.push(JSON.parse(init.body)); + return new Response(JSON.stringify({ search_id: 'search-1', results: [] })); + }, + }); + + await client.discover({ query: 'weather', limit: 5, sessionId: 'run-1' }); + await client.inspect({ toolIds: ['weather.tool'], discoveryId: 'search-1', sessionId: 'run-1' }); + assert.deepEqual(bodies[0], { query: 'weather', limit: 5, session_id: 'run-1', view: 'routing' }); + assert.deepEqual(bodies[1], { + tool_ids: ['weather.tool'], + search_id: 'search-1', + session_id: 'run-1', + view: 'lean', + }); }); test('cancels unsuccessful response bodies before reporting API errors', async () => { diff --git a/benchmarks/discover-call/test/harness.test.mjs b/benchmarks/discover-call/test/harness.test.mjs index 220a022..06a5e0a 100644 --- a/benchmarks/discover-call/test/harness.test.mjs +++ b/benchmarks/discover-call/test/harness.test.mjs @@ -12,6 +12,7 @@ const task = { test('orchestrates discover, select, inspect, parameterize, and call', async () => { const events = []; + const checkpointSizes = []; const records = await runBenchmark({ tasks: [task], model: 'model-a', @@ -28,6 +29,7 @@ test('orchestrates discover, select, inspect, parameterize, and call', async () results: [ { tool_id: 'weather.forecast', + one_of_required: [], params: [ { name: 'city', type: 'string', required: true }, { name: 'units', type: 'string', required: false, enum: ['metric', 'imperial'] }, @@ -46,7 +48,12 @@ test('orchestrates discover, select, inspect, parameterize, and call', async () assert.equal(payload.messages.length, 2); assert.equal(payload.messages[1].content, JSON.stringify(payload.input)); events.push([payload.stage, payload.input.task_id]); - if (payload.stage === 'select') return { tool_id: 'weather.forecast' }; + if (payload.stage === 'select') { + assert.equal('search_id' in payload.input.discovery, false); + return { tool_id: 'weather.forecast' }; + } + assert.equal('discovery_id' in payload.input, false); + assert.deepEqual(Object.keys(payload.input.selected_tool).sort(), ['one_of_required', 'params', 'tool_id']); assert.deepEqual(payload.response_schema.properties.parameters.required, ['city', 'units']); assert.deepEqual(payload.response_schema.properties.parameters.properties.units.type, ['string', 'null']); return { parameters: { city: 'London', units: null } }; @@ -54,6 +61,10 @@ test('orchestrates discover, select, inspect, parameterize, and call', async () metadata: { adapter_revision: 'adapter-sha' }, now: () => '2026-07-15T00:00:00.000Z', newRunId: () => 'run-1', + newSessionId: () => 'session-1', + async onRecord(records) { + checkpointSizes.push(records.length); + }, }); assert.deepEqual( @@ -68,6 +79,11 @@ test('orchestrates discover, select, inspect, parameterize, and call', async () assert.equal(records[0].call.result_nonempty, true); assert.equal('execution_id' in records[0].call, false); assert.equal(records[0].metadata.adapter_revision, 'adapter-sha'); + assert.equal(events[0][1].sessionId, 'session-1'); + assert.equal(events[2][1].sessionId, 'session-1'); + assert.equal(events[4][1].sessionId, 'session-1'); + assert.equal(events[4][1].model, 'model-a'); + assert.deepEqual(checkpointSizes, [1]); }); test('builds a strict provider-neutral schema from inspected parameters', () => { @@ -117,6 +133,7 @@ test('builds a strict provider-neutral schema from inspected parameters', () => }, }, ); + assert.doesNotThrow(() => parameterResponseSchema({ params: [], one_of_required: [] })); }); test('rejects required object parameters without a usable property schema', () => { @@ -238,7 +255,7 @@ test('records adapter failures without copying their error text', async () => { trials: 1, api: { async discover() { - return { results: [{ tool_id: 'weather.forecast' }] }; + return { search_id: 'search-1', results: [{ tool_id: 'weather.forecast' }] }; }, async inspect() { throw new Error('not reached'); @@ -306,7 +323,7 @@ test('does not inspect or execute a tool outside the discovery results', async ( api: { async discover() { events.push('discover'); - return { results: [{ tool_id: 'weather.forecast' }] }; + return { search_id: 'search-1', results: [{ tool_id: 'weather.forecast' }] }; }, async inspect() { events.push('inspect'); @@ -378,6 +395,28 @@ test('validates task constraints and trial bounds', async () => { }), /primitive finite value/, ); + assert.throws(() => validateTask({ ...task, discover_query: 123 }), /discover_query/); + assert.throws( + () => + validateTask({ + ...task, + constraints: [{ ...task.constraints[0], aliases: ['city', 'city'] }], + }), + /string aliases/, + ); + assert.throws( + () => + validateTask({ + ...task, + reference: { + candidates: [ + { tool_id: 'weather.forecast', parameters: { city: 'London' } }, + { tool_id: 'weather.forecast', parameters: { city: 'Paris' } }, + ], + }, + }), + /unique tool ids/, + ); await assert.rejects( runBenchmark({ tasks: [task], model: 'model-a', trials: 0, api: {}, invokeAdapter() {} }), /trials/, @@ -415,7 +454,63 @@ test('validates the complete task set before any external call', async () => { assert.equal(externalCalls, 0); }); -test('does not parameterize or execute when inspection omits the selected tool', async () => { +test('validates execution controls and all run ids before external calls', async () => { + let externalCalls = 0; + const api = { + async discover() { + externalCalls++; + return { search_id: 'search-1', results: [] }; + }, + async inspect() { + externalCalls++; + return { results: [] }; + }, + async call() { + externalCalls++; + return { success: false }; + }, + }; + const invokeAdapter = async () => { + externalCalls++; + return {}; + }; + + await assert.rejects( + runBenchmark({ tasks: [task], model: 'model-a', execute: 'false', api, invokeAdapter }), + /execute must be a boolean/, + ); + await assert.rejects( + runBenchmark({ tasks: [task], model: 'model-a', limit: 0, api, invokeAdapter }), + /limit must be an integer/, + ); + await assert.rejects( + runBenchmark({ + tasks: [task], + model: 'model-a', + trials: 2, + api, + invokeAdapter, + newRunId: () => 'duplicate-run', + }), + /Duplicate benchmark run id/, + ); + let runId = 0; + await assert.rejects( + runBenchmark({ + tasks: [task], + model: 'model-a', + trials: 2, + api, + invokeAdapter, + newRunId: () => `run-${++runId}`, + newSessionId: () => 'duplicate-session', + }), + /Duplicate benchmark session id/, + ); + assert.equal(externalCalls, 0); +}); + +test('stops before selection when discover omits its correlation id', async () => { const events = []; const [record] = await runBenchmark({ tasks: [task], @@ -427,6 +522,133 @@ test('does not parameterize or execute when inspection omits the selected tool', events.push('discover'); return { results: [{ tool_id: 'weather.forecast' }] }; }, + async inspect() { + events.push('inspect'); + throw new Error('not reached'); + }, + async call() { + events.push('call'); + throw new Error('not reached'); + }, + }, + async invokeAdapter() { + events.push('adapter'); + return {}; + }, + }); + + assert.deepEqual(events, ['discover']); + assert.deepEqual(record.error, { + stage: 'discover', + reason_code: 'missing_search_id', + message: 'Discover did not return a search_id', + }); + assert.equal(record.call.attempted, false); +}); + +test('validates adapter parameters and one-of requirements before a billed call', async () => { + const variants = [ + { parameters: {}, message: /omitted required parameter city/ }, + { parameters: { city: 123 }, message: /invalid string value/ }, + { parameters: { city: 'London', units: 'kelvin' }, message: /outside the parameter enum/ }, + { parameters: { city: 'London', unknown: true }, message: /unknown parameter unknown/ }, + { parameters: { city: 'London' }, oneOf: [['query', 'url']], message: /one_of_required group/ }, + ]; + + for (const [index, variant] of variants.entries()) { + let calls = 0; + const [record] = await runBenchmark({ + tasks: [task], + model: 'model-a', + trials: 1, + execute: true, + api: { + async discover() { + return { search_id: `search-${index}`, results: [{ tool_id: 'weather.forecast' }] }; + }, + async inspect() { + return { + results: [ + { + tool_id: 'weather.forecast', + params: [ + { name: 'city', type: 'string', required: true }, + { name: 'units', type: 'string', required: false, enum: ['metric', 'imperial'] }, + { name: 'query', type: 'string', required: false }, + { name: 'url', type: 'string', required: false }, + ], + ...(variant.oneOf ? { one_of_required: variant.oneOf } : {}), + }, + ], + }; + }, + async call() { + calls++; + return { success: true, result: { data: { ok: true } } }; + }, + }, + async invokeAdapter(payload) { + return payload.stage === 'select' ? { tool_id: 'weather.forecast' } : variant; + }, + }); + + assert.equal(calls, 0); + assert.equal(record.error.stage, 'parameterize'); + assert.equal(record.error.reason_code, 'invalid_parameter_values'); + assert.match(record.error.message, variant.message); + } +}); + +test('adapter projections cannot mutate the inspected schema used for call validation', async () => { + let calls = 0; + const [record] = await runBenchmark({ + tasks: [task], + model: 'model-a', + trials: 1, + execute: true, + api: { + async discover() { + return { search_id: 'search-1', results: [{ tool_id: 'weather.forecast' }] }; + }, + async inspect() { + return { + results: [ + { + tool_id: 'weather.forecast', + params: [{ name: 'city', type: 'string', required: true }], + }, + ], + }; + }, + async call() { + calls++; + return { success: true, result: { data: { ok: true } } }; + }, + }, + async invokeAdapter(payload) { + if (payload.stage === 'select') return { tool_id: 'weather.forecast' }; + payload.input.selected_tool.params[0].required = false; + return { parameters: {} }; + }, + }); + + assert.equal(calls, 0); + assert.equal(record.error.stage, 'parameterize'); + assert.equal(record.error.reason_code, 'invalid_parameter_values'); +}); + +test('does not parameterize or execute when inspection omits the selected tool', async () => { + const events = []; + const [record] = await runBenchmark({ + tasks: [task], + model: 'model-a', + trials: 1, + execute: true, + api: { + async discover() { + events.push('discover'); + return { search_id: 'search-1', results: [{ tool_id: 'weather.forecast' }] }; + }, async inspect() { events.push('inspect'); return { results: [{ tool_id: 'different.tool', params: [] }] }; diff --git a/benchmarks/discover-call/test/io.test.mjs b/benchmarks/discover-call/test/io.test.mjs index 9096d63..f9b87b9 100644 --- a/benchmarks/discover-call/test/io.test.mjs +++ b/benchmarks/discover-call/test/io.test.mjs @@ -1,10 +1,10 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { mkdtemp, readFile, readdir, rm, stat } from 'node:fs/promises'; +import { link, mkdtemp, readFile, readdir, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { writeJsonLines, writeTextAtomic } from '../src/io.mjs'; +import { validatePathSeparation, writeJsonLines, writeTextAtomic } from '../src/io.mjs'; test('atomically replaces private benchmark output files', async () => { const directory = await mkdtemp(join(tmpdir(), 'qveris-benchmark-io-')); @@ -26,3 +26,24 @@ test('atomically replaces private benchmark output files', async () => { await rm(directory, { recursive: true, force: true }); } }); + +test('rejects outputs that alias an input path or inode', async () => { + const directory = await mkdtemp(join(tmpdir(), 'qveris-benchmark-paths-')); + const input = join(directory, 'tasks.jsonl'); + const alias = join(directory, 'tasks-hardlink.jsonl'); + try { + await writeTextAtomic(input, '{}\n'); + await link(input, alias); + await assert.rejects(validatePathSeparation({ inputs: [input], outputs: [input] }), /must not overwrite/); + await assert.rejects(validatePathSeparation({ inputs: [input], outputs: [alias] }), /must not overwrite/); + await assert.rejects( + validatePathSeparation({ + inputs: [input], + outputs: [join(directory, 'out.jsonl'), join(directory, 'out.jsonl')], + }), + /different files/, + ); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/benchmarks/discover-call/test/publication.test.mjs b/benchmarks/discover-call/test/publication.test.mjs index 8042ecc..453aa5d 100644 --- a/benchmarks/discover-call/test/publication.test.mjs +++ b/benchmarks/discover-call/test/publication.test.mjs @@ -27,6 +27,7 @@ const policy = { forbidden_fields: [ 'execution_id', 'search_id', + 'session_id', 'connection_id', 'remaining_credits', 'result_tool_ids', @@ -111,6 +112,51 @@ test('hashes an unapproved selected tool instead of expanding the public catalog assert.equal(record.discovery.selection_grounded, true); }); +test('preserves a verifiably pinned lane while demoting legacy unpinned labels', () => { + const [pinned] = sanitizePublicRecords( + [ + rawRecord({ + metadata: { + ...rawRecord().metadata, + lane: 'pinned-model', + model_revision: 'provider-snapshot-2026-07-23', + }, + }), + ], + policy, + tasks, + ); + const [legacy] = sanitizePublicRecords( + [ + rawRecord({ + metadata: { + ...rawRecord().metadata, + lane: 'pinned-model', + model_revision: 'unreported', + }, + }), + ], + policy, + tasks, + ); + + assert.equal(pinned.metadata.lane, 'pinned-model'); + assert.equal(legacy.metadata.lane, 'configured-model'); + assert.throws( + () => + validatePublicRecords( + [ + { + ...pinned, + metadata: { ...pinned.metadata, model_revision: 'unreported' }, + }, + ], + policy, + ), + /immutable model revision/, + ); +}); + test('recomputes parameterization scores instead of trusting raw attestations', () => { const [record] = sanitizePublicRecords( [ @@ -378,7 +424,11 @@ test('rejects impossible public grounding attestations', () => { [ { ...record, - discovery: { ...record.discovery, result_count: 0 }, + discovery: { + ...record.discovery, + result_count: 0, + snapshot_sha256: '4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945', + }, }, ], policy, @@ -411,6 +461,22 @@ test('rejects impossible public grounding attestations', () => { ), /inconsistent grounding attestations/, ); + assert.throws( + () => + validatePublicRecords( + [ + { + ...record, + discovery: { + ...record.discovery, + result_count: record.metadata.discovery_limit + 1, + }, + }, + ], + policy, + ), + /count and SHA-256 snapshot/, + ); }); test('requires a categorical error stage and safe optional reason code', () => { @@ -521,6 +587,50 @@ test('official public runs require the exact task digest, execution, and three t ), /must execute every trial/, ); + assert.throws( + () => + validateOfficialPublicRun( + records.map((record) => ({ + ...record, + benchmark_version: 'v2', + metadata: { ...record.metadata, lane: 'model' }, + })), + { taskSetSha256 }, + ), + /explicit comparison lane/, + ); + assert.throws( + () => + validateOfficialPublicRun( + records.map((record) => ({ + ...record, + benchmark_version: 'v2', + metadata: { + ...record.metadata, + lane: 'reference', + model_revision: 'unreported', + }, + })), + { taskSetSha256 }, + ), + /immutable model revision/, + ); + assert.throws( + () => + validateOfficialPublicRun( + records.map((record) => ({ + ...record, + benchmark_version: 'v2', + metadata: { + ...record.metadata, + lane: 'configured-model', + toolkit_revision: 'unreported', + }, + })), + { taskSetSha256 }, + ), + /toolkit commit SHA/, + ); }); function rawRecord(overrides = {}) { diff --git a/benchmarks/discover-call/test/scoring.test.mjs b/benchmarks/discover-call/test/scoring.test.mjs index 4c62434..84bdc1b 100644 --- a/benchmarks/discover-call/test/scoring.test.mjs +++ b/benchmarks/discover-call/test/scoring.test.mjs @@ -198,13 +198,27 @@ test('aggregates per model with parameter and workflow rates', () => { calls: 0.5, preResult: 0.5, workflow: 0.5, - failures: {}, - failureReasons: {}, + failures: { select: 1 }, + failureReasons: { ungrounded_selection: 1 }, }, ); assert.equal(summary.models[0].workflow_success_task_cluster_bootstrap_95.length, 2); }); +test('classifies the earliest failed strict gate in aggregate diagnostics', () => { + const records = [ + { + ...resultRecord({ taskId: task.id }), + parameters: { city: 'Paris' }, + call: { attempted: true, success: false, result_nonempty: false }, + }, + ]; + const summary = scoreRecords([task], records); + + assert.deepEqual(summary.models[0].failures_by_stage, { parameterize: 1 }); + assert.deepEqual(summary.models[0].failures_by_reason, { constraint_mismatch: 1 }); +}); + test('does not infer non-empty results when a successful legacy call lacks evidence', () => { const record = { run_id: 'run-unobserved', @@ -296,6 +310,44 @@ test('scores sanitized parameterization attestations without publishing paramete ); }); +test('does not trust parameterization attestations when raw parameters are present', () => { + const result = scoreRecord(task, { + ...resultRecord({ taskId: task.id }), + inspection: { + selection_grounded: true, + required_parameters: ['city', ['query', 'url']], + }, + parameters: { city: '', query: '' }, + parameterization: { + required_parameter_accuracy: 1, + constraint_accuracy: 1, + }, + }); + + assert.equal(result.required_parameter_accuracy, 0); + assert.equal(result.constraint_accuracy, 0); + assert.equal(result.workflow_success, false); +}); + +test('scores each one-of-required group as one requirement', () => { + const base = { + ...resultRecord({ taskId: task.id }), + inspection: { + selection_grounded: true, + required_parameters: ['city', ['query', 'url']], + }, + parameters: { city: 'London', url: 'https://example.com' }, + }; + assert.equal(scoreRecord(task, base).required_parameter_accuracy, 1); + assert.equal( + scoreRecord(task, { + ...base, + parameters: { city: 'London' }, + }).required_parameter_accuracy, + 0.5, + ); +}); + test('rejects run records for unknown tasks', () => { assert.throws( () => scoreRecords([task], [{ run_id: 'run-unknown', task_id: 'unknown', model: 'model-a', trial: 1 }]), @@ -415,4 +467,23 @@ test('rejects duplicate run ids and mixed benchmark conditions', () => { ), /cannot mix benchmark versions/, ); + assert.throws( + () => + scoreRecords([task], [{ ...record, metadata: { ...record.metadata, task_set_sha256: 'b'.repeat(64) } }], { + taskSetSha256: 'a'.repeat(64), + }), + /task-set digest does not match/, + ); + assert.doesNotThrow(() => + scoreRecords([task], [{ ...record, benchmark_version: 'v1' }], { + taskSetSha256: 'a'.repeat(64), + }), + ); + assert.throws( + () => + scoreRecords([task], [{ ...record, benchmark_version: 'v2' }], { + taskSetSha256: 'a'.repeat(64), + }), + /task-set digest does not match/, + ); }); diff --git a/docs/en-US/discover-call-benchmark.md b/docs/en-US/discover-call-benchmark.md index e8e5529..683e3f2 100644 --- a/docs/en-US/discover-call-benchmark.md +++ b/docs/en-US/discover-call-benchmark.md @@ -55,32 +55,38 @@ task. They record the model identifier and provider revision (or revision, catalog revision when reported, catalog-observation digest, endpoint, and discovery limit. -Only sanitized JSONL is committed. Public artifacts omit execution, search, and -connection identifiers, raw parameter values, and the ordered discovery -catalog. Approved selected tool IDs may remain visible; other selected tools -are represented only by a digest. Parameter quality is published only as -required-parameter and constraint-accuracy attestations; inspected parameter -names are omitted so hashed tools do not leak schema details. See the +Only sanitized JSONL is committed. Public artifacts omit execution, search, +session, and connection identifiers, raw parameter values, and the ordered +discovery catalog. Approved selected tool IDs may remain visible; other +selected tools are represented only by a digest. Parameter quality is published +only as required-parameter and constraint-accuracy attestations; inspected +parameter names are omitted so hashed tools do not leak schema details. See the [publication policy](../../benchmarks/discover-call/PUBLICATION_POLICY.md). ## Published results -The official v4 baseline was run on 2026-07-23 over 18 immutable tasks, three -trials each, with real calls. +The 2026-07-23 v4 run is a **diagnostic baseline candidate**, not an official +quality baseline. It covers 18 immutable tasks, three trials each, with real +calls, but a later collector audit found that result non-emptiness was tested +against the full `result` wrapper instead of `result.data`. A wrapper containing +empty data could therefore be recorded as non-empty. Raw result bodies were +intentionally not retained, so the affected attestations cannot be recomputed. -| Metric | Curated reference route | `gpt-5.6-sol` configured model | +| Historical diagnostic | Curated reference route | `gpt-5.6-sol` configured model | | --- | ---: | ---: | | Completed and executed | 51 / 54 | 51 / 54 | | Constraint accuracy | 94.44% | 88.89% | -| Call and non-empty-result success | 100% (51 / 51) | 88.24% (45 / 51) | -| Strict workflow success | 94.44% (51 / 54) | 77.78% (42 / 54) | -| Workflow success, 95% task-cluster bootstrap | 83.33%–100% | 55.56%–94.44% | - -The strict benchmark gap is 16.66 percentage points. The reference route's -three failures are Tokyo-timezone coverage misses. The configured model's 12 -strict failures are three Tokyo constraint misses, three IP lookup call -failures, three company-profile call failures, and three safely classified -`tool_use_rejected` adapter failures. +| Call success | 100% (51 / 51) | 88.24% (45 / 51) | +| Wrapper-level non-empty observation | 100% (51 / 51) | 88.24% (45 / 51) | +| Strict workflow output | 94.44% (51 / 54) | 77.78% (42 / 54) | +| Workflow interval | 83.33%–100% | 55.56%–94.44% | + +The historical strict benchmark gap output is 16.66 percentage points. It must +not be presented as a current quality or routing baseline. The reference +route's three diagnostic failures are Tokyo-timezone coverage misses. The +configured model's 12 diagnostic strict failures are three Tokyo constraint +misses, three IP lookup call failures, three company-profile call failures, and +three safely classified `tool_use_rejected` adapter failures. The configured lane used `gpt-5.6-sol`, medium reasoning, and Codex CLI 0.144.1. Its provider model revision is `unreported`, so this is not presented as a @@ -88,7 +94,10 @@ pinned model snapshot. Both lanes observed API revision `2026-07-22.1`; the API did not report a catalog revision, and the separate catalog-observation digests differ. -The earlier v3 run is retained only as a diagnostic baseline. Its three +Promotion requires a fresh run under the corrected collector, which now checks +`result.data`, requests the full result projection, prevents ambiguous Execute +retries, and pins the reproducibility inputs before the first external call. +The earlier v3 run is also retained only as a diagnostic baseline. Its three successful Bitcoin calls used provider-specific `id=1`, which v3 incorrectly scored as constraint failures. Immutable `tasks/v4.jsonl` explicitly recognizes that mapping, and all three Bitcoin trials pass in v4. diff --git a/docs/zh-CN/discover-call-benchmark.md b/docs/zh-CN/discover-call-benchmark.md index 4e39e89..f84eeae 100644 --- a/docs/zh-CN/discover-call-benchmark.md +++ b/docs/zh-CN/discover-call-benchmark.md @@ -40,35 +40,42 @@ reference 与模型的严格工作流成功率之差称为 **strict benchmark ga `unreported`）、adapter 与 toolkit revision、任务集摘要、runtime、API revision、服务端有返回时 的 catalog revision、catalog observation 摘要、endpoint 和 discovery limit。 -仓库只提交脱敏后的 JSONL。公开 artifact 不包含 execution、search、connection 标识、原始参数值， -也不包含完整有序 discovery 目录。已批准的 selected tool ID 可以保留；其他 selected tool 只保留 -摘要。参数质量仅以必填参数完整率和任务约束准确率证明的形式公开；inspect 返回的参数名也会移除， -避免已哈希工具泄露 schema 细节。详见 +仓库只提交脱敏后的 JSONL。公开 artifact 不包含 execution、search、session、connection 标识、 +原始参数值，也不包含完整有序 discovery 目录。已批准的 selected tool ID 可以保留；其他 selected +tool 只保留摘要。参数质量仅以必填参数完整率和任务约束准确率证明的形式公开；inspect 返回的参数名 +也会移除，避免已哈希工具泄露 schema 细节。详见 [公开策略](../../benchmarks/discover-call/PUBLICATION_POLICY.md)。 ## 已发布结果 -正式 v4 基线于 2026-07-23 运行：18 个不可变任务，每个任务三个 trial，并启用真实调用。 +2026-07-23 的 v4 运行是 **diagnostic baseline candidate**，不是正式质量基线。它包含 18 个不可变 +任务、每个任务三个 trial，并启用真实调用；但后续 collector 审核发现，结果非空判断检查的是完整 +`result` wrapper，而不是 `result.data`。因此，即使 data 为空，只要 wrapper 非空也可能被记录为 +非空。原始结果正文按策略未保留，受影响的证明无法重新计算。 -| 指标 | Curated reference route | `gpt-5.6-sol` configured model | +| 历史诊断输出 | Curated reference route | `gpt-5.6-sol` configured model | | --- | ---: | ---: | | 完成参数化并执行 | 51 / 54 | 51 / 54 | | 任务约束准确率 | 94.44% | 88.89% | -| call 成功且结果非空 | 100%（51 / 51） | 88.24%（45 / 51） | -| 严格工作流成功率 | 94.44%（51 / 54） | 77.78%（42 / 54） | -| 工作流成功率 95% task-cluster bootstrap | 83.33%–100% | 55.56%–94.44% | +| call 成功率 | 100%（51 / 51） | 88.24%（45 / 51） | +| wrapper 级非空观察 | 100%（51 / 51） | 88.24%（45 / 51） | +| 严格工作流输出 | 94.44%（51 / 54） | 77.78%（42 / 54） | +| 工作流区间 | 83.33%–100% | 55.56%–94.44% | -strict benchmark gap 为 16.66 个百分点。reference 的三次失败均为东京时区目录覆盖缺口。configured -model 的 12 次严格失败包括：三次东京约束未命中、三次 IP lookup 调用失败、三次 company profile -调用失败，以及三次安全归类的 `tool_use_rejected` adapter 失败。 +历史 strict benchmark gap 输出为 16.66 个百分点，不得将其表述为当前质量基线或纯路由基线。 +reference 的三次诊断失败均为东京时区目录覆盖缺口。configured model 的 12 次诊断严格失败包括： +三次东京约束未命中、三次 IP lookup 调用失败、三次 company profile 调用失败，以及三次安全归类的 +`tool_use_rejected` adapter 失败。 configured lane 使用 `gpt-5.6-sol`、medium reasoning 和 Codex CLI 0.144.1。provider 模型 revision 为 `unreported`，因此不会称为 pinned model snapshot。两个 lane 都观察到 API revision `2026-07-22.1`；API 未报告 catalog revision，且两次独立运行的 catalog observation 摘要不同。 -较早的 v3 只保留为 diagnostic baseline。其三次 Bitcoin 调用通过 provider 特定的 `id=1` 成功返回， -但 v3 错误地将其计为约束失败。不可变 `tasks/v4.jsonl` 已明确识别该映射，v4 的三次 Bitcoin trial -全部通过。 +只有在修正后的 collector 上重新运行，才能晋升为正式基线；当前 collector 已改为检查 +`result.data`、显式请求完整结果投影、禁止对结果不明确的 Execute 失败自动重试，并在首次外部调用前 +固定所有复现输入。较早的 v3 同样只保留为 diagnostic baseline。其三次 Bitcoin 调用通过 provider +特定的 `id=1` 成功返回，但 v3 错误地将其计为约束失败。不可变 `tasks/v4.jsonl` 已明确识别该映射， +v4 的三次 Bitcoin trial 全部通过。 完整的[结果说明、revision、脱敏 JSONL 与生成汇总](../../benchmarks/discover-call/results/README.md) 均已公开。scorer fixture 仍只用于测试，不代表产品性能。 From 987aca0fa704a5aea9bc9e1011df21e46ec4a8b7 Mon Sep 17 00:00:00 2001 From: Linfang Wang Date: Fri, 24 Jul 2026 00:00:55 +0800 Subject: [PATCH 15/19] docs(benchmark): clarify private session handling --- benchmarks/discover-call/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/benchmarks/discover-call/README.md b/benchmarks/discover-call/README.md index 1b2a022..1bd57b3 100644 --- a/benchmarks/discover-call/README.md +++ b/benchmarks/discover-call/README.md @@ -133,8 +133,9 @@ official publication completeness checks. Within a trial, Discover, Inspect, and Execute share an independent private `session_id`, and Execute also records the model identifier in the API request. -The session id is not derived from the public run id. Neither operational field -is sent to the model adapter or retained in public artifacts. +The session id is not derived from the public run id, sent to the model adapter, +or retained in public artifacts. The model identifier remains part of the +benchmark's public reproducibility metadata. ### Comparison lanes From c2b1b052f0271df3b12e8055c7f6ea01b87c7556 Mon Sep 17 00:00:00 2001 From: Linfang Wang Date: Fri, 24 Jul 2026 00:04:40 +0800 Subject: [PATCH 16/19] test(benchmark): wait for adapter process cleanup --- benchmarks/discover-call/test/adapter.test.mjs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/benchmarks/discover-call/test/adapter.test.mjs b/benchmarks/discover-call/test/adapter.test.mjs index 30089b9..2cef181 100644 --- a/benchmarks/discover-call/test/adapter.test.mjs +++ b/benchmarks/discover-call/test/adapter.test.mjs @@ -149,7 +149,7 @@ test('process adapter force-kills a child that ignores SIGTERM', async () => { }); await assert.rejects(invoke({ stage: 'select' }), /timed out/); const pid = Number(await readFile(pidPath, 'utf8')); - assert.equal(processExists(pid), false); + assert.equal(await waitForProcessExit(pid), true); } finally { await rm(directory, { recursive: true, force: true }); } @@ -437,3 +437,12 @@ function processExists(pid) { throw error; } } + +async function waitForProcessExit(pid, timeoutMs = 1_000) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (!processExists(pid)) return true; + await new Promise((resolvePromise) => setTimeout(resolvePromise, 10)); + } + return !processExists(pid); +} From dd09a5fdd3f6b169a9bd344b91393d7b52fd4530 Mon Sep 17 00:00:00 2001 From: Linfang Wang Date: Fri, 24 Jul 2026 11:56:54 +0800 Subject: [PATCH 17/19] fix(benchmark): enforce remaining integrity boundaries --- .../discover-call/adapters/claude-cli.mjs | 2 +- .../discover-call/adapters/codex-cli.mjs | 2 +- benchmarks/discover-call/src/harness.mjs | 31 +++++++-- benchmarks/discover-call/src/run.mjs | 26 ++++++- benchmarks/discover-call/src/scoring.mjs | 12 ++-- benchmarks/discover-call/src/validate.mjs | 14 +++- .../discover-call/test/adapter.test.mjs | 37 +++++++++- .../discover-call/test/harness.test.mjs | 68 +++++++++++++++++++ .../discover-call/test/scoring.test.mjs | 20 ++++++ 9 files changed, 194 insertions(+), 18 deletions(-) diff --git a/benchmarks/discover-call/adapters/claude-cli.mjs b/benchmarks/discover-call/adapters/claude-cli.mjs index 90e21f1..6284f64 100644 --- a/benchmarks/discover-call/adapters/claude-cli.mjs +++ b/benchmarks/discover-call/adapters/claude-cli.mjs @@ -154,7 +154,7 @@ export function runClaude(invocation, { outputLimit = 1_000_000, forceKillAfterM function stripQverisEnvironment(env) { for (const name of Object.keys(env)) { - if (name.startsWith('QVERIS_')) delete env[name]; + if (name.toUpperCase().startsWith('QVERIS_')) delete env[name]; } } diff --git a/benchmarks/discover-call/adapters/codex-cli.mjs b/benchmarks/discover-call/adapters/codex-cli.mjs index bfb7b7c..9badbdf 100644 --- a/benchmarks/discover-call/adapters/codex-cli.mjs +++ b/benchmarks/discover-call/adapters/codex-cli.mjs @@ -204,7 +204,7 @@ export function runCodex(invocation, { outputLimit = 1_000_000, forceKillAfterMs function stripQverisEnvironment(env) { for (const name of Object.keys(env)) { - if (name.startsWith('QVERIS_')) delete env[name]; + if (name.toUpperCase().startsWith('QVERIS_')) delete env[name]; } } diff --git a/benchmarks/discover-call/src/harness.mjs b/benchmarks/discover-call/src/harness.mjs index 09efa5b..1c021f2 100644 --- a/benchmarks/discover-call/src/harness.mjs +++ b/benchmarks/discover-call/src/harness.mjs @@ -171,6 +171,7 @@ export async function runBenchmark({ throw new Error('newRunId must return a safe non-empty string'); } if (runIds.has(runId)) throw new Error(`Duplicate benchmark run id: ${runId}`); + if (sessionIds.has(runId)) throw new Error('Benchmark run and session ids must be disjoint'); const sessionId = newSessionId(); if ( typeof sessionId !== 'string' || @@ -181,6 +182,9 @@ export async function runBenchmark({ throw new Error('newSessionId must return a safe non-empty string'); } if (sessionIds.has(sessionId)) throw new Error('Duplicate benchmark session id'); + if (sessionId === runId || runIds.has(sessionId)) { + throw new Error('Benchmark run and session ids must be disjoint'); + } runIds.add(runId); sessionIds.add(sessionId); runPlan.push({ task, trial, runId, sessionId }); @@ -339,7 +343,7 @@ function validateAdapterParameters(tool, parameters) { validateParameterValue(value, definition); } for (const definition of definitions.values()) { - if (definition.required === true && !hasParameterValue(parameters[definition.name])) { + if (definition.required === true && !hasOwnParameterValue(parameters, definition.name)) { throw stageError( 'parameterize', `Adapter omitted required parameter ${definition.name}`, @@ -348,7 +352,7 @@ function validateAdapterParameters(tool, parameters) { } } for (const group of oneOfRequiredGroups(tool)) { - if (!group.some((name) => hasParameterValue(parameters[name]))) { + if (!group.some((name) => hasOwnParameterValue(parameters, name))) { throw stageError( 'parameterize', `Adapter omitted one_of_required group ${group.join(',')}`, @@ -407,7 +411,7 @@ function validateParameterValue(value, parameter, forceRequired = parameter?.req validateParameterValue(child, properties[name]); } for (const [name, child] of Object.entries(properties)) { - if (child?.required === true && !hasParameterValue(value[name])) { + if (child?.required === true && !hasOwnParameterValue(value, name)) { throw stageError('parameterize', 'Adapter omitted a required nested parameter', 'invalid_parameter_values'); } } @@ -429,7 +433,7 @@ export function parameterResponseSchema(tool) { 'unsupported_parameter_schema', ); } - properties[parameter.name] = strictParameterSchema(parameter); + defineOwn(properties, parameter.name, strictParameterSchema(parameter)); } return { type: 'object', @@ -557,7 +561,11 @@ function strictParameterSchema(parameter, forceRequired = parameter?.required == } const nestedProperties = {}; for (const [name, definition] of Object.entries(nestedDefinitions)) { - nestedProperties[name] = strictParameterSchema(isPlainObject(definition) ? definition : { type: 'string' }); + defineOwn( + nestedProperties, + name, + strictParameterSchema(isPlainObject(definition) ? definition : { type: 'string' }), + ); } schema.additionalProperties = false; schema.required = Object.keys(nestedProperties); @@ -607,6 +615,19 @@ function hasParameterValue(value) { return value !== undefined && value !== null && value !== ''; } +function hasOwnParameterValue(parameters, name) { + return Object.hasOwn(parameters, name) && hasParameterValue(parameters[name]); +} + +function defineOwn(object, name, value) { + Object.defineProperty(object, name, { + configurable: true, + enumerable: true, + value, + writable: true, + }); +} + function hasNonemptyResult(result) { if (result === null || result === undefined) return false; if (typeof result === 'string') return result.trim().length > 0; diff --git a/benchmarks/discover-call/src/run.mjs b/benchmarks/discover-call/src/run.mjs index 9b10f7e..0a37159 100644 --- a/benchmarks/discover-call/src/run.mjs +++ b/benchmarks/discover-call/src/run.mjs @@ -155,7 +155,7 @@ export function createProcessAdapter({ command, args = [], timeoutMs = 120_000, function stripQverisEnvironment(env) { for (const name of Object.keys(env)) { - if (name.startsWith('QVERIS_')) delete env[name]; + if (name.toUpperCase().startsWith('QVERIS_')) delete env[name]; } } @@ -192,6 +192,11 @@ function parseArgs(argv) { if (!options.help && !options.adapterRevision) throw new Error('--adapter-revision is required'); if (!options.help && !options.tasks) throw new Error('--tasks is required'); if (!options.help && !options.lane) throw new Error('--lane is required'); + if (!options.help) { + safeProvenanceValue(options.model, '--model'); + safeProvenanceValue(options.adapterRevision, '--adapter-revision'); + safeProvenanceValue(options.modelRevision, '--model-revision'); + } if (!options.help && ['reference', 'pinned-model'].includes(options.lane) && options.modelRevision === 'unreported') { throw new Error(`--model-revision is required for --lane ${options.lane}`); } @@ -217,6 +222,18 @@ function lane(value) { return value; } +function safeProvenanceValue(value, flag) { + if ( + typeof value !== 'string' || + !value.trim() || + value.length > 256 || + /[\p{Cc}\p{Cf}]/u.test(value) + ) { + throw new Error(`${flag} must be a safe non-empty string of at most 256 characters`); + } + return value; +} + function catalogObservationSha256(records) { const observation = records.map((record) => ({ task_id: record.task_id, @@ -262,14 +279,17 @@ export function resolveToolkitRevision( if (dirty) { throw new Error('Working tree has tracked changes; commit them before running the benchmark'); } - if (declaredRevision) return declaredRevision; let revision; try { revision = execFile('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).trim(); } catch { throw new Error('--toolkit-revision is required outside a clean git checkout'); } - return commitSha(revision, 'git HEAD'); + const headRevision = commitSha(revision, 'git HEAD'); + if (declaredRevision && declaredRevision !== headRevision) { + throw new Error('Declared toolkit revision does not match the checked-out git HEAD'); + } + return declaredRevision ?? headRevision; } function commitSha(value, source) { diff --git a/benchmarks/discover-call/src/scoring.mjs b/benchmarks/discover-call/src/scoring.mjs index 54c24ad..ee54102 100644 --- a/benchmarks/discover-call/src/scoring.mjs +++ b/benchmarks/discover-call/src/scoring.mjs @@ -137,8 +137,8 @@ export function scoreRecord(task, record) { : mean( required.map((requirement) => Array.isArray(requirement) - ? requirement.some((name) => hasValue(parameters[name])) - : hasValue(parameters[requirement]), + ? requirement.some((name) => hasOwnValue(parameters, name)) + : hasOwnValue(parameters, requirement), ), )); const constraintAccuracy = @@ -241,7 +241,7 @@ function strictFailure(record) { function constraintSatisfied(parameters, constraint, constraints) { for (const alias of constraint.aliases) { - if (!hasValue(parameters[alias])) continue; + if (!hasOwnValue(parameters, alias)) continue; const actual = parameters[alias]; if (constraint.match === 'contains') { if (normalize(actual, constraint).includes(normalize(constraint.value, constraint))) return 1; @@ -253,7 +253,7 @@ function constraintSatisfied(parameters, constraint, constraints) { } } for (const alias of array(constraint.composite_aliases)) { - if (!hasValue(parameters[alias])) continue; + if (!hasOwnValue(parameters, alias)) continue; const compositeConstraints = constraints.filter((candidate) => array(candidate.composite_aliases).includes(alias)); const values = compositeConstraints.map((candidate) => normalize(candidate.value, candidate)); const representations = new Set([ @@ -299,6 +299,10 @@ function hasValue(value) { return value !== undefined && value !== null && value !== ''; } +function hasOwnValue(parameters, name) { + return Object.hasOwn(parameters, name) && hasValue(parameters[name]); +} + function mean(values) { if (values.length === 0) return 1; return values.reduce((sum, value) => sum + Number(value), 0) / values.length; diff --git a/benchmarks/discover-call/src/validate.mjs b/benchmarks/discover-call/src/validate.mjs index c2a7d2c..66e0b95 100644 --- a/benchmarks/discover-call/src/validate.mjs +++ b/benchmarks/discover-call/src/validate.mjs @@ -31,7 +31,19 @@ scoreRecords(v1, fixture); const policy = JSON.parse(await readFile(resolve(root, 'publication-policy.json'), 'utf8')); validatePolicy(policy); -const resultFiles = (await readdir(resolve(root, 'results'))).filter((name) => /\.runs\.jsonl$/.test(name)).sort(); +const resultDirectoryFiles = await readdir(resolve(root, 'results')); +const resultFiles = resultDirectoryFiles.filter((name) => /\.runs\.jsonl$/.test(name)).sort(); +const resultStems = new Set(resultFiles.map((name) => name.replace(/\.runs\.jsonl$/, ''))); +const summaryStems = new Set( + resultDirectoryFiles.filter((name) => /\.summary\.json$/.test(name)).map((name) => name.replace(/\.summary\.json$/, '')), +); +const orphanRuns = [...resultStems].filter((stem) => !summaryStems.has(stem)); +const orphanSummaries = [...summaryStems].filter((stem) => !resultStems.has(stem)); +if (orphanRuns.length || orphanSummaries.length) { + throw new Error( + `Public results need matching runs and summary files; missing summaries: ${orphanRuns.join(', ') || 'none'}; missing runs: ${orphanSummaries.join(', ') || 'none'}`, + ); +} for (const file of resultFiles) { const version = file.match(/-v(\d+)\.runs\.jsonl$/)?.[1]; if (!version) throw new Error(`${file}: result filename must identify its task-set version`); diff --git a/benchmarks/discover-call/test/adapter.test.mjs b/benchmarks/discover-call/test/adapter.test.mjs index 2cef181..30cd8f2 100644 --- a/benchmarks/discover-call/test/adapter.test.mjs +++ b/benchmarks/discover-call/test/adapter.test.mjs @@ -32,14 +32,16 @@ test('process adapter exchanges one JSON object without shell parsing', async () test('process adapter cannot read QVeris environment values', async () => { const previousKey = process.env.QVERIS_API_KEY; const previousToken = process.env.QVERIS_MCP_HTTP_AUTH_TOKEN; + const previousLowercaseProbe = process.env.qveris_lowercase_probe; process.env.QVERIS_API_KEY = 'must-not-reach-adapter'; process.env.QVERIS_MCP_HTTP_AUTH_TOKEN = 'must-also-not-reach-adapter'; + process.env.qveris_lowercase_probe = 'must-not-reach-adapter-either'; try { const invoke = createProcessAdapter({ command: process.execPath, args: [ '-e', - "process.stdin.resume(); process.stdin.on('end', () => process.stdout.write(JSON.stringify({visible: Object.keys(process.env).some((name) => name.startsWith('QVERIS_'))})))", + "process.stdin.resume(); process.stdin.on('end', () => process.stdout.write(JSON.stringify({visible: Object.keys(process.env).some((name) => name.toUpperCase().startsWith('QVERIS_'))})))", ], timeoutMs: 5_000, }); @@ -49,11 +51,28 @@ test('process adapter cannot read QVeris environment values', async () => { else process.env.QVERIS_API_KEY = previousKey; if (previousToken === undefined) delete process.env.QVERIS_MCP_HTTP_AUTH_TOKEN; else process.env.QVERIS_MCP_HTTP_AUTH_TOKEN = previousToken; + if (previousLowercaseProbe === undefined) delete process.env.qveris_lowercase_probe; + else process.env.qveris_lowercase_probe = previousLowercaseProbe; } }); test('runner requires an immutable adapter revision before API setup', async () => { await assert.rejects(main(['--model', 'model-a', '--adapter', process.execPath]), /adapter-revision/); + await assert.rejects( + main([ + '--model', + 'model-a\ninjected', + '--lane', + 'configured-model', + '--adapter', + process.execPath, + '--adapter-revision', + 'adapter-sha', + '--tasks', + v4TasksPath, + ]), + /--model must be a safe non-empty string/, + ); await assert.rejects( main([ '--model', @@ -80,6 +99,7 @@ test('runner records only commit-shaped toolkit revisions from a clean checkout' }, }; assert.equal(resolveToolkitRevision(sha.toUpperCase(), cleanGit), sha); + assert.throws(() => resolveToolkitRevision('b'.repeat(40), cleanGit), /does not match.*HEAD/); assert.throws(() => resolveToolkitRevision('main'), /commit SHA/); assert.equal(resolveToolkitRevision(undefined, cleanGit), sha); assert.throws( @@ -258,12 +278,17 @@ test('Claude adapter extracts only structured model output', () => { test('Claude adapter removes QVeris environment values from its model subprocess', async () => { const previousKey = process.env.QVERIS_API_KEY; const previousToken = process.env.QVERIS_MCP_HTTP_AUTH_TOKEN; + const previousLowercaseProbe = process.env.qveris_lowercase_probe; process.env.QVERIS_API_KEY = 'must-not-reach-model'; process.env.QVERIS_MCP_HTTP_AUTH_TOKEN = 'must-also-not-reach-model'; + process.env.qveris_lowercase_probe = 'must-not-reach-model-either'; try { const output = await runClaude({ command: process.execPath, - args: ['-e', "process.stdout.write(String(Object.keys(process.env).some((name) => name.startsWith('QVERIS_'))))"], + args: [ + '-e', + "process.stdout.write(String(Object.keys(process.env).some((name) => name.toUpperCase().startsWith('QVERIS_'))))", + ], stdin: '', }); assert.equal(output, 'false'); @@ -272,6 +297,8 @@ test('Claude adapter removes QVeris environment values from its model subprocess else process.env.QVERIS_API_KEY = previousKey; if (previousToken === undefined) delete process.env.QVERIS_MCP_HTTP_AUTH_TOKEN; else process.env.QVERIS_MCP_HTTP_AUTH_TOKEN = previousToken; + if (previousLowercaseProbe === undefined) delete process.env.qveris_lowercase_probe; + else process.env.qveris_lowercase_probe = previousLowercaseProbe; } }); @@ -385,15 +412,17 @@ test('Codex adapter extracts the final structured message and rejects tool use', test('Codex adapter removes QVeris environment values from its model subprocess', async () => { const previousKey = process.env.QVERIS_API_KEY; const previousToken = process.env.QVERIS_MCP_HTTP_AUTH_TOKEN; + const previousLowercaseProbe = process.env.qveris_lowercase_probe; process.env.QVERIS_API_KEY = 'must-not-reach-model'; process.env.QVERIS_MCP_HTTP_AUTH_TOKEN = 'must-also-not-reach-model'; + process.env.qveris_lowercase_probe = 'must-not-reach-model-either'; try { const result = parseCodexEvents( await runCodex({ command: process.execPath, args: [ '-e', - `const visible = Object.keys(process.env).some((name) => name.startsWith('QVERIS_')); + `const visible = Object.keys(process.env).some((name) => name.toUpperCase().startsWith('QVERIS_')); process.stdout.write([ JSON.stringify({type:'item.completed',item:{type:'agent_message',text:JSON.stringify({visible})}}), JSON.stringify({type:'turn.completed'}) @@ -408,6 +437,8 @@ test('Codex adapter removes QVeris environment values from its model subprocess' else process.env.QVERIS_API_KEY = previousKey; if (previousToken === undefined) delete process.env.QVERIS_MCP_HTTP_AUTH_TOKEN; else process.env.QVERIS_MCP_HTTP_AUTH_TOKEN = previousToken; + if (previousLowercaseProbe === undefined) delete process.env.qveris_lowercase_probe; + else process.env.qveris_lowercase_probe = previousLowercaseProbe; } }); diff --git a/benchmarks/discover-call/test/harness.test.mjs b/benchmarks/discover-call/test/harness.test.mjs index 06a5e0a..2b66a86 100644 --- a/benchmarks/discover-call/test/harness.test.mjs +++ b/benchmarks/discover-call/test/harness.test.mjs @@ -134,6 +134,11 @@ test('builds a strict provider-neutral schema from inspected parameters', () => }, ); assert.doesNotThrow(() => parameterResponseSchema({ params: [], one_of_required: [] })); + const prototypeNamedSchema = parameterResponseSchema({ + params: [{ name: '__proto__', type: 'string', required: true }], + }); + assert.equal(Object.hasOwn(prototypeNamedSchema.properties.parameters.properties, '__proto__'), true); + assert.deepEqual(prototypeNamedSchema.properties.parameters.required, ['__proto__']); }); test('rejects required object parameters without a usable property schema', () => { @@ -507,9 +512,72 @@ test('validates execution controls and all run ids before external calls', async }), /Duplicate benchmark session id/, ); + await assert.rejects( + runBenchmark({ + tasks: [task], + model: 'model-a', + trials: 1, + api, + invokeAdapter, + newRunId: () => 'shared-id', + newSessionId: () => 'shared-id', + }), + /run and session ids must be disjoint/, + ); + let plannedRun = 0; + let plannedSession = 0; + await assert.rejects( + runBenchmark({ + tasks: [task], + model: 'model-a', + trials: 2, + api, + invokeAdapter, + newRunId: () => ['run-1', 'session-1'][plannedRun++], + newSessionId: () => ['session-1', 'session-2'][plannedSession++], + }), + /run and session ids must be disjoint/, + ); assert.equal(externalCalls, 0); }); +test('prototype-named parameters cannot bypass the billed-call gate', async () => { + let callCount = 0; + const [record] = await runBenchmark({ + tasks: [task], + model: 'model-a', + trials: 1, + execute: true, + api: { + async discover() { + return { search_id: 'search-1', results: [{ tool_id: 'weather.forecast' }] }; + }, + async inspect() { + return { + results: [ + { + tool_id: 'weather.forecast', + params: [{ name: 'toString', type: 'string', required: true }], + }, + ], + }; + }, + async call() { + callCount++; + return { success: true, result: { data: 'unexpected' } }; + }, + }, + async invokeAdapter(payload) { + return payload.stage === 'select' ? { tool_id: 'weather.forecast' } : { parameters: {} }; + }, + }); + + assert.equal(callCount, 0); + assert.equal(record.status, 'failed'); + assert.equal(record.error.stage, 'parameterize'); + assert.equal(record.error.reason_code, 'invalid_parameter_values'); +}); + test('stops before selection when discover omits its correlation id', async () => { const events = []; const [record] = await runBenchmark({ diff --git a/benchmarks/discover-call/test/scoring.test.mjs b/benchmarks/discover-call/test/scoring.test.mjs index 84bdc1b..f34b608 100644 --- a/benchmarks/discover-call/test/scoring.test.mjs +++ b/benchmarks/discover-call/test/scoring.test.mjs @@ -348,6 +348,26 @@ test('scores each one-of-required group as one requirement', () => { ); }); +test('does not score inherited prototype properties as supplied parameters', () => { + const prototypeTask = { + id: 'prototype-parameter', + prompt: 'Prototype parameter', + constraints: [{ id: 'prototype', aliases: ['__proto__'], value: '{}' }], + }; + const result = scoreRecord(prototypeTask, { + ...resultRecord({ taskId: prototypeTask.id }), + inspection: { + selection_grounded: true, + required_parameters: ['toString'], + }, + parameters: {}, + }); + + assert.equal(result.required_parameter_accuracy, 0); + assert.equal(result.constraint_accuracy, 0); + assert.equal(result.workflow_success, false); +}); + test('rejects run records for unknown tasks', () => { assert.throws( () => scoreRecords([task], [{ run_id: 'run-unknown', task_id: 'unknown', model: 'model-a', trial: 1 }]), From a7f2aa60ef143dbbb35eaf9006ed8123d778fb13 Mon Sep 17 00:00:00 2001 From: Linfang Wang Date: Fri, 24 Jul 2026 12:03:35 +0800 Subject: [PATCH 18/19] fix(benchmark): publish artifact pairs transactionally --- benchmarks/discover-call/README.md | 6 ++ benchmarks/discover-call/src/io.mjs | 98 +++++++++++++++++++++++ benchmarks/discover-call/src/publish.mjs | 14 +++- benchmarks/discover-call/test/io.test.mjs | 64 ++++++++++++++- 4 files changed, 177 insertions(+), 5 deletions(-) diff --git a/benchmarks/discover-call/README.md b/benchmarks/discover-call/README.md index 1bd57b3..36a18d0 100644 --- a/benchmarks/discover-call/README.md +++ b/benchmarks/discover-call/README.md @@ -308,6 +308,12 @@ The adapter uses the CLI's existing authentication and never receives ## Publication rules +The publication command stages and synchronizes the public runs and summary +before replacing either destination. If any replacement fails, it restores the +previous pair before returning an error. Repository validation also rejects +missing or orphaned counterparts, so an interrupted process cannot silently +leave a publishable mixed generation. + An official result must include: - sanitized public JSONL records and generated summary; raw records remain diff --git a/benchmarks/discover-call/src/io.mjs b/benchmarks/discover-call/src/io.mjs index 9ec97de..39e1373 100644 --- a/benchmarks/discover-call/src/io.mjs +++ b/benchmarks/discover-call/src/io.mjs @@ -39,6 +39,62 @@ export async function writeTextAtomic(path, content) { } } +export async function writeFileSetTransactional( + files, + { openFile = open, renameFile = rename, unlinkFile = unlink } = {}, +) { + if ( + !Array.isArray(files) || + files.length < 2 || + files.some( + (file) => + !file || + typeof file.path !== 'string' || + !file.path || + typeof file.content !== 'string', + ) + ) { + throw new Error('Transactional output requires at least two path/content files'); + } + if (new Set(files.map((file) => resolve(file.path))).size !== files.length) { + throw new Error('Transactional output paths must be unique'); + } + + const transactionId = `${process.pid}.${randomUUID()}`; + const entries = files.map((file) => ({ + path: file.path, + content: file.content, + temporaryPath: `${file.path}.${transactionId}.tmp`, + backupPath: `${file.path}.${transactionId}.bak`, + originalMoved: false, + installed: false, + })); + + try { + for (const entry of entries) { + await writeStagedText(entry.temporaryPath, entry.content, openFile, unlinkFile); + } + for (const entry of entries) { + try { + await renameFile(entry.path, entry.backupPath); + entry.originalMoved = true; + } catch (error) { + if (error?.code !== 'ENOENT') throw error; + } + await renameFile(entry.temporaryPath, entry.path); + entry.installed = true; + } + } catch (error) { + const rollbackErrors = await rollbackFileSet(entries, { renameFile, unlinkFile }); + if (rollbackErrors.length > 0) { + throw new AggregateError([error, ...rollbackErrors], 'Transactional output failed and rollback was incomplete'); + } + throw error; + } + + await Promise.all(entries.filter((entry) => entry.originalMoved).map((entry) => unlinkFile(entry.backupPath))); +} + export async function validatePathSeparation({ inputs, outputs }) { if (!Array.isArray(inputs) || !Array.isArray(outputs) || outputs.length === 0) { throw new Error('Path validation needs input and output arrays'); @@ -59,6 +115,48 @@ export async function validatePathSeparation({ inputs, outputs }) { } } +async function writeStagedText(path, content, openFile, unlinkFile) { + let handle; + try { + handle = await openFile(path, 'wx', 0o600); + await handle.writeFile(content, { encoding: 'utf8' }); + await handle.sync(); + await handle.close(); + handle = undefined; + } catch (error) { + await handle?.close().catch(() => {}); + await unlinkFile(path).catch(() => {}); + throw error; + } +} + +async function rollbackFileSet(entries, { renameFile, unlinkFile }) { + const errors = []; + for (const entry of [...entries].reverse()) { + if (entry.installed) { + try { + await unlinkFile(entry.path); + } catch (error) { + if (error?.code !== 'ENOENT') errors.push(error); + } + } + if (entry.originalMoved) { + try { + await renameFile(entry.backupPath, entry.path); + entry.originalMoved = false; + } catch (error) { + errors.push(error); + } + } + try { + await unlinkFile(entry.temporaryPath); + } catch (error) { + if (error?.code !== 'ENOENT') errors.push(error); + } + } + return errors; +} + async function fileIdentities(path) { const absolute = resolve(path); const identities = new Set([`path:${absolute}`]); diff --git a/benchmarks/discover-call/src/publish.mjs b/benchmarks/discover-call/src/publish.mjs index 5b6a689..19fcd6a 100644 --- a/benchmarks/discover-call/src/publish.mjs +++ b/benchmarks/discover-call/src/publish.mjs @@ -5,7 +5,7 @@ import { mkdir, readFile } from 'node:fs/promises'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { readJsonLines, validatePathSeparation, writeJsonLines, writeTextAtomic } from './io.mjs'; +import { readJsonLines, validatePathSeparation, writeFileSetTransactional } from './io.mjs'; import { sanitizePublicRecords, validateOfficialPublicRun, validatePublicRecords } from './publication.mjs'; import { scoreRecords } from './scoring.mjs'; @@ -39,8 +39,16 @@ export async function main(argv = process.argv.slice(2)) { mkdir(dirname(outputRuns), { recursive: true }), mkdir(dirname(outputSummary), { recursive: true }), ]); - await writeJsonLines(outputRuns, publicRecords); - await writeTextAtomic(outputSummary, JSON.stringify(summary, null, 2) + '\n'); + await writeFileSetTransactional([ + { + path: outputRuns, + content: publicRecords.map((record) => JSON.stringify(record)).join('\n') + '\n', + }, + { + path: outputSummary, + content: JSON.stringify(summary, null, 2) + '\n', + }, + ]); process.stdout.write(`Wrote ${publicRecords.length} sanitized records and summary\n`); } diff --git a/benchmarks/discover-call/test/io.test.mjs b/benchmarks/discover-call/test/io.test.mjs index f9b87b9..b49204f 100644 --- a/benchmarks/discover-call/test/io.test.mjs +++ b/benchmarks/discover-call/test/io.test.mjs @@ -1,10 +1,15 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { link, mkdtemp, readFile, readdir, rm, stat } from 'node:fs/promises'; +import { link, mkdtemp, readFile, readdir, rename, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { validatePathSeparation, writeJsonLines, writeTextAtomic } from '../src/io.mjs'; +import { + validatePathSeparation, + writeFileSetTransactional, + writeJsonLines, + writeTextAtomic, +} from '../src/io.mjs'; test('atomically replaces private benchmark output files', async () => { const directory = await mkdtemp(join(tmpdir(), 'qveris-benchmark-io-')); @@ -27,6 +32,61 @@ test('atomically replaces private benchmark output files', async () => { } }); +test('transactionally replaces a public runs and summary pair', async () => { + const directory = await mkdtemp(join(tmpdir(), 'qveris-benchmark-pair-')); + const runsPath = join(directory, 'runs.jsonl'); + const summaryPath = join(directory, 'summary.json'); + try { + await writeTextAtomic(runsPath, 'old-runs\n'); + await writeTextAtomic(summaryPath, 'old-summary\n'); + await writeFileSetTransactional([ + { path: runsPath, content: 'new-runs\n' }, + { path: summaryPath, content: 'new-summary\n' }, + ]); + + assert.equal(await readFile(runsPath, 'utf8'), 'new-runs\n'); + assert.equal(await readFile(summaryPath, 'utf8'), 'new-summary\n'); + assert.deepEqual((await readdir(directory)).sort(), ['runs.jsonl', 'summary.json']); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test('restores the old public pair when the second replacement fails', async () => { + const directory = await mkdtemp(join(tmpdir(), 'qveris-benchmark-pair-rollback-')); + const runsPath = join(directory, 'runs.jsonl'); + const summaryPath = join(directory, 'summary.json'); + try { + await writeTextAtomic(runsPath, 'old-runs\n'); + await writeTextAtomic(summaryPath, 'old-summary\n'); + await assert.rejects( + writeFileSetTransactional( + [ + { path: runsPath, content: 'new-runs\n' }, + { path: summaryPath, content: 'new-summary\n' }, + ], + { + async renameFile(source, destination) { + if (source.endsWith('.tmp') && destination === summaryPath) { + const error = new Error('simulated second install failure'); + error.code = 'EIO'; + throw error; + } + await rename(source, destination); + }, + }, + ), + /simulated second install failure/, + ); + + assert.equal(await readFile(runsPath, 'utf8'), 'old-runs\n'); + assert.equal(await readFile(summaryPath, 'utf8'), 'old-summary\n'); + assert.deepEqual((await readdir(directory)).sort(), ['runs.jsonl', 'summary.json']); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + test('rejects outputs that alias an input path or inode', async () => { const directory = await mkdtemp(join(tmpdir(), 'qveris-benchmark-paths-')); const input = join(directory, 'tasks.jsonl'); From de7c3ae74f78f8990076d25b4b89ff552e89c84d Mon Sep 17 00:00:00 2001 From: Linfang Wang Date: Fri, 24 Jul 2026 12:51:35 +0800 Subject: [PATCH 19/19] benchmark: publish corrected v2 baseline --- ...07-24-gpt-5.6-sol-configured-v4.runs.jsonl | 54 + ...-24-gpt-5.6-sol-configured-v4.summary.json | 2036 +++++++++++++++++ .../2026-07-24-reference-v1-v4.runs.jsonl | 54 + .../2026-07-24-reference-v1-v4.summary.json | 2034 ++++++++++++++++ benchmarks/discover-call/results/README.md | 71 + docs/en-US/discover-call-benchmark.md | 45 +- docs/zh-CN/discover-call-benchmark.md | 36 +- 7 files changed, 4323 insertions(+), 7 deletions(-) create mode 100644 benchmarks/discover-call/results/2026-07-24-gpt-5.6-sol-configured-v4.runs.jsonl create mode 100644 benchmarks/discover-call/results/2026-07-24-gpt-5.6-sol-configured-v4.summary.json create mode 100644 benchmarks/discover-call/results/2026-07-24-reference-v1-v4.runs.jsonl create mode 100644 benchmarks/discover-call/results/2026-07-24-reference-v1-v4.summary.json diff --git a/benchmarks/discover-call/results/2026-07-24-gpt-5.6-sol-configured-v4.runs.jsonl b/benchmarks/discover-call/results/2026-07-24-gpt-5.6-sol-configured-v4.runs.jsonl new file mode 100644 index 0000000..9e2d9ac --- /dev/null +++ b/benchmarks/discover-call/results/2026-07-24-gpt-5.6-sol-configured-v4.runs.jsonl @@ -0,0 +1,54 @@ +{"schema_version":1,"benchmark_version":"v2","run_id":"455e8f6e-2e69-4cc7-8e96-b1e94ad9b465","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":1,"started_at":"2026-07-24T04:09:48.336Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d0ee0051eac08420f0c62a77a2f4b97572faf390cb1325fccee36ebe558aa923","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:10:13.649Z","source_record_sha256":"c16ac56dde6fc2a1472dd2aba659add23a648cb0d67abd9d964a9968411b2a31"} +{"schema_version":1,"benchmark_version":"v2","run_id":"a770fde8-d591-4cdd-8048-d76d84df5f9a","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":2,"started_at":"2026-07-24T04:10:13.659Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d0ee0051eac08420f0c62a77a2f4b97572faf390cb1325fccee36ebe558aa923","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:10:39.775Z","source_record_sha256":"f45519ffaa39e6a0ec5363ca71a84f635683531dcbad8e05dac59e570fb9da89"} +{"schema_version":1,"benchmark_version":"v2","run_id":"c0a4fc1a-7403-4d99-847b-159b4ee0152e","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":3,"started_at":"2026-07-24T04:10:39.782Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d0ee0051eac08420f0c62a77a2f4b97572faf390cb1325fccee36ebe558aa923","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:11:10.558Z","source_record_sha256":"2da27788524e4c41b7032df3d32bc4339de915b65894a606deb6bc97cf9d3edf"} +{"schema_version":1,"benchmark_version":"v2","run_id":"d34b7123-74cd-4070-9f40-3349ee12e39c","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-24T04:11:10.567Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"789a6d764e491fb5c59b5d721dd0d6a3098287a31c0f804a68e2ad29ff3c7857","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:11:38.263Z","source_record_sha256":"cefa8343eadf471f8427bedf5bd24a875951d00ecc2393114aec0f2b4c2639a9"} +{"schema_version":1,"benchmark_version":"v2","run_id":"47157200-609c-4443-9bb1-2d4f820fae82","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-24T04:11:38.269Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"09762a6dca4789f975ab0f90c42fa01cea412305bd50d451beb451d1428da6f6","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:12:07.437Z","source_record_sha256":"2455541e225007f62095c6b0a3180a44ae749132a93ea5eafafd7b76850a177c"} +{"schema_version":1,"benchmark_version":"v2","run_id":"b4aaa50a-a69e-43f5-b29f-c85e82119f32","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-24T04:12:07.443Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"09762a6dca4789f975ab0f90c42fa01cea412305bd50d451beb451d1428da6f6","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:12:48.440Z","source_record_sha256":"cb7e30501d0168f0d12020b38144ace0331717c3b777d65253222ab14d1417b9"} +{"schema_version":1,"benchmark_version":"v2","run_id":"9135937c-fb0b-41d1-8a08-654f0b8ff7fb","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-24T04:12:48.449Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"af3e994c19575bb543d3f3c752988c893ff54a87ade39849fa3a6cdc42e55bed","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:13:13.695Z","source_record_sha256":"5328569d6a69bbdd96524b649f53f132f86d2c3cab15a02ddb00a8d58251ba94"} +{"schema_version":1,"benchmark_version":"v2","run_id":"7f7ded00-01d6-4e2e-a9e8-8c6c1268701e","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-24T04:13:13.705Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"af3e994c19575bb543d3f3c752988c893ff54a87ade39849fa3a6cdc42e55bed","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:13:41.319Z","source_record_sha256":"7c4ea1e8f3bad6035b85c85f99cc19e77ed64e4c1f532cd9f082379699bc1b7d"} +{"schema_version":1,"benchmark_version":"v2","run_id":"491ff401-58cc-4ada-b878-60b5cae3a199","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-24T04:13:41.328Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"af3e994c19575bb543d3f3c752988c893ff54a87ade39849fa3a6cdc42e55bed","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:14:11.050Z","source_record_sha256":"661a22ace621b5ed261a178080209016567bb39a01e6cc5acbb76710695a1411"} +{"schema_version":1,"benchmark_version":"v2","run_id":"919603ef-8b77-4d1a-ab16-e5ae181a7b7d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":1,"started_at":"2026-07-24T04:14:11.058Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"ad985521c1ad3443f4fed2ec0c8e15879c240574db99adc9f5f24fc71ad99571","selection_grounded":true},"selection":{"tool_id":null,"tool_id_sha256":"6ac4128c9a87575059e577a383d690713e1f37d6cbee3261cfca8bb28d7cd463"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:14:54.309Z","source_record_sha256":"7b19843696e57e87927e570fdcddd03693ae7689486f0948ddd711a0eacc56db"} +{"schema_version":1,"benchmark_version":"v2","run_id":"c4b72e5f-9003-4f8b-bd7e-a8f32af46737","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":2,"started_at":"2026-07-24T04:14:54.315Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"000664a0813cb41ce6d4964b0bb24cfafa71b63d9cb6f42e9311507a6530f4ed","selection_grounded":true},"selection":{"tool_id":null,"tool_id_sha256":"6ac4128c9a87575059e577a383d690713e1f37d6cbee3261cfca8bb28d7cd463"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:15:29.692Z","source_record_sha256":"8f4179b9fef1aae515cd90d532e57f5c2f1a344a82f35841d87c50c897c25874"} +{"schema_version":1,"benchmark_version":"v2","run_id":"cd825fb1-e091-4a7a-bcb3-5efbaaed2369","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":3,"started_at":"2026-07-24T04:15:29.701Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"000664a0813cb41ce6d4964b0bb24cfafa71b63d9cb6f42e9311507a6530f4ed","selection_grounded":true},"selection":{"tool_id":null,"tool_id_sha256":"6ac4128c9a87575059e577a383d690713e1f37d6cbee3261cfca8bb28d7cd463"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:15:52.757Z","source_record_sha256":"4618c35df28f33cf299404e9977525eea9777a53ec451814ce3c894cdc030ef9"} +{"schema_version":1,"benchmark_version":"v2","run_id":"710d7e00-9cde-4927-97cc-6516e6b270de","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":1,"started_at":"2026-07-24T04:15:52.767Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:16:17.212Z","source_record_sha256":"bd1d202cdd98115284ea4f4f51575af41bf3938820ac2b392745bcd20197d171"} +{"schema_version":1,"benchmark_version":"v2","run_id":"d8ebb80c-1183-4a3b-90c2-e8901d1bcd0b","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":2,"started_at":"2026-07-24T04:16:17.219Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:16:39.599Z","source_record_sha256":"22a4a0e9eea36be0f78bb346dc40b7d15824e048ee72bdee07af86f6392c29c6"} +{"schema_version":1,"benchmark_version":"v2","run_id":"c12ff489-38a8-43ff-9c12-ae584d20a9fb","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":3,"started_at":"2026-07-24T04:16:39.605Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6f49509de707f85cab78e3df391add2e436a1dc27da75654026ffc51bba83826","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:17:11.794Z","source_record_sha256":"8af0471d3810ce716f12e79bd16cd7fb05d27654c8219c8f5001423bb54df47b"} +{"schema_version":1,"benchmark_version":"v2","run_id":"a1523c18-3d86-4ed7-a0ba-e23531e09efc","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-24T04:17:11.808Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"60190a396d1b43ec99d676ca46b3ac7d7ba69fdc6959c748b6654a0713ca3a82","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:17:37.541Z","source_record_sha256":"e7fb03c28e25cc9b009eb2e6091d1cbedea7a41f7082c5a90b74a2e161db3e5f"} +{"schema_version":1,"benchmark_version":"v2","run_id":"400f56f4-6866-4333-8e7c-b53e67dd2248","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-24T04:17:37.548Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"60190a396d1b43ec99d676ca46b3ac7d7ba69fdc6959c748b6654a0713ca3a82","selection_grounded":true},"selection":{"tool_id":null,"tool_id_sha256":"ccdb8d35111102def22650a2fa05c73864d05eb1e2c61b61431c23f6f23944f1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:18:08.541Z","source_record_sha256":"835b3edbaff5e6ceae865390381f26d5fa1f4d99fcfced8d61c542616d1300c9"} +{"schema_version":1,"benchmark_version":"v2","run_id":"a33968df-64b7-45c4-be58-e47938e4dc3f","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-24T04:18:08.546Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"60190a396d1b43ec99d676ca46b3ac7d7ba69fdc6959c748b6654a0713ca3a82","selection_grounded":true},"selection":{"tool_id":"api_sports.timezone.retrieve.v1.e993615d"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:18:39.423Z","source_record_sha256":"d9760e473fbab1d702b4f9f428e2293168c9eb54240229d4ed0fcc051dd66181"} +{"schema_version":1,"benchmark_version":"v2","run_id":"b3e3440a-aa4d-40df-bfaa-bb2cbd199d2d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":1,"started_at":"2026-07-24T04:18:39.429Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6e4f037886f5960652181897f43d1042a45c691e072d95b5f1a36755cceb3599","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:19:18.331Z","source_record_sha256":"af73c147d1e642bbe1cbd9240120f5a3b6eeb620bf7a24fc8602b529a738fbb3"} +{"schema_version":1,"benchmark_version":"v2","run_id":"0200c436-efa3-45d4-8071-cd0a69992fb9","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":2,"started_at":"2026-07-24T04:19:18.336Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6e4f037886f5960652181897f43d1042a45c691e072d95b5f1a36755cceb3599","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:19:47.720Z","source_record_sha256":"5080d750ea2a2587cf1a7c8ac875f56f8704a4af598c656d131b6b48820fef1f"} +{"schema_version":1,"benchmark_version":"v2","run_id":"95525bcf-8520-466a-bd96-0e6409e65d21","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":3,"started_at":"2026-07-24T04:19:47.732Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"6e4f037886f5960652181897f43d1042a45c691e072d95b5f1a36755cceb3599","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:20:12.640Z","source_record_sha256":"1b4d67570071c3e124577ee99bbc3a3118d90f27662815b218dbf3ada08a4690"} +{"schema_version":1,"benchmark_version":"v2","run_id":"afe48b93-6490-4229-a0af-647ac15b7dae","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-24T04:20:12.648Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"290f5403764c776eeca23357a68009854005dea5f41b9810144cb1f3731a9d3e","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:20:36.690Z","source_record_sha256":"a615782884df3613a3bbd594d7ac1629e95a63d09a74d3cdeaa6ad168139192e"} +{"schema_version":1,"benchmark_version":"v2","run_id":"231ff0aa-b81f-4cb7-9ba9-1639900f380b","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-24T04:20:36.695Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"290f5403764c776eeca23357a68009854005dea5f41b9810144cb1f3731a9d3e","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:21:05.696Z","source_record_sha256":"f82f14737b98174260d3f61a46df78eef25055b5dab05d9197af2513037cffaf"} +{"schema_version":1,"benchmark_version":"v2","run_id":"5a1e2c16-a5a2-4c35-894b-97693b31ce50","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-24T04:21:05.702Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"290f5403764c776eeca23357a68009854005dea5f41b9810144cb1f3731a9d3e","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:21:30.600Z","source_record_sha256":"4f1661474ebb474ebb1a5f6b38129f4487da959e3cb76d626d0f067e18774ab8"} +{"schema_version":1,"benchmark_version":"v2","run_id":"91977029-da2a-46b5-b3c7-1820ad93cee2","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":1,"started_at":"2026-07-24T04:21:30.608Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"322da1c141c4538e374f4e36f7473e4f55cdde54fce11efb00778057d59094de","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:21:58.161Z","source_record_sha256":"d09c7b362519c16a48a8ba84935c2666b8ab8e093e2af25b1924d8ff12659e3b"} +{"schema_version":1,"benchmark_version":"v2","run_id":"6d2b2052-d419-4250-8b06-e61ea55d6228","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":2,"started_at":"2026-07-24T04:21:58.171Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"322da1c141c4538e374f4e36f7473e4f55cdde54fce11efb00778057d59094de","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:22:21.038Z","source_record_sha256":"ba6a20f5ecf92c6f8b6b038a6fc484a3b9aaff665eb4ad3aab140fdde295ba72"} +{"schema_version":1,"benchmark_version":"v2","run_id":"cc29ee36-d3ed-4577-b949-c61d3e4d3b06","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":3,"started_at":"2026-07-24T04:22:21.046Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"322da1c141c4538e374f4e36f7473e4f55cdde54fce11efb00778057d59094de","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:22:48.582Z","source_record_sha256":"83f78d77ef03ea1a642e2bee92eaf9cba7b30d0ae712139ecfb71d06bcac98cd"} +{"schema_version":1,"benchmark_version":"v2","run_id":"009af626-2704-490d-b597-285e182b0c7b","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":1,"started_at":"2026-07-24T04:22:48.593Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"a8eeb4c87b54497cdf66ef397ebcd20f27f3cdefa2200878f8fc6666d26bc407","selection_grounded":true},"selection":{"tool_id":null,"tool_id_sha256":"3493c7908865b99e36e316dd181fd2b65ba3df70ad65b3964bd5b0a22eb93bf0"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:23:15.793Z","source_record_sha256":"5711535d4fee9b8ae4c95bb652db6d0ddb7c73256a893c8f9e689204496ee874"} +{"schema_version":1,"benchmark_version":"v2","run_id":"d9ebcb2b-b5d6-4640-aa18-7fe476080246","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":2,"started_at":"2026-07-24T04:23:15.799Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"a8eeb4c87b54497cdf66ef397ebcd20f27f3cdefa2200878f8fc6666d26bc407","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:23:45.782Z","source_record_sha256":"b87d132b543df7b2c9d2984c6e4438927ea005accb4860ba2f57b73c42785ca9"} +{"schema_version":1,"benchmark_version":"v2","run_id":"00197aa1-8b77-489a-bce8-126a4a57ca16","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":3,"started_at":"2026-07-24T04:23:45.787Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"a8eeb4c87b54497cdf66ef397ebcd20f27f3cdefa2200878f8fc6666d26bc407","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:24:24.176Z","source_record_sha256":"0ad4e974b9355933ca054cf90c6f3b08d223437e65e1edfa0045531783c0f906"} +{"schema_version":1,"benchmark_version":"v2","run_id":"6c134017-eebe-47df-a7f1-2545dcd51abf","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":1,"started_at":"2026-07-24T04:24:24.184Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:25:00.534Z","source_record_sha256":"de4630d61d0d385aff5a96e48e419694d9ba6dfefdd5463255dc7409aca6f884"} +{"schema_version":1,"benchmark_version":"v2","run_id":"c0835c55-1355-48bb-b120-89ed5e9d180d","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":2,"started_at":"2026-07-24T04:25:00.545Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:25:22.140Z","source_record_sha256":"7aa3902061f4a7149b26828b2bcc705409dd78e3210d3dc98f75e364861afd55"} +{"schema_version":1,"benchmark_version":"v2","run_id":"6bd07aed-4515-4ee2-8327-d6182bbc45d0","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":3,"started_at":"2026-07-24T04:25:22.149Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:25:47.481Z","source_record_sha256":"5828f9995d61bdb7121db9e96a3e2475c0ba1a20a085a030f34c44a8278ed634"} +{"schema_version":1,"benchmark_version":"v2","run_id":"1e6c409b-8cb5-47fb-9b8b-5e4f572178e4","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":1,"started_at":"2026-07-24T04:25:47.492Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"33ba73bb29b47ae3cfb6f376fdd2187ba20a4a4fd63d5177a1b7de691645f0d7","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:26:09.845Z","source_record_sha256":"33f622538290d039a33a84b7497984cf7deecbc78ca73d3ddb832b69c92c7ea8"} +{"schema_version":1,"benchmark_version":"v2","run_id":"d7d1f8bd-c2fd-4f43-b5d2-6d310815fbe1","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":2,"started_at":"2026-07-24T04:26:09.856Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"33ba73bb29b47ae3cfb6f376fdd2187ba20a4a4fd63d5177a1b7de691645f0d7","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:26:35.960Z","source_record_sha256":"d825cd89fe5aa6e419210426369a47327081019cf2588cc565fcda345a7ce788"} +{"schema_version":1,"benchmark_version":"v2","run_id":"d3a7aca2-c0b0-4bcb-95f9-b8f7f9fd7298","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":3,"started_at":"2026-07-24T04:26:35.966Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"33ba73bb29b47ae3cfb6f376fdd2187ba20a4a4fd63d5177a1b7de691645f0d7","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:27:04.941Z","source_record_sha256":"347cb1f1d19e61d372bdbb3057ece1db7cf2feaf2b8fa5772101feccefed8b5d"} +{"schema_version":1,"benchmark_version":"v2","run_id":"850f04e6-bfb6-4437-9a24-ec405751a640","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-24T04:27:04.954Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"3ed2d45dea90f4613325a4f8dc66c9d35d6fb3fa7dfc5f4d1877cca3f97e2658","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0},"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected"},"finished_at":"2026-07-24T04:27:54.841Z","source_record_sha256":"2575f89b7fab619b52b355095ed03b96b404c2da32b46538b96b86e8be30073a"} +{"schema_version":1,"benchmark_version":"v2","run_id":"f36d4b46-e116-4097-b0f6-0b8f132b618c","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":2,"started_at":"2026-07-24T04:27:54.850Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"3ed2d45dea90f4613325a4f8dc66c9d35d6fb3fa7dfc5f4d1877cca3f97e2658","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0},"call":{"attempted":false,"success":null},"error":{"stage":"adapter","reason_code":"tool_use_rejected"},"finished_at":"2026-07-24T04:28:48.327Z","source_record_sha256":"623f2da9eca4bb6e293080b4ccfdd76cdc6993d25bad1dfbe730b875c9732218"} +{"schema_version":1,"benchmark_version":"v2","run_id":"e5034b26-4de0-4c3b-93d4-067a04a089af","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-24T04:28:48.380Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3ed2d45dea90f4613325a4f8dc66c9d35d6fb3fa7dfc5f4d1877cca3f97e2658","selection_grounded":true},"selection":{"tool_id":"dataforseo.domain_analytics.whois.domain_whois_overview.retrieve.v3.335940e8"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:29:27.706Z","source_record_sha256":"6d3a22603a9f0702bf892509b63f1fe85703dd2dd6a620e0ac17cc654e27e16f"} +{"schema_version":1,"benchmark_version":"v2","run_id":"7a658b96-e1a7-4103-b8f0-092b2d204e6f","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":1,"started_at":"2026-07-24T04:29:27.725Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3f4705c58394ff4e6210add57ab33c71c4f240c26dc1e72acab02720d8508018","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:29:55.677Z","source_record_sha256":"3f8c98e51b4e9334e7456b5d2ab6d02591c4da8c8874ed4d0f2198c8826b70d8"} +{"schema_version":1,"benchmark_version":"v2","run_id":"6d1313ec-f23b-4edd-8127-a647f62026c0","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":2,"started_at":"2026-07-24T04:29:55.683Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3f4705c58394ff4e6210add57ab33c71c4f240c26dc1e72acab02720d8508018","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:30:23.350Z","source_record_sha256":"30f64f94e1ff57189b4c94f491f76a757fad6610707651d2b5edcb1333809c31"} +{"schema_version":1,"benchmark_version":"v2","run_id":"4e23ec45-7377-4e4b-ab62-9bc2326f1a85","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":3,"started_at":"2026-07-24T04:30:23.375Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3f4705c58394ff4e6210add57ab33c71c4f240c26dc1e72acab02720d8508018","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:33:14.048Z","source_record_sha256":"7bfe21815dd43a854c37478061f4d7edcb0905174124bb81107e46647202e939"} +{"schema_version":1,"benchmark_version":"v2","run_id":"d395d080-42fd-488b-9518-5a7410b18842","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":1,"started_at":"2026-07-24T04:33:14.055Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"553e6e7b44455f2c130f50f4bd43e2ae5c1c7625da3421e59640ba60fb6bed64","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:33:45.340Z","source_record_sha256":"bb92d271451049f1c37eb19d4fda2bd692a10211ba652f028865e065b0cb8719"} +{"schema_version":1,"benchmark_version":"v2","run_id":"f5b6f0f2-4482-492c-a2bf-90ee53c6172a","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":2,"started_at":"2026-07-24T04:33:45.350Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"553e6e7b44455f2c130f50f4bd43e2ae5c1c7625da3421e59640ba60fb6bed64","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:34:09.580Z","source_record_sha256":"0bfe7deae7e95dba405c343783c04974c7b3a53ee40e0c877bbeee6d026bee9e"} +{"schema_version":1,"benchmark_version":"v2","run_id":"9ca0decc-e710-4af1-9707-a8f4db1eeb44","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":3,"started_at":"2026-07-24T04:34:09.585Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"553e6e7b44455f2c130f50f4bd43e2ae5c1c7625da3421e59640ba60fb6bed64","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:34:36.909Z","source_record_sha256":"a2b026d6c56c30294f0326756d94cacb8ff222b0c2dc9ada05c9038de86b0baf"} +{"schema_version":1,"benchmark_version":"v2","run_id":"94e68a65-5dcc-4cdf-9600-5d7eaea79047","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-24T04:34:36.919Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:35:07.798Z","source_record_sha256":"b643962365cc0f51df680e06e8af404ac10b049f6e7bde59c0dc7cf3a7a24b64"} +{"schema_version":1,"benchmark_version":"v2","run_id":"46d9fc2f-e71d-4ef1-8fe5-eae7f8c9fe01","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-24T04:35:07.804Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:35:44.021Z","source_record_sha256":"0f694e6b1265f0351692af53230b49f6ceaff3a330e573b1820a8569c7a63f53"} +{"schema_version":1,"benchmark_version":"v2","run_id":"9a097f03-8f74-4695-b598-d0834472a94c","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-24T04:35:44.029Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:36:23.273Z","source_record_sha256":"ba0e70eb2452e97661e632a63a6e1221176357595fc207004e43cd890e2c49a8"} +{"schema_version":1,"benchmark_version":"v2","run_id":"457a771d-1271-4114-a5f2-1c9bb6bdcea9","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":1,"started_at":"2026-07-24T04:36:23.282Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d38e5c6dbcda77133f433246cda53c67c73c059b7b8f850d39795d9fff7326e0","selection_grounded":true},"selection":{"tool_id":null,"tool_id_sha256":"b33a03c54130dd583ab9e831db279346c92685982f118aba3b0dd25b0eb54dc5"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:36:57.304Z","source_record_sha256":"5c7a2beae8d7ff58b8a07006e9a3b8026c9b4a405a11ccb046116fafe8516df0"} +{"schema_version":1,"benchmark_version":"v2","run_id":"dcc128d8-91e7-4474-8b74-6673f0ca30a7","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":2,"started_at":"2026-07-24T04:36:57.316Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d38e5c6dbcda77133f433246cda53c67c73c059b7b8f850d39795d9fff7326e0","selection_grounded":true},"selection":{"tool_id":null,"tool_id_sha256":"b33a03c54130dd583ab9e831db279346c92685982f118aba3b0dd25b0eb54dc5"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:37:26.283Z","source_record_sha256":"2bb026598792b2d8f8ee5ba7ed13a9e90f96b3f66379eb3e353792b23588309b"} +{"schema_version":1,"benchmark_version":"v2","run_id":"2fbc87cf-d39d-4b29-a4ff-41f99b4dde64","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":3,"started_at":"2026-07-24T04:37:26.293Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d38e5c6dbcda77133f433246cda53c67c73c059b7b8f850d39795d9fff7326e0","selection_grounded":true},"selection":{"tool_id":null,"tool_id_sha256":"b33a03c54130dd583ab9e831db279346c92685982f118aba3b0dd25b0eb54dc5"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:37:54.422Z","source_record_sha256":"b60568fee6ddf80c603dd3981086b5ae6f7f096b8dca5f7ce28cad152d21603a"} +{"schema_version":1,"benchmark_version":"v2","run_id":"6d4fd215-d3ed-417c-86d3-cdd7ba032315","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":1,"started_at":"2026-07-24T04:37:54.428Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e00caf1bfbf8cdf710d8d4334e89bd1bf283c58a2356412612f67c498eea1f9b","selection_grounded":true},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:38:23.766Z","source_record_sha256":"ac1c7f79deec38c27311765b58d0fab649c95592aacca1cb4e6797424e43945b"} +{"schema_version":1,"benchmark_version":"v2","run_id":"51c456e2-44a4-4896-99bf-b4776bcb17d1","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":2,"started_at":"2026-07-24T04:38:23.774Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e00caf1bfbf8cdf710d8d4334e89bd1bf283c58a2356412612f67c498eea1f9b","selection_grounded":true},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:38:58.779Z","source_record_sha256":"ff15dcb5e1dc230bbb2f956f72cc501a98c21483640992e30ef3b5d9e16a4bc1"} +{"schema_version":1,"benchmark_version":"v2","run_id":"1e40fe97-19e8-41d2-99e6-b0cc07397cb1","model":"gpt-5.6-sol","metadata":{"lane":"configured-model","model_revision":"unreported","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":3,"started_at":"2026-07-24T04:38:58.786Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e00caf1bfbf8cdf710d8d4334e89bd1bf283c58a2356412612f67c498eea1f9b","selection_grounded":true},"selection":{"tool_id":"api-football.standings.list.v3.76f7703e"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:39:20.340Z","source_record_sha256":"b3c54925ab24895f0fef03a809b0766e3c147dd6424b3d0e0765305f0bbb1958"} diff --git a/benchmarks/discover-call/results/2026-07-24-gpt-5.6-sol-configured-v4.summary.json b/benchmarks/discover-call/results/2026-07-24-gpt-5.6-sol-configured-v4.summary.json new file mode 100644 index 0000000..de1d71d --- /dev/null +++ b/benchmarks/discover-call/results/2026-07-24-gpt-5.6-sol-configured-v4.summary.json @@ -0,0 +1,2036 @@ +{ + "schema_version": 1, + "methodology": "discover-call-v2", + "generated_at": "2026-07-24T04:40:35.132Z", + "models": [ + { + "model": "gpt-5.6-sol", + "lane": "configured-model", + "tasks": 18, + "trials_per_task": 3, + "runs": 54, + "completed": 52, + "executed": 52, + "selection_grounded_rate": 1, + "inspection_grounded_rate": 1, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0.8889, + "call_success_rate": 1, + "result_nonempty_rate": 1, + "pre_result_gate_workflow_success_rate": 0.8889, + "workflow_success_rate": 0.8889, + "workflow_success_task_cluster_bootstrap_95": [ + 0.7222, + 1 + ], + "failures_by_stage": { + "parameterize": 4, + "adapter": 2 + }, + "failures_by_reason": { + "constraint_mismatch": 4, + "tool_use_rejected": 2 + } + } + ], + "records": [ + { + "run_id": "455e8f6e-2e69-4cc7-8e96-b1e94ad9b465", + "task_id": "weather-london", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "a770fde8-d591-4cdd-8048-d76d84df5f9a", + "task_id": "weather-london", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "c0a4fc1a-7403-4d99-847b-159b4ee0152e", + "task_id": "weather-london", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "d34b7123-74cd-4070-9f40-3349ee12e39c", + "task_id": "stock-aapl", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "47157200-609c-4443-9bb1-2d4f820fae82", + "task_id": "stock-aapl", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "b4aaa50a-a69e-43f5-b29f-c85e82119f32", + "task_id": "stock-aapl", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "9135937c-fb0b-41d1-8a08-654f0b8ff7fb", + "task_id": "exchange-usd-eur", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "7f7ded00-01d6-4e2e-a9e8-8c6c1268701e", + "task_id": "exchange-usd-eur", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "491ff401-58cc-4ada-b878-60b5cae3a199", + "task_id": "exchange-usd-eur", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "919603ef-8b77-4d1a-ab16-e5ae181a7b7d", + "task_id": "news-ai", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "c4b72e5f-9003-4f8b-bd7e-a8f32af46737", + "task_id": "news-ai", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "cd825fb1-e091-4a7a-bcb3-5efbaaed2369", + "task_id": "news-ai", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "710d7e00-9cde-4927-97cc-6516e6b270de", + "task_id": "papers-climate", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "d8ebb80c-1183-4a3b-90c2-e8901d1bcd0b", + "task_id": "papers-climate", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "c12ff489-38a8-43ff-9c12-ae584d20a9fb", + "task_id": "papers-climate", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "a1523c18-3d86-4ed7-a0ba-e23531e09efc", + "task_id": "timezone-tokyo", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": false, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "400f56f4-6866-4333-8e7c-b53e67dd2248", + "task_id": "timezone-tokyo", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": false, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "a33968df-64b7-45c4-be58-e47938e4dc3f", + "task_id": "timezone-tokyo", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": false, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "b3e3440a-aa4d-40df-bfaa-bb2cbd199d2d", + "task_id": "geocode-address", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "0200c436-efa3-45d4-8071-cd0a69992fb9", + "task_id": "geocode-address", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "95525bcf-8520-466a-bd96-0e6409e65d21", + "task_id": "geocode-address", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "afe48b93-6490-4229-a0af-647ac15b7dae", + "task_id": "ip-geolocation", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "231ff0aa-b81f-4cb7-9ba9-1639900f380b", + "task_id": "ip-geolocation", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "5a1e2c16-a5a2-4c35-894b-97693b31ce50", + "task_id": "ip-geolocation", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "91977029-da2a-46b5-b3c7-1820ad93cee2", + "task_id": "crypto-price", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "6d2b2052-d419-4250-8b06-e61ea55d6228", + "task_id": "crypto-price", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "cc29ee36-d3ed-4577-b949-c61d3e4d3b06", + "task_id": "crypto-price", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "009af626-2704-490d-b597-285e182b0c7b", + "task_id": "web-search", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "d9ebcb2b-b5d6-4640-aa18-7fe476080246", + "task_id": "web-search", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "00197aa1-8b77-489a-bce8-126a4a57ca16", + "task_id": "web-search", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "6c134017-eebe-47df-a7f1-2545dcd51abf", + "task_id": "air-quality", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "c0835c55-1355-48bb-b120-89ed5e9d180d", + "task_id": "air-quality", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "6bd07aed-4515-4ee2-8327-d6182bbc45d0", + "task_id": "air-quality", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "1e6c409b-8cb5-47fb-9b8b-5e4f572178e4", + "task_id": "market-news", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "d7d1f8bd-c2fd-4f43-b5d2-6d310815fbe1", + "task_id": "market-news", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "d3a7aca2-c0b0-4bcb-95f9-b8f7f9fd7298", + "task_id": "market-news", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "850f04e6-bfb6-4437-9a24-ec405751a640", + "task_id": "domain-intel", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": false, + "executed": false, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": null, + "result_nonempty": null, + "pre_result_gate_workflow_success": false, + "workflow_success": false, + "error_stage": "adapter", + "error_reason": "tool_use_rejected", + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "f36d4b46-e116-4097-b0f6-0b8f132b618c", + "task_id": "domain-intel", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": false, + "executed": false, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": null, + "result_nonempty": null, + "pre_result_gate_workflow_success": false, + "workflow_success": false, + "error_stage": "adapter", + "error_reason": "tool_use_rejected", + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "e5034b26-4de0-4c3b-93d4-067a04a089af", + "task_id": "domain-intel", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": false, + "workflow_success": false, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "7a658b96-e1a7-4103-b8f0-092b2d204e6f", + "task_id": "image-search", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "6d1313ec-f23b-4edd-8127-a647f62026c0", + "task_id": "image-search", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "4e23ec45-7377-4e4b-ab62-9bc2326f1a85", + "task_id": "image-search", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "d395d080-42fd-488b-9518-5a7410b18842", + "task_id": "market-holidays", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "f5b6f0f2-4482-492c-a2bf-90ee53c6172a", + "task_id": "market-holidays", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "9ca0decc-e710-4af1-9707-a8f4db1eeb44", + "task_id": "market-holidays", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "94e68a65-5dcc-4cdf-9600-5d7eaea79047", + "task_id": "earthquake-catalog", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "46d9fc2f-e71d-4ef1-8fe5-eae7f8c9fe01", + "task_id": "earthquake-catalog", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "9a097f03-8f74-4695-b598-d0834472a94c", + "task_id": "earthquake-catalog", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "457a771d-1271-4114-a5f2-1c9bb6bdcea9", + "task_id": "company-profile", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "dcc128d8-91e7-4474-8b74-6673f0ca30a7", + "task_id": "company-profile", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "2fbc87cf-d39d-4b29-a4ff-41f99b4dde64", + "task_id": "company-profile", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "6d4fd215-d3ed-417c-86d3-cdd7ba032315", + "task_id": "sports-standings", + "model": "gpt-5.6-sol", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "51c456e2-44a4-4896-99bf-b4776bcb17d1", + "task_id": "sports-standings", + "model": "gpt-5.6-sol", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "1e40fe97-19e8-41d2-99e6-b0cc07397cb1", + "task_id": "sports-standings", + "model": "gpt-5.6-sol", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "configured-model", + "model_revision": "unreported", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + } + ] +} diff --git a/benchmarks/discover-call/results/2026-07-24-reference-v1-v4.runs.jsonl b/benchmarks/discover-call/results/2026-07-24-reference-v1-v4.runs.jsonl new file mode 100644 index 0000000..f5cadd7 --- /dev/null +++ b/benchmarks/discover-call/results/2026-07-24-reference-v1-v4.runs.jsonl @@ -0,0 +1,54 @@ +{"schema_version":1,"benchmark_version":"v2","run_id":"0a929658-b437-44b2-abc3-6a2851ac251a","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":1,"started_at":"2026-07-24T04:04:47.356Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d0ee0051eac08420f0c62a77a2f4b97572faf390cb1325fccee36ebe558aa923","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:04:52.102Z","source_record_sha256":"1c149d6d4876a6bca58cbf91109775fd26397a4a937c1124291792e1158068a0"} +{"schema_version":1,"benchmark_version":"v2","run_id":"fe9b3456-fbdc-4a04-930c-0e6f0fbdc58a","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":2,"started_at":"2026-07-24T04:04:52.109Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d0ee0051eac08420f0c62a77a2f4b97572faf390cb1325fccee36ebe558aa923","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:04:55.840Z","source_record_sha256":"759c0e0618d451c1d03e9fdd73dd5b3087b8a47f15b625e6ec2662a4a2f0bb77"} +{"schema_version":1,"benchmark_version":"v2","run_id":"d18949e0-3fae-4b1e-a0e9-7d4b8c13b803","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"weather-london","trial":3,"started_at":"2026-07-24T04:04:55.846Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d0ee0051eac08420f0c62a77a2f4b97572faf390cb1325fccee36ebe558aa923","selection_grounded":true},"selection":{"tool_id":"visualcrossing.timeline.retrieve.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:04:59.183Z","source_record_sha256":"fd401259715b09c6b6cec22b3a511e0003546f6c5bc81c733d788662e16b4119"} +{"schema_version":1,"benchmark_version":"v2","run_id":"f75c5749-09a4-4325-939a-90e8edcaacc2","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":1,"started_at":"2026-07-24T04:04:59.188Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"789a6d764e491fb5c59b5d721dd0d6a3098287a31c0f804a68e2ad29ff3c7857","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:05:02.894Z","source_record_sha256":"51b6582c5d162785f5e36646bff14d0cec70badf954ba5e13e34e509fda7d325"} +{"schema_version":1,"benchmark_version":"v2","run_id":"9bd4b7c1-1766-42ff-8cd4-03f49cc46e95","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":2,"started_at":"2026-07-24T04:05:02.902Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"789a6d764e491fb5c59b5d721dd0d6a3098287a31c0f804a68e2ad29ff3c7857","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:05:06.679Z","source_record_sha256":"6ce8d2b956a8c26343569e5b69502cc602d63fbf3996908ae0e0e9ee4093b936"} +{"schema_version":1,"benchmark_version":"v2","run_id":"2643e262-2eb7-43af-a92e-51e759692643","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"stock-aapl","trial":3,"started_at":"2026-07-24T04:05:06.686Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"789a6d764e491fb5c59b5d721dd0d6a3098287a31c0f804a68e2ad29ff3c7857","selection_grounded":true},"selection":{"tool_id":"twelvedata.quote.retrieve.v1.affbefe3"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:05:09.722Z","source_record_sha256":"31db5cb2a142607e6d48722a7c7a53d44494a9460397ca7c9f489900c0b9ff09"} +{"schema_version":1,"benchmark_version":"v2","run_id":"190f5741-2976-4edb-a433-6029cc6c0a05","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":1,"started_at":"2026-07-24T04:05:09.731Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"cd25a024487adcfd340fedd1ae4e66d7cc3ab56e0bf9d2a9a96d81dda2d01653","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:05:12.497Z","source_record_sha256":"3e3d44b1dc5f405e6e78f797463b8ae97f9b167127c5d393683c28772c34aeb7"} +{"schema_version":1,"benchmark_version":"v2","run_id":"9f36d1e5-9134-4af1-8656-9c8ba39921b2","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":2,"started_at":"2026-07-24T04:05:12.508Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"cd25a024487adcfd340fedd1ae4e66d7cc3ab56e0bf9d2a9a96d81dda2d01653","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:05:15.346Z","source_record_sha256":"2a9376d218f875d8f690df3df44d68c17597628c83959587c04d01caadf7d9cc"} +{"schema_version":1,"benchmark_version":"v2","run_id":"e26110f8-0509-42c4-85a4-db0f48840dc4","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"exchange-usd-eur","trial":3,"started_at":"2026-07-24T04:05:15.354Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"cd25a024487adcfd340fedd1ae4e66d7cc3ab56e0bf9d2a9a96d81dda2d01653","selection_grounded":true},"selection":{"tool_id":"twelvedata.currencyconversion.retrieve.v1.60fe48d4"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:05:22.345Z","source_record_sha256":"12c4ccae17363e06ae0800608a121374ca6d87ccf375e2991f0c33082f888584"} +{"schema_version":1,"benchmark_version":"v2","run_id":"63939db0-f5e8-4511-9424-a7908b1ef178","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":1,"started_at":"2026-07-24T04:05:22.350Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e1762a99b050c284298ef7fc8185a95bd7fabb8adce6f179abdd64f6ef05614","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:05:31.233Z","source_record_sha256":"4a3690b8ab98b189d7aad2c8caf715d41030400b4a904a8e06971c14aacc1717"} +{"schema_version":1,"benchmark_version":"v2","run_id":"55892a25-c775-481f-bcf9-80f22db0b269","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":2,"started_at":"2026-07-24T04:05:31.240Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e1762a99b050c284298ef7fc8185a95bd7fabb8adce6f179abdd64f6ef05614","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:05:50.661Z","source_record_sha256":"14a3c3849152a84a914e843038b08b6702c940006b6c4333259eb6f3a3da68b4"} +{"schema_version":1,"benchmark_version":"v2","run_id":"eb15137b-5f73-442c-bceb-f4e0bb28aed5","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"news-ai","trial":3,"started_at":"2026-07-24T04:05:50.669Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e1762a99b050c284298ef7fc8185a95bd7fabb8adce6f179abdd64f6ef05614","selection_grounded":true},"selection":{"tool_id":"newsapi.everything.retrieve.v2.e77ab1df"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:06:04.865Z","source_record_sha256":"4caf35658d4d5b8ac2bd2fa3f090dfccf8828bdba55e4dccd2462c923a4221df"} +{"schema_version":1,"benchmark_version":"v2","run_id":"b2f10c44-67a2-4ab4-a8dd-18813c91aec2","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":1,"started_at":"2026-07-24T04:06:04.870Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e9eba15927b05f4c9785aeb71d02e4cfd6c4b08ce1682bf93e9186d4494fc657","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:06:18.510Z","source_record_sha256":"e1e5b4dac0b27e0bd657a8759dd883ec2111accd15fc0ba2310ea9c99d205564"} +{"schema_version":1,"benchmark_version":"v2","run_id":"87fdedb1-6bda-4ae4-8bfb-0dec587bb7ed","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":2,"started_at":"2026-07-24T04:06:18.515Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e9eba15927b05f4c9785aeb71d02e4cfd6c4b08ce1682bf93e9186d4494fc657","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:06:26.979Z","source_record_sha256":"d585ea5764ea4098ae8858f84e6c2d09251dfce9c2064be9acda564dbd53390f"} +{"schema_version":1,"benchmark_version":"v2","run_id":"6a1707b3-0496-4c1d-a74e-d925bb168882","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"papers-climate","trial":3,"started_at":"2026-07-24T04:06:26.989Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"e9eba15927b05f4c9785aeb71d02e4cfd6c4b08ce1682bf93e9186d4494fc657","selection_grounded":true},"selection":{"tool_id":"qveris_academic.paper_aggregation_v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:06:37.314Z","source_record_sha256":"032b730effdb2bf8157dd68fd703b0ef39948580d8718ebebf6d40783c9ff644"} +{"schema_version":1,"benchmark_version":"v2","run_id":"4937c77c-4c23-4a62-a60a-2a2d400dd502","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":1,"started_at":"2026-07-24T04:06:37.327Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"4f569ea548e7a80398d62551811b74981c0349d92bf482efc6e2a9236bf2fa9d","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0},"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id"},"finished_at":"2026-07-24T04:06:38.513Z","source_record_sha256":"458a5faf0c7989a1d7a36ec2f7e2bb9c6c3cd8606aaa2d5b6a1353c3166d71f1"} +{"schema_version":1,"benchmark_version":"v2","run_id":"e0d45a4d-2cab-4f24-a206-e31fbb0b2fbc","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":2,"started_at":"2026-07-24T04:06:38.517Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"4f569ea548e7a80398d62551811b74981c0349d92bf482efc6e2a9236bf2fa9d","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0},"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id"},"finished_at":"2026-07-24T04:06:39.608Z","source_record_sha256":"315f5ad081f787263c2c22b0fd952af3dcb56deb1477712b9e1fd64ad11964e2"} +{"schema_version":1,"benchmark_version":"v2","run_id":"d41af726-12aa-4a16-b2c4-1fe3c6209e22","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"timezone-tokyo","trial":3,"started_at":"2026-07-24T04:06:39.614Z","status":"failed","discovery":{"result_count":10,"snapshot_sha256":"4f569ea548e7a80398d62551811b74981c0349d92bf482efc6e2a9236bf2fa9d","selection_grounded":false},"selection":{"tool_id":null},"inspection":{"selection_grounded":false},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":0},"call":{"attempted":false,"success":null},"error":{"stage":"select","reason_code":"missing_tool_id"},"finished_at":"2026-07-24T04:06:40.780Z","source_record_sha256":"4d9faf1faf80526e116fd2f73f201323260147cf62924ab156481861b7783d84"} +{"schema_version":1,"benchmark_version":"v2","run_id":"cc3386ef-c067-42e0-b6c2-d289efb9ccbe","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":1,"started_at":"2026-07-24T04:06:40.786Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"29ff149a752b11194a4994b50191f41e23ba2eb2f9ba6d1dde78fb98d3f72b00","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:06:44.052Z","source_record_sha256":"3424c59d8cb2dbe94c149695023e812847f7da82d0feff14d8f28278b3bd4196"} +{"schema_version":1,"benchmark_version":"v2","run_id":"56fe8408-0c1e-4a04-a73b-aee950af9fb5","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":2,"started_at":"2026-07-24T04:06:44.062Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"29ff149a752b11194a4994b50191f41e23ba2eb2f9ba6d1dde78fb98d3f72b00","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:06:47.180Z","source_record_sha256":"4235bc8a790e338f5c68c1fe7ecae8c0c8ead7f87aa87f2105b3df6aa045f4b1"} +{"schema_version":1,"benchmark_version":"v2","run_id":"12e0e9a6-6bd7-412b-98e3-d7973badac8c","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"geocode-address","trial":3,"started_at":"2026-07-24T04:06:47.186Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"29ff149a752b11194a4994b50191f41e23ba2eb2f9ba6d1dde78fb98d3f72b00","selection_grounded":true},"selection":{"tool_id":"amap_webservice.geocode.geo.retrieve.v3"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:06:51.273Z","source_record_sha256":"8982d559a573e370680abe3517afc06063e213ba01065cd38bd128d114411b79"} +{"schema_version":1,"benchmark_version":"v2","run_id":"f15ccdde-8373-40fa-b664-73d50e65a819","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":1,"started_at":"2026-07-24T04:06:51.279Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"cc8ee75a8f11cb1eb7fc493855c333800cc94964e70af5ecca36757d04f6d4e9","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:06:55.173Z","source_record_sha256":"1cf2b4f8471f1446f8d51404682d243e0afde43359df55d5b0b4e9ce0d83020f"} +{"schema_version":1,"benchmark_version":"v2","run_id":"5f857a5b-9a5f-4ec8-85c4-50ede5b7472d","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":2,"started_at":"2026-07-24T04:06:55.182Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"cc8ee75a8f11cb1eb7fc493855c333800cc94964e70af5ecca36757d04f6d4e9","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:06:58.547Z","source_record_sha256":"1f201977e0dd0ced2ccc2ee7331d4207098867a01a2f0307ad1f869ed603de67"} +{"schema_version":1,"benchmark_version":"v2","run_id":"2b5c778e-f319-4d92-a6f0-7edc7a65c2d9","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"ip-geolocation","trial":3,"started_at":"2026-07-24T04:06:58.556Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"cc8ee75a8f11cb1eb7fc493855c333800cc94964e70af5ecca36757d04f6d4e9","selection_grounded":true},"selection":{"tool_id":"virustotal.ip.info.retrieve.v3.ab34b0d1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:07:02.595Z","source_record_sha256":"60b1b59fb3d1fe793a818ae711c735aa1327b74ff014f29d238b682ca9b26df2"} +{"schema_version":1,"benchmark_version":"v2","run_id":"350b386d-0e36-4970-b1d4-4188da565056","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":1,"started_at":"2026-07-24T04:07:02.602Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7b9e02da9971f5b62bec1b3b762c809081c2e0ddd1f283ed8a837dff7e4807f0","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:07:05.291Z","source_record_sha256":"fd9080c5b80aded324352235c3b0ed10592afb5f0353a4a74a180b5cd5d7163a"} +{"schema_version":1,"benchmark_version":"v2","run_id":"e32e3deb-54b1-428f-91bb-5e77f22bb813","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":2,"started_at":"2026-07-24T04:07:05.296Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7b9e02da9971f5b62bec1b3b762c809081c2e0ddd1f283ed8a837dff7e4807f0","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:07:07.894Z","source_record_sha256":"a6ed55ab7592c18b0b25dfcf6e7b7dbda915311884809b5a2b33aaf3bd4e23f7"} +{"schema_version":1,"benchmark_version":"v2","run_id":"531b08ec-c6f8-406b-b400-4453a68205b7","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"crypto-price","trial":3,"started_at":"2026-07-24T04:07:07.900Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"7b9e02da9971f5b62bec1b3b762c809081c2e0ddd1f283ed8a837dff7e4807f0","selection_grounded":true},"selection":{"tool_id":"binance.ticker.price.list.v3.8675eca0"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:07:10.497Z","source_record_sha256":"ac70a660305c93f3be18edb3e028284ddf8822599c4bd132723428f848b99194"} +{"schema_version":1,"benchmark_version":"v2","run_id":"dd2858ba-eb7e-4720-b79c-8677e5dce881","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":1,"started_at":"2026-07-24T04:07:10.503Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fe8399ce332ee68b4ddd51a2be5638df79a64becbb69eedfaaac2a38b5b9fe3e","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:07:13.866Z","source_record_sha256":"581f820194ea33e88b6207cd4047d0524de4aa53e55a1e035a954d5c8c388cc0"} +{"schema_version":1,"benchmark_version":"v2","run_id":"871b5609-875c-4303-8999-8ba516b46c32","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":2,"started_at":"2026-07-24T04:07:13.874Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fe8399ce332ee68b4ddd51a2be5638df79a64becbb69eedfaaac2a38b5b9fe3e","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:07:16.458Z","source_record_sha256":"1d51595360142fde82f9cf4bc7b96f110849dd34f42d97c36c2332539a1de450"} +{"schema_version":1,"benchmark_version":"v2","run_id":"b9a39493-2947-4650-b16d-eab986c4fabd","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"web-search","trial":3,"started_at":"2026-07-24T04:07:16.464Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"fe8399ce332ee68b4ddd51a2be5638df79a64becbb69eedfaaac2a38b5b9fe3e","selection_grounded":true},"selection":{"tool_id":"cloudsway.smart_search_overseas"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:07:19.570Z","source_record_sha256":"dd6b98b704c45b78f54fea92fed0c45b3fe612f3274391c5211d188708edde64"} +{"schema_version":1,"benchmark_version":"v2","run_id":"70e3b565-acb6-4ace-97c6-39512af06dc4","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":1,"started_at":"2026-07-24T04:07:19.578Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:07:22.094Z","source_record_sha256":"d559dda01f6167ac4cbe7697aabf241aca18653517e4427ab390c26769ce306b"} +{"schema_version":1,"benchmark_version":"v2","run_id":"595adcb7-410e-4462-8e0e-10c6d28d1783","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":2,"started_at":"2026-07-24T04:07:22.100Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:07:24.459Z","source_record_sha256":"2d05625c6c341b9cae11691808cfa40005d5e8994cd1d6ba18e92af1f639bb46"} +{"schema_version":1,"benchmark_version":"v2","run_id":"1e6f4c74-bde2-4e19-a9d9-feff3c1ce0a0","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"air-quality","trial":3,"started_at":"2026-07-24T04:07:24.464Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"9fd956758be691114b15537f836c625a9a96db3d781dfdbef2d3d36cd32dfd3e","selection_grounded":true},"selection":{"tool_id":"openweathermap.air_pollution.current.v2_5"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:07:27.895Z","source_record_sha256":"bb51471fb16c5be194158e900f5d6f96d9e04ecaafa9937d9d8aa8077d9b0e05"} +{"schema_version":1,"benchmark_version":"v2","run_id":"3f407e76-2b0a-4ad8-ab65-bfce16a0c388","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":1,"started_at":"2026-07-24T04:07:27.903Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d20624e1546a6e42dda035c03d4c0ab395ceb258e75941fb0c88909fef47544f","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:07:41.688Z","source_record_sha256":"6d91b058c67ffdc8fe146d4319430d76e6f4dc84acbda14c9ee86d1754d6bfd1"} +{"schema_version":1,"benchmark_version":"v2","run_id":"eb51c0fd-aaa0-46af-a4f0-d0e8710809c8","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":2,"started_at":"2026-07-24T04:07:41.697Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d20624e1546a6e42dda035c03d4c0ab395ceb258e75941fb0c88909fef47544f","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:07:45.107Z","source_record_sha256":"ed60adfaf48bd78977442a44b347c5e6d80db0e34d2b8ca120aeb0165954e8c2"} +{"schema_version":1,"benchmark_version":"v2","run_id":"11c7762e-b9b0-4567-a586-af7d0b242921","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-news","trial":3,"started_at":"2026-07-24T04:07:45.115Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"d20624e1546a6e42dda035c03d4c0ab395ceb258e75941fb0c88909fef47544f","selection_grounded":true},"selection":{"tool_id":"finnhub.market.news.list.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:07:49.393Z","source_record_sha256":"44f4380c82f41d8bb627369ba4190543122cafcbd11f46a9a9dbbac448087fea"} +{"schema_version":1,"benchmark_version":"v2","run_id":"59bc05f1-29c9-4448-8da7-5b7ba5883579","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":1,"started_at":"2026-07-24T04:07:49.400Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3ed2d45dea90f4613325a4f8dc66c9d35d6fb3fa7dfc5f4d1877cca3f97e2658","selection_grounded":true},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:07:52.778Z","source_record_sha256":"89fa40d5663921d0a9f309620b835408fb6f863f9b3d2cc941101112550ea862"} +{"schema_version":1,"benchmark_version":"v2","run_id":"7c0a548c-4fd4-4224-bd78-fb9a5d77f93b","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":2,"started_at":"2026-07-24T04:07:52.789Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3ed2d45dea90f4613325a4f8dc66c9d35d6fb3fa7dfc5f4d1877cca3f97e2658","selection_grounded":true},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:07:56.573Z","source_record_sha256":"69260f8816297fdb21ea8719fc2864df5b92b1577101aee0a7f5aa40567d048f"} +{"schema_version":1,"benchmark_version":"v2","run_id":"556779a9-caa5-4832-b46f-4cf0fe3f98f7","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"domain-intel","trial":3,"started_at":"2026-07-24T04:07:56.582Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"3ed2d45dea90f4613325a4f8dc66c9d35d6fb3fa7dfc5f4d1877cca3f97e2658","selection_grounded":true},"selection":{"tool_id":"virustotal.domains.report.retrieve.v3.c5459740"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:08:00.032Z","source_record_sha256":"6ed3577bc87ac010c39abf33a045cba72e5c1b8d23fe7904321f989419f8feab"} +{"schema_version":1,"benchmark_version":"v2","run_id":"0e930700-3c99-4997-ab3a-1860d4be5e27","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":1,"started_at":"2026-07-24T04:08:00.042Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"badbe6a505f7a1d8bbd319e522036d55a4c26b7f79535aefcf4c4e33656d02e4","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:08:04.962Z","source_record_sha256":"9a70a76a77a5cc4d06ca8b6650e4b633bc86b678c81901cf67c2fa5ab4624a12"} +{"schema_version":1,"benchmark_version":"v2","run_id":"82a85463-1048-4274-965b-cd60e55ef866","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":2,"started_at":"2026-07-24T04:08:04.969Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"badbe6a505f7a1d8bbd319e522036d55a4c26b7f79535aefcf4c4e33656d02e4","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:08:09.378Z","source_record_sha256":"aa00fd78ed6238ece8b48cb138f741a0327bf9016cfe5fb466484567c5536c2d"} +{"schema_version":1,"benchmark_version":"v2","run_id":"5c452f50-1723-4e6a-9293-96c604992466","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"image-search","trial":3,"started_at":"2026-07-24T04:08:09.384Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"badbe6a505f7a1d8bbd319e522036d55a4c26b7f79535aefcf4c4e33656d02e4","selection_grounded":true},"selection":{"tool_id":"brave_search.images.search.list.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:08:12.748Z","source_record_sha256":"a3133b9e3611e898dc321180058c59c2411b29bd60ec9bd76a69fd6fad42f8a6"} +{"schema_version":1,"benchmark_version":"v2","run_id":"5587cd4b-0703-4967-a6c7-0019195961fd","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":1,"started_at":"2026-07-24T04:08:12.756Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"69997c19a168d711188f930b4e9467ad4d53a8d47cab00eaaca8223a11a25961","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:08:16.398Z","source_record_sha256":"a1d1c4371621167c23849906ef3706c20ab1dd1b4a651ccf42c909fb6c18e198"} +{"schema_version":1,"benchmark_version":"v2","run_id":"bacfdb82-0045-4bdb-a25f-88cfc0df79bb","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":2,"started_at":"2026-07-24T04:08:16.406Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"69997c19a168d711188f930b4e9467ad4d53a8d47cab00eaaca8223a11a25961","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:08:20.400Z","source_record_sha256":"7a600d823d78a1041565ac70f5cc033371541ceed4d0dd430bb4c0e43917365b"} +{"schema_version":1,"benchmark_version":"v2","run_id":"08faf7ed-b96d-45d1-969c-5962138faa5e","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"market-holidays","trial":3,"started_at":"2026-07-24T04:08:20.407Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"69997c19a168d711188f930b4e9467ad4d53a8d47cab00eaaca8223a11a25961","selection_grounded":true},"selection":{"tool_id":"finnhub.stock.marketholiday.retrieve.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:08:23.705Z","source_record_sha256":"c7ee89906fc791305cf62c237f0dadb414bdd446a5a5c706809bc3248f310471"} +{"schema_version":1,"benchmark_version":"v2","run_id":"dfb0f281-4fd3-4f4e-85d6-a6ea65aa7904","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":1,"started_at":"2026-07-24T04:08:23.713Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:08:30.189Z","source_record_sha256":"36c3bcbd371523aae8095c060a2590bd98582005bb492d249f24ba8a3bfbf720"} +{"schema_version":1,"benchmark_version":"v2","run_id":"5c0c04ce-94ee-460a-b3d9-a6a78e173732","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":2,"started_at":"2026-07-24T04:08:30.196Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:08:37.393Z","source_record_sha256":"57f2115913b798ddbf9656811be494bb094e73ed6bc15c5dd33ed5d3b07a0fa3"} +{"schema_version":1,"benchmark_version":"v2","run_id":"e2b5de64-cb4a-45d7-ac1f-9d31381cec7c","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"earthquake-catalog","trial":3,"started_at":"2026-07-24T04:08:37.400Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"1e52f7609408728eebee5467bb751fa5f3e7e5de11f9ee2ed2f1448cfa6ffe28","selection_grounded":true},"selection":{"tool_id":"usgs_earthquake_catalog.event.events.list.v1"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:08:44.247Z","source_record_sha256":"0be2a1495ae141410b61c0a049b4d98e162b3ad90e89ab3cad9f3636b1a2f07d"} +{"schema_version":1,"benchmark_version":"v2","run_id":"6b18bea9-34ff-4792-81d8-ab1590b0b15a","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":1,"started_at":"2026-07-24T04:08:44.255Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"b8dd08606dfdbced3f76018d977e82d7706053c775429aa736ecf9f217c930c2","selection_grounded":true},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:08:47.563Z","source_record_sha256":"eaf759123234453d6effa216bf794dcd49b9521b3b7fbe8faa0c5b0706d0b911"} +{"schema_version":1,"benchmark_version":"v2","run_id":"87424a80-ffa0-40aa-9ff7-f9ac14468b8d","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":2,"started_at":"2026-07-24T04:08:47.572Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"b8dd08606dfdbced3f76018d977e82d7706053c775429aa736ecf9f217c930c2","selection_grounded":true},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:08:50.765Z","source_record_sha256":"2ad48bf72d030d847eb02329941605d8287a3ddfe53e997f926c43eb1a5b33fd"} +{"schema_version":1,"benchmark_version":"v2","run_id":"31356409-e707-44c5-8cfb-8f7426a71e74","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"company-profile","trial":3,"started_at":"2026-07-24T04:08:50.773Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"b8dd08606dfdbced3f76018d977e82d7706053c775429aa736ecf9f217c930c2","selection_grounded":true},"selection":{"tool_id":"financialmodelingprep.stable.industryclassificationsearch.retrieve.v1.177b5bc2"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:08:54.322Z","source_record_sha256":"c37affb2b892e7ee181e9ff4ee87dbe0cbca40c6d1f18a092e3027acbc5d9318"} +{"schema_version":1,"benchmark_version":"v2","run_id":"5c44fdf4-2b43-41ca-8af2-9dc6f427c411","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":1,"started_at":"2026-07-24T04:08:54.330Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"90be874755269c810ebe258206ad0629946a4fc1755c652cd738666606d29043","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:08:57.185Z","source_record_sha256":"857b28aca68f97e212726d48974de975f13cc3e274ca4bc1163e3fd587df9152"} +{"schema_version":1,"benchmark_version":"v2","run_id":"fd7e7714-3696-4542-9fd5-032a1c2a5821","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":2,"started_at":"2026-07-24T04:08:57.192Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"90be874755269c810ebe258206ad0629946a4fc1755c652cd738666606d29043","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:09:00.047Z","source_record_sha256":"39ddf980ebeaa169eaa93fcd741a921ba66ece7dd313ccd6a3fedfb6413df537"} +{"schema_version":1,"benchmark_version":"v2","run_id":"75e5a133-d0c4-491c-bbf8-7888d03fb6d0","model":"reference-v1","metadata":{"lane":"reference","model_revision":"deterministic-reference-v1","adapter_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1","toolkit_revision":"a7f2aa60ef143dbbb35eaf9006ed8123d778fb13","task_set_sha256":"afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed","api_base_url":"https://qveris.ai/api/v1","api_revision":"2026-07-23.2","catalog_revision":"unreported","catalog_observation_sha256":"7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76","runtime":{"node":"v24.2.0","platform":"darwin","arch":"arm64"},"discovery_limit":10,"execute":true,"publication_policy":"public-artifact-v1"},"task_id":"sports-standings","trial":3,"started_at":"2026-07-24T04:09:00.057Z","status":"completed","discovery":{"result_count":10,"snapshot_sha256":"90be874755269c810ebe258206ad0629946a4fc1755c652cd738666606d29043","selection_grounded":true},"selection":{"tool_id":"api_sports.standings.list.v1.39568ba4"},"inspection":{"selection_grounded":true},"parameterization":{"required_parameter_accuracy":1,"constraint_accuracy":1},"call":{"attempted":true,"success":true,"result_nonempty":true},"finished_at":"2026-07-24T04:09:02.840Z","source_record_sha256":"602cd1bbbde09a90e937b8d24fea39d74370ea558a78b467c257278204b981c9"} diff --git a/benchmarks/discover-call/results/2026-07-24-reference-v1-v4.summary.json b/benchmarks/discover-call/results/2026-07-24-reference-v1-v4.summary.json new file mode 100644 index 0000000..cdf9280 --- /dev/null +++ b/benchmarks/discover-call/results/2026-07-24-reference-v1-v4.summary.json @@ -0,0 +1,2034 @@ +{ + "schema_version": 1, + "methodology": "discover-call-v2", + "generated_at": "2026-07-24T04:40:24.583Z", + "models": [ + { + "model": "reference-v1", + "lane": "reference", + "tasks": 18, + "trials_per_task": 3, + "runs": 54, + "completed": 51, + "executed": 51, + "selection_grounded_rate": 0.9444, + "inspection_grounded_rate": 0.9444, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0.9444, + "call_success_rate": 1, + "result_nonempty_rate": 1, + "pre_result_gate_workflow_success_rate": 0.9444, + "workflow_success_rate": 0.9444, + "workflow_success_task_cluster_bootstrap_95": [ + 0.8333, + 1 + ], + "failures_by_stage": { + "select": 3 + }, + "failures_by_reason": { + "missing_tool_id": 3 + } + } + ], + "records": [ + { + "run_id": "0a929658-b437-44b2-abc3-6a2851ac251a", + "task_id": "weather-london", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "fe9b3456-fbdc-4a04-930c-0e6f0fbdc58a", + "task_id": "weather-london", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "d18949e0-3fae-4b1e-a0e9-7d4b8c13b803", + "task_id": "weather-london", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "f75c5749-09a4-4325-939a-90e8edcaacc2", + "task_id": "stock-aapl", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "9bd4b7c1-1766-42ff-8cd4-03f49cc46e95", + "task_id": "stock-aapl", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "2643e262-2eb7-43af-a92e-51e759692643", + "task_id": "stock-aapl", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "190f5741-2976-4edb-a433-6029cc6c0a05", + "task_id": "exchange-usd-eur", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "9f36d1e5-9134-4af1-8656-9c8ba39921b2", + "task_id": "exchange-usd-eur", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "e26110f8-0509-42c4-85a4-db0f48840dc4", + "task_id": "exchange-usd-eur", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "63939db0-f5e8-4511-9424-a7908b1ef178", + "task_id": "news-ai", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "55892a25-c775-481f-bcf9-80f22db0b269", + "task_id": "news-ai", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "eb15137b-5f73-442c-bceb-f4e0bb28aed5", + "task_id": "news-ai", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "b2f10c44-67a2-4ab4-a8dd-18813c91aec2", + "task_id": "papers-climate", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "87fdedb1-6bda-4ae4-8bfb-0dec587bb7ed", + "task_id": "papers-climate", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "6a1707b3-0496-4c1d-a74e-d925bb168882", + "task_id": "papers-climate", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "4937c77c-4c23-4a62-a60a-2a2d400dd502", + "task_id": "timezone-tokyo", + "model": "reference-v1", + "trial": 1, + "completed": false, + "executed": false, + "selection_grounded": false, + "inspection_grounded": false, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": null, + "result_nonempty": null, + "pre_result_gate_workflow_success": false, + "workflow_success": false, + "error_stage": "select", + "error_reason": "missing_tool_id", + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "e0d45a4d-2cab-4f24-a206-e31fbb0b2fbc", + "task_id": "timezone-tokyo", + "model": "reference-v1", + "trial": 2, + "completed": false, + "executed": false, + "selection_grounded": false, + "inspection_grounded": false, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": null, + "result_nonempty": null, + "pre_result_gate_workflow_success": false, + "workflow_success": false, + "error_stage": "select", + "error_reason": "missing_tool_id", + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "d41af726-12aa-4a16-b2c4-1fe3c6209e22", + "task_id": "timezone-tokyo", + "model": "reference-v1", + "trial": 3, + "completed": false, + "executed": false, + "selection_grounded": false, + "inspection_grounded": false, + "required_parameter_accuracy": 1, + "constraint_accuracy": 0, + "call_success": null, + "result_nonempty": null, + "pre_result_gate_workflow_success": false, + "workflow_success": false, + "error_stage": "select", + "error_reason": "missing_tool_id", + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "cc3386ef-c067-42e0-b6c2-d289efb9ccbe", + "task_id": "geocode-address", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "56fe8408-0c1e-4a04-a73b-aee950af9fb5", + "task_id": "geocode-address", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "12e0e9a6-6bd7-412b-98e3-d7973badac8c", + "task_id": "geocode-address", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "f15ccdde-8373-40fa-b664-73d50e65a819", + "task_id": "ip-geolocation", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "5f857a5b-9a5f-4ec8-85c4-50ede5b7472d", + "task_id": "ip-geolocation", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "2b5c778e-f319-4d92-a6f0-7edc7a65c2d9", + "task_id": "ip-geolocation", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "350b386d-0e36-4970-b1d4-4188da565056", + "task_id": "crypto-price", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "e32e3deb-54b1-428f-91bb-5e77f22bb813", + "task_id": "crypto-price", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "531b08ec-c6f8-406b-b400-4453a68205b7", + "task_id": "crypto-price", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "dd2858ba-eb7e-4720-b79c-8677e5dce881", + "task_id": "web-search", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "871b5609-875c-4303-8999-8ba516b46c32", + "task_id": "web-search", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "b9a39493-2947-4650-b16d-eab986c4fabd", + "task_id": "web-search", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "70e3b565-acb6-4ace-97c6-39512af06dc4", + "task_id": "air-quality", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "595adcb7-410e-4462-8e0e-10c6d28d1783", + "task_id": "air-quality", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "1e6f4c74-bde2-4e19-a9d9-feff3c1ce0a0", + "task_id": "air-quality", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "3f407e76-2b0a-4ad8-ab65-bfce16a0c388", + "task_id": "market-news", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "eb51c0fd-aaa0-46af-a4f0-d0e8710809c8", + "task_id": "market-news", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "11c7762e-b9b0-4567-a586-af7d0b242921", + "task_id": "market-news", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "59bc05f1-29c9-4448-8da7-5b7ba5883579", + "task_id": "domain-intel", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "7c0a548c-4fd4-4224-bd78-fb9a5d77f93b", + "task_id": "domain-intel", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "556779a9-caa5-4832-b46f-4cf0fe3f98f7", + "task_id": "domain-intel", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "0e930700-3c99-4997-ab3a-1860d4be5e27", + "task_id": "image-search", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "82a85463-1048-4274-965b-cd60e55ef866", + "task_id": "image-search", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "5c452f50-1723-4e6a-9293-96c604992466", + "task_id": "image-search", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "5587cd4b-0703-4967-a6c7-0019195961fd", + "task_id": "market-holidays", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "bacfdb82-0045-4bdb-a25f-88cfc0df79bb", + "task_id": "market-holidays", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "08faf7ed-b96d-45d1-969c-5962138faa5e", + "task_id": "market-holidays", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "dfb0f281-4fd3-4f4e-85d6-a6ea65aa7904", + "task_id": "earthquake-catalog", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "5c0c04ce-94ee-460a-b3d9-a6a78e173732", + "task_id": "earthquake-catalog", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "e2b5de64-cb4a-45d7-ac1f-9d31381cec7c", + "task_id": "earthquake-catalog", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "6b18bea9-34ff-4792-81d8-ab1590b0b15a", + "task_id": "company-profile", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "87424a80-ffa0-40aa-9ff7-f9ac14468b8d", + "task_id": "company-profile", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "31356409-e707-44c5-8cfb-8f7426a71e74", + "task_id": "company-profile", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "5c44fdf4-2b43-41ca-8af2-9dc6f427c411", + "task_id": "sports-standings", + "model": "reference-v1", + "trial": 1, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "fd7e7714-3696-4542-9fd5-032a1c2a5821", + "task_id": "sports-standings", + "model": "reference-v1", + "trial": 2, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + }, + { + "run_id": "75e5a133-d0c4-491c-bbf8-7888d03fb6d0", + "task_id": "sports-standings", + "model": "reference-v1", + "trial": 3, + "completed": true, + "executed": true, + "selection_grounded": true, + "inspection_grounded": true, + "required_parameter_accuracy": 1, + "constraint_accuracy": 1, + "call_success": true, + "result_nonempty": true, + "pre_result_gate_workflow_success": true, + "workflow_success": true, + "error_stage": null, + "error_reason": null, + "metadata": { + "lane": "reference", + "model_revision": "deterministic-reference-v1", + "adapter_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1", + "toolkit_revision": "a7f2aa60ef143dbbb35eaf9006ed8123d778fb13", + "task_set_sha256": "afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed", + "api_base_url": "https://qveris.ai/api/v1", + "api_revision": "2026-07-23.2", + "catalog_revision": "unreported", + "catalog_observation_sha256": "7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76", + "runtime": { + "node": "v24.2.0", + "platform": "darwin", + "arch": "arm64" + }, + "discovery_limit": 10, + "execute": true, + "publication_policy": "public-artifact-v1" + } + } + ] +} diff --git a/benchmarks/discover-call/results/README.md b/benchmarks/discover-call/results/README.md index 0c2dada..2bbb92d 100644 --- a/benchmarks/discover-call/results/README.md +++ b/benchmarks/discover-call/results/README.md @@ -1,5 +1,76 @@ # Published benchmark results +## 2026-07-24 — corrected v2 official configured-model baseline + +This is the first official configured-model baseline collected with benchmark +methodology `discover-call-v2`. It compares the curated reference route and +`gpt-5.6-sol` over immutable `tasks/v4.jsonl`: 18 tasks, three trials per task, +Top 10 discovery, and real calls. No failed trial was removed or selectively +rerun. Public artifacts were generated from the complete private records with +`public-artifact-v1`. + +| Metric | Curated reference route | `gpt-5.6-sol` configured model | +| --- | ---: | ---: | +| Runs | 54 | 54 | +| Completed and executed | 51 / 54 | 52 / 54 | +| Selection grounded | 94.44% | 100% | +| Inspection grounded | 94.44% | 100% | +| Required-parameter accuracy | 100% | 100% | +| Constraint accuracy | 94.44% | 88.89% | +| Call success among attempted calls | 100% (51 / 51) | 100% (52 / 52) | +| Result non-empty among successful calls | 100% (51 / 51) | 100% (52 / 52) | +| Strict workflow success | 94.44% (51 / 54) | 88.89% (48 / 54) | +| Task-cluster bootstrap 95% interval | 83.33%–100% | 72.22%–100% | + +The strict benchmark gap is **3 / 54 = 5.56 percentage points**. It is an +end-to-end benchmark difference, not a pure model-routing effect: the +sequential lanes observed different live Top-10 catalog snapshots. Their +intervals also overlap, so this 18-task baseline is not a claim of a +statistically significant difference. + +Shared reproducibility metadata: + +- task set: `tasks/v4.jsonl` +- task-set SHA-256: + `afe9a0d083765011817be01d688ef4f1bb04282604c3a0cfdc84f700951eb3ed` +- toolkit revision: + `a7f2aa60ef143dbbb35eaf9006ed8123d778fb13` +- API base URL: `https://qveris.ai/api/v1` +- observed API revision: `2026-07-23.2` +- catalog revision: `unreported` +- discovery limit: `10` +- runtime: Node.js `v24.2.0`, macOS arm64 +- publication policy: `public-artifact-v1` + +Lane metadata: + +- curated reference route: model `reference-v1`, model revision + `deterministic-reference-v1`, adapter revision + `a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/reference-v1`, + catalog-observation SHA-256 + `7d416f6aba2e1c18ea1cbde2dd4edef70a5e99f0e91fcbecd048ca8d389d3e76` +- configured model: model `gpt-5.6-sol`, provider model revision `unreported`, + reasoning effort `medium`, Codex CLI `0.144.1`, adapter revision + `a7f2aa60ef143dbbb35eaf9006ed8123d778fb13/codex-cli-0.144.1/medium`, + catalog-observation SHA-256 + `ab0333ae9e7c68702ff76e7db8d3ce3a81095acba004a81523f90a553e5d20ad` + +The reference route's three strict failures are all `timezone-tokyo` coverage +misses. The configured model completed and returned successful non-empty calls +for all three Tokyo trials, but the selected capability accepted no Tokyo +input, so they remain constraint failures. For `domain-intel`, two trials were +safely rejected as `tool_use_rejected`; the remaining trial called +successfully but did not encode the requested domain, producing one additional +constraint failure. The configured model revision is unreported, so this is +not a pinned-model snapshot. + +Artifacts: + +- [`2026-07-24-reference-v1-v4.runs.jsonl`](2026-07-24-reference-v1-v4.runs.jsonl) +- [`2026-07-24-reference-v1-v4.summary.json`](2026-07-24-reference-v1-v4.summary.json) +- [`2026-07-24-gpt-5.6-sol-configured-v4.runs.jsonl`](2026-07-24-gpt-5.6-sol-configured-v4.runs.jsonl) +- [`2026-07-24-gpt-5.6-sol-configured-v4.summary.json`](2026-07-24-gpt-5.6-sol-configured-v4.summary.json) + ## 2026-07-23 — v4 diagnostic baseline candidate This v4 run compares the curated reference route and the configured model diff --git a/docs/en-US/discover-call-benchmark.md b/docs/en-US/discover-call-benchmark.md index 683e3f2..a91abda 100644 --- a/docs/en-US/discover-call-benchmark.md +++ b/docs/en-US/discover-call-benchmark.md @@ -65,6 +65,44 @@ parameter names are omitted so hashed tools do not leak schema details. See the ## Published results +### Current official configured-model baseline + +The 2026-07-24 run is the first official configured-model baseline collected +with corrected `discover-call-v2`, immutable `tasks/v4.jsonl`, three trials per +task, real calls, and complete failed-trial retention. + +| 2026-07-24 baseline | Curated reference route | `gpt-5.6-sol` configured model | +| --- | ---: | ---: | +| Completed and executed | 51 / 54 | 52 / 54 | +| Selection grounded | 94.44% | 100% | +| Inspection grounded | 94.44% | 100% | +| Required-parameter accuracy | 100% | 100% | +| Constraint accuracy | 94.44% | 88.89% | +| Call success | 100% (51 / 51) | 100% (52 / 52) | +| Result non-empty | 100% (51 / 51) | 100% (52 / 52) | +| Strict workflow success | 94.44% (51 / 54) | 88.89% (48 / 54) | +| Workflow interval | 83.33%–100% | 72.22%–100% | + +The strict benchmark gap is **3 / 54 = 5.56 percentage points**. This is not a +pure routing delta: both lanes observed API revision `2026-07-23.2`, but the +API did not report a catalog revision and their catalog-observation digests +differ. The intervals overlap, so this 18-task baseline is not evidence of a +statistically significant difference. + +The reference route's three failures are Tokyo-timezone coverage misses. The +configured model has six strict failures: three Tokyo constraint mismatches, +two domain-intelligence `tool_use_rejected` adapter failures, and one +domain-intelligence constraint mismatch. Every attempted call in both lanes +reported success and a non-empty result. + +The configured lane used `gpt-5.6-sol`, medium reasoning, Codex CLI 0.144.1, +and toolkit revision +`a7f2aa60ef143dbbb35eaf9006ed8123d778fb13`. Its provider model revision is +`unreported`, so this is an official configured-model baseline rather than a +pinned-model snapshot. + +### Historical diagnostic + The 2026-07-23 v4 run is a **diagnostic baseline candidate**, not an official quality baseline. It covers 18 immutable tasks, three trials each, with real calls, but a later collector audit found that result non-emptiness was tested @@ -94,10 +132,9 @@ pinned model snapshot. Both lanes observed API revision `2026-07-22.1`; the API did not report a catalog revision, and the separate catalog-observation digests differ. -Promotion requires a fresh run under the corrected collector, which now checks -`result.data`, requests the full result projection, prevents ambiguous Execute -retries, and pins the reproducibility inputs before the first external call. -The earlier v3 run is also retained only as a diagnostic baseline. Its three +The fresh corrected run above supersedes this diagnostic as the current +configured-model baseline. The earlier v3 run is also retained only as a +diagnostic baseline. Its three successful Bitcoin calls used provider-specific `id=1`, which v3 incorrectly scored as constraint failures. Immutable `tasks/v4.jsonl` explicitly recognizes that mapping, and all three Bitcoin trials pass in v4. diff --git a/docs/zh-CN/discover-call-benchmark.md b/docs/zh-CN/discover-call-benchmark.md index f84eeae..eb538c8 100644 --- a/docs/zh-CN/discover-call-benchmark.md +++ b/docs/zh-CN/discover-call-benchmark.md @@ -48,6 +48,37 @@ tool 只保留摘要。参数质量仅以必填参数完整率和任务约束准 ## 已发布结果 +### 当前正式 configured-model 基线 + +2026-07-24 的运行是首个使用修正后 `discover-call-v2`、不可变 `tasks/v4.jsonl`、每个任务三个 +trial、真实调用并完整保留失败 trial 的正式 configured-model 基线。 + +| 2026-07-24 基线 | Curated reference route | `gpt-5.6-sol` configured model | +| --- | ---: | ---: | +| 完成参数化并执行 | 51 / 54 | 52 / 54 | +| Selection grounded | 94.44% | 100% | +| Inspection grounded | 94.44% | 100% | +| 必填参数准确率 | 100% | 100% | +| 任务约束准确率 | 94.44% | 88.89% | +| call 成功率 | 100%（51 / 51） | 100%（52 / 52） | +| 结果非空率 | 100%（51 / 51） | 100%（52 / 52） | +| 严格工作流成功率 | 94.44%（51 / 54） | 88.89%（48 / 54） | +| 工作流区间 | 83.33%–100% | 72.22%–100% | + +strict benchmark gap 为 **3 / 54 = 5.56 个百分点**。它不是纯 routing 差值：两条 lane 都观察到 +API revision `2026-07-23.2`，但 API 未报告 catalog revision，且 catalog-observation 摘要不同。 +两个区间也存在重叠，因此这组 18 任务基线不能证明差异具有统计显著性。 + +reference 的三次失败均为东京时区目录覆盖缺口。configured model 有六次严格失败：三次东京约束 +不匹配、两次 domain-intelligence `tool_use_rejected` adapter 失败，以及一次 domain-intelligence +约束不匹配。两条 lane 的所有实际调用都返回 success 且结果非空。 + +configured lane 使用 `gpt-5.6-sol`、medium reasoning、Codex CLI 0.144.1，以及 toolkit revision +`a7f2aa60ef143dbbb35eaf9006ed8123d778fb13`。provider model revision 为 `unreported`，因此这是 +正式 configured-model 基线，而不是 pinned-model snapshot。 + +### 历史诊断结果 + 2026-07-23 的 v4 运行是 **diagnostic baseline candidate**，不是正式质量基线。它包含 18 个不可变 任务、每个任务三个 trial，并启用真实调用；但后续 collector 审核发现，结果非空判断检查的是完整 `result` wrapper，而不是 `result.data`。因此，即使 data 为空，只要 wrapper 非空也可能被记录为 @@ -71,9 +102,8 @@ configured lane 使用 `gpt-5.6-sol`、medium reasoning 和 Codex CLI 0.144.1。 revision 为 `unreported`，因此不会称为 pinned model snapshot。两个 lane 都观察到 API revision `2026-07-22.1`；API 未报告 catalog revision，且两次独立运行的 catalog observation 摘要不同。 -只有在修正后的 collector 上重新运行，才能晋升为正式基线；当前 collector 已改为检查 -`result.data`、显式请求完整结果投影、禁止对结果不明确的 Execute 失败自动重试，并在首次外部调用前 -固定所有复现输入。较早的 v3 同样只保留为 diagnostic baseline。其三次 Bitcoin 调用通过 provider +上面的修正后新运行已取代该诊断结果，成为当前 configured-model 基线。较早的 v3 同样只保留为 +diagnostic baseline。其三次 Bitcoin 调用通过 provider 特定的 `id=1` 成功返回，但 v3 错误地将其计为约束失败。不可变 `tasks/v4.jsonl` 已明确识别该映射， v4 的三次 Bitcoin trial 全部通过。