From 25c35c0db75df536b0cd315f1a7ce98cda28bec5 Mon Sep 17 00:00:00 2001 From: Eric Sun <141227631+EricSun0218@users.noreply.github.com> Date: Sat, 15 Aug 2026 19:01:33 +0800 Subject: [PATCH 1/2] feat: add durable image input and game perception --- CHANGELOG.md | 11 +- OpenGameAgent.sln | 21 + README.md | 6 +- README.zh-CN.md | 6 +- docs/architecture.md | 5 +- docs/deployment-and-security.md | 6 +- docs/features.md | 3 + docs/game-integration-patterns.md | 2 + docs/getting-started.md | 33 ++ docs/image-input.md | 90 +++ docs/nuget-package-readme.md | 1 + .../open_game_agent/OpenGameAgent.Godot.props | 4 + .../addons/open_game_agent/packages.lock.json | 5 + engines/godot/build-package.ps1 | 2 +- engines/godot/test-package.ps1 | 1 + engines/unity/build-package.ps1 | 1 + engines/unity/packages.lock.json | 5 + engines/unity/test-package.ps1 | 2 + .../OpenGameAgent.Example/packages.lock.json | 4 + .../FileGameImageAttachmentStore.cs | 512 ++++++++++++++++++ .../OpenGameAgent.Attachments.Local.csproj | 14 + .../packages.lock.json | 42 ++ .../ImageAttachments.cs | 255 +++++++++ .../OpenGameAgent.Attachments.csproj | 7 + .../packages.lock.json | 6 + .../OpenGameAgent.Client.csproj | 1 + .../ServerGameAgentClient.cs | 117 ++++ src/OpenGameAgent.Client/packages.lock.json | 4 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + src/OpenGameAgent.Kernel/AgentOptions.cs | 12 + src/OpenGameAgent.Kernel/AgentValidator.cs | 81 ++- src/OpenGameAgent.Kernel/AssemblyInfo.cs | 3 + src/OpenGameAgent.Kernel/Content.cs | 13 + src/OpenGameAgent.Kernel/Models.cs | 9 + .../OpenGameAgent.Kernel.csproj | 3 + src/OpenGameAgent.Kernel/packages.lock.json | 3 + src/OpenGameAgent.Media/packages.lock.json | 4 + src/OpenGameAgent.Memory/packages.lock.json | 4 + .../packages.lock.json | 4 + .../BuiltInGameModelRuntime.cs | 10 +- .../packages.lock.json | 4 + src/OpenGameAgent.Models/ProviderCatalog.cs | 44 +- src/OpenGameAgent.Models/packages.lock.json | 4 + .../AgentMessageCodec.cs | 24 + .../packages.lock.json | 4 + src/OpenGameAgent.Plugins/packages.lock.json | 4 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../OpenGameAgent.Server.csproj | 2 + src/OpenGameAgent.Server/Program.cs | 10 + src/OpenGameAgent.Server/ServerAudience.cs | 8 +- .../ServerAuthorization.cs | 1 + src/OpenGameAgent.Server/ServerEndpoints.cs | 104 +++- src/OpenGameAgent.Server/packages.lock.json | 36 ++ src/OpenGameAgent/GameAgentRuntime.cs | 379 ++++++++++++- src/OpenGameAgent/GameAgentValueComparer.cs | 7 + src/OpenGameAgent/GameAgentWire.cs | 146 +++-- src/OpenGameAgent/GameData.cs | 57 +- src/OpenGameAgent/ModelProviders.cs | 20 +- src/OpenGameAgent/Transcripts.cs | 29 +- src/OpenGameAgent/packages.lock.json | 4 + .../ImageAttachmentTests.cs | 216 ++++++++ .../OpenGameAgent.Attachments.Tests.csproj | 19 + .../packages.lock.json | 237 ++++++++ .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../ImageValidationTests.cs | 103 ++++ .../ProjectDependencyBoundaryTests.cs | 7 +- .../PublicApiCompatibilityTests.cs | 2 +- .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../BuiltInGameModelRuntimeTests.cs | 17 +- .../packages.lock.json | 4 + .../ModelCatalogTests.cs | 38 ++ .../packages.lock.json | 4 + .../PersistenceTests.cs | 12 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../packages.lock.json | 4 + .../OpenGameAgent.Server.Tests/ServerTests.cs | 168 +++++- .../packages.lock.json | 39 ++ .../OpenGameAgent.Tests/ImageRuntimeTests.cs | 308 +++++++++++ .../PublicApiCompatibilityTests.cs | 2 +- tests/OpenGameAgent.Tests/RuntimeTests.cs | 2 +- tests/OpenGameAgent.Tests/packages.lock.json | 4 + tools/release-packages.json | 8 + 106 files changed, 3408 insertions(+), 101 deletions(-) create mode 100644 docs/image-input.md create mode 100644 src/OpenGameAgent.Attachments.Local/FileGameImageAttachmentStore.cs create mode 100644 src/OpenGameAgent.Attachments.Local/OpenGameAgent.Attachments.Local.csproj create mode 100644 src/OpenGameAgent.Attachments.Local/packages.lock.json create mode 100644 src/OpenGameAgent.Attachments/ImageAttachments.cs create mode 100644 src/OpenGameAgent.Attachments/OpenGameAgent.Attachments.csproj create mode 100644 src/OpenGameAgent.Attachments/packages.lock.json create mode 100644 src/OpenGameAgent.Kernel/AssemblyInfo.cs create mode 100644 tests/OpenGameAgent.Attachments.Tests/ImageAttachmentTests.cs create mode 100644 tests/OpenGameAgent.Attachments.Tests/OpenGameAgent.Attachments.Tests.csproj create mode 100644 tests/OpenGameAgent.Attachments.Tests/packages.lock.json create mode 100644 tests/OpenGameAgent.Kernel.Tests/ImageValidationTests.cs create mode 100644 tests/OpenGameAgent.Tests/ImageRuntimeTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index a7ad0b4..0808392 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,13 +2,16 @@ ## Unreleased -- Add the optional `TaskPlanExtension` for session/actor-scoped persistent ordered checklists, revision-checked mutations, host-validated evidence, per-input advancement guards, pending-work routing, typed UI projection events, and bounded terminal retention. -- Add typed, model-free host queries for persisted goals and task plans, including session revisions, and scope goal-change events with their session/actor key and input ID. -- Add batched, payload-free mailbox pending-status queries that distinguish ready work from active leases without claiming delivery or incrementing attempts. -- Add backward-compatible durable task-plan pause/resume with revision checks, preserved in-progress steps, non-runnable paused routing, typed change reasons, and restart coverage. +No changes yet. ## 0.3.0-alpha.2 +- Add durable image input for game observations: bounded PNG/JPEG/WebP/GIF decode admission, immutable content-addressed local objects, reference-only transcripts, provider/model preflight before reads, tool-result image persistence, JSON/SSE transport, and owner-authorized retrieval. +- Document the recommended large-world perception stack: bounded structured state, sparse BEV/topological summaries, selective screenshots, exact query tools, and deterministic game-owned execution rather than raw voxel dumps. +- Add the optional `TaskPlanExtension` for session/actor-scoped persistent ordered checklists, revision-checked mutations, host-validated evidence, per-input advancement guards, pending-work routing, typed UI projection events, and bounded terminal retention. +- Add typed, model-free host queries for persisted goals and task plans, including session revisions, and scope goal-change events with their session/actor key and input ID. +- Add batched, payload-free mailbox pending-status queries that distinguish ready work from active leases without claiming delivery or incrementing attempts. +- Add durable task-plan pause/resume with revision checks, preserved in-progress steps, non-runnable paused routing, typed change reasons, and restart coverage. - Add the optional `OpenGameAgent.Memory` package with a model-agnostic embedding provider contract, authoritative-save verification, rebuildable local vector indexes, hybrid lexical/vector recall, structured diagnostics, and game-time-aware reranking. - Add deterministic authoritative memory snapshots for in-memory and local-file stores so derived indexes can be rebuilt explicitly after embedding model or preprocessing changes. - Document local source references and game-provided local embedding integration, including BGE-M3-compatible query/document adapters and save boundaries. diff --git a/OpenGameAgent.sln b/OpenGameAgent.sln index b8771af..c174cfa 100644 --- a/OpenGameAgent.sln +++ b/OpenGameAgent.sln @@ -102,6 +102,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenGameAgent.Memory", "src EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenGameAgent.Memory.Tests", "tests\OpenGameAgent.Memory.Tests\OpenGameAgent.Memory.Tests.csproj", "{5E8B096B-DD5F-4463-B841-7675F560B52D}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenGameAgent.Attachments", "src\OpenGameAgent.Attachments\OpenGameAgent.Attachments.csproj", "{02764E37-C515-48D2-BBFA-BA9C8C71425C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenGameAgent.Attachments.Local", "src\OpenGameAgent.Attachments.Local\OpenGameAgent.Attachments.Local.csproj", "{967FB56B-BC1A-4FEF-A2C8-289CED38A733}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenGameAgent.Attachments.Tests", "tests\OpenGameAgent.Attachments.Tests\OpenGameAgent.Attachments.Tests.csproj", "{F0ECE44A-ABCF-482C-9F81-D878B28C250D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -300,6 +306,18 @@ Global {5E8B096B-DD5F-4463-B841-7675F560B52D}.Debug|Any CPU.Build.0 = Debug|Any CPU {5E8B096B-DD5F-4463-B841-7675F560B52D}.Release|Any CPU.ActiveCfg = Release|Any CPU {5E8B096B-DD5F-4463-B841-7675F560B52D}.Release|Any CPU.Build.0 = Release|Any CPU + {02764E37-C515-48D2-BBFA-BA9C8C71425C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {02764E37-C515-48D2-BBFA-BA9C8C71425C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {02764E37-C515-48D2-BBFA-BA9C8C71425C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {02764E37-C515-48D2-BBFA-BA9C8C71425C}.Release|Any CPU.Build.0 = Release|Any CPU + {967FB56B-BC1A-4FEF-A2C8-289CED38A733}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {967FB56B-BC1A-4FEF-A2C8-289CED38A733}.Debug|Any CPU.Build.0 = Debug|Any CPU + {967FB56B-BC1A-4FEF-A2C8-289CED38A733}.Release|Any CPU.ActiveCfg = Release|Any CPU + {967FB56B-BC1A-4FEF-A2C8-289CED38A733}.Release|Any CPU.Build.0 = Release|Any CPU + {F0ECE44A-ABCF-482C-9F81-D878B28C250D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F0ECE44A-ABCF-482C-9F81-D878B28C250D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F0ECE44A-ABCF-482C-9F81-D878B28C250D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F0ECE44A-ABCF-482C-9F81-D878B28C250D}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {01759D73-7B80-47A2-9D7D-154CC64C6851} = {EA3AF59A-9A1C-4197-B2A3-F93894D131B8} @@ -334,5 +352,8 @@ Global {5697E98C-2249-4D4C-894B-CB0A8732238E} = {86AE6217-BFEE-4349-945A-70ECEC211437} {00AE7836-01FA-4151-A38A-8263D9164A75} = {EA3AF59A-9A1C-4197-B2A3-F93894D131B8} {5E8B096B-DD5F-4463-B841-7675F560B52D} = {86AE6217-BFEE-4349-945A-70ECEC211437} + {02764E37-C515-48D2-BBFA-BA9C8C71425C} = {EA3AF59A-9A1C-4197-B2A3-F93894D131B8} + {967FB56B-BC1A-4FEF-A2C8-289CED38A733} = {EA3AF59A-9A1C-4197-B2A3-F93894D131B8} + {F0ECE44A-ABCF-482C-9F81-D878B28C250D} = {86AE6217-BFEE-4349-945A-70ECEC211437} EndGlobalSection EndGlobal diff --git a/README.md b/README.md index f85dede..401cd95 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ OpenGameAgent is a compact, hackable C# runtime that lets game characters observ OpenGameAgent starts with a small, composable agent kernel. The stateful core streams model output, executes validated tools, accepts steering while running, and continues the model/tool loop until work is complete. Use the kernel by itself, add the game runtime for game time and durable state, then opt into extensions for memory, goals, host-verified task plans, artifacts, delegation, external tools, structured interaction, and workflow graphs. -Inputs are bounded JSON. They may represent dialogue, combat observations, simulation ticks, UI events, plans, sensor state, or any other game-owned data; natural language is optional. No model is bundled. Cloud and local API endpoints are both supported. +Inputs are bounded JSON plus optional durable image observations. They may represent dialogue, combat observations, simulation ticks, UI events, plans, sensor state, screenshots, or any other game-owned data; natural language is optional. No model is bundled. Cloud and local API endpoints are both supported. ## A programmable agent runtime built for games @@ -57,6 +57,7 @@ Install the complete game runtime from NuGet: ```bash dotnet add package OpenGameAgent --version 0.3.0-alpha.2 dotnet add package OpenGameAgent.Memory --version 0.3.0-alpha.2 # optional semantic memory +dotnet add package OpenGameAgent.Attachments.Local --version 0.3.0-alpha.2 # optional durable image input ``` The kernel, persistence, providers, and engine-compatible client are also published as separate `OpenGameAgent.*` packages. Godot, Unity, and portable server archives are available on the [Releases](https://github.com/EricSun0218/OpenGameAgent/releases) page. See [Getting started](docs/getting-started.md) and [Engine integration](docs/engine-integration.md) before connecting a game. @@ -69,6 +70,7 @@ OpenGameAgent keeps the reusable agent machinery independent from the game while - named timelines and integer ticks, with optional calendar JSON; - structured observations and context slices with floating-point values intact; +- content-addressed screenshot/image input with decode validation, model-capability preflight, and session-authorized retrieval; - quick-response, full-agent, and deterministic-workflow routes; - per-actor serialization with bounded cross-actor concurrency; - journaled action intents and authoritative game receipts; @@ -116,6 +118,7 @@ Read [Architecture](docs/architecture.md) for the ownership and failure boundari | Agent kernel | Streaming typed messages, tool loop, typed partial tool results, steering, follow-up, hooks, cancellation, strict transcript validation, provider failures as results | | Tool execution | Provider-request schema preflight plus execution-time validation over a bounded JSON Schema subset, guaranteed result for every accepted call, safe parallel reads, conflict-key serialization, policy blocking/termination, timeouts, uncertain write outcomes | | Game runtime | Arbitrary JSON input, game clocks/timelines, fast/full/workflow routing, optimistic sessions, duplicate-input protection, actor concurrency, active-run steering/abort | +| Image input | PNG/JPEG/WebP/GIF admission, immutable content-addressed storage, reference-only transcripts, capability preflight, tool-result images, and authorized server retrieval | | Extension API | Immutable builder; prompt/context/tool/skill/route/workflow/hook/provider/service registration; typed lifecycle events and channels; namespaced persistent state | | Official extensions | Tool policy and search, structured player questions/recommended replies, goals, host-verified ordered task plans with durable pause/resume, memory, artifacts, knowledge, delegation, tracing, and durable parallel workflow graphs | | World primitives | Durable actions, resumable workflows, memories, skills, signals, game-time schedules, actor mailboxes with batch read-only pending status | @@ -224,6 +227,7 @@ Real-editor gates are documented in [Engine integration](docs/engine-integration - [Engine integration](docs/engine-integration.md) - [Deployment and security](docs/deployment-and-security.md) - [Generated media](docs/media.md) +- [Image input and game perception](docs/image-input.md) ## Project boundary diff --git a/README.zh-CN.md b/README.zh-CN.md index bd954d6..a5ef369 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -22,7 +22,7 @@ OpenGameAgent 是一个紧凑、可修改的 C# Runtime，让游戏角色能够 OpenGameAgent 从小型、可组合的 Agent 内核出发。有状态核心会流式接收模型输出、执行经过校验的工具、在运行中接受 steering，并持续进行模型/工具循环直到任务结束。开发者既可以只使用内核，也可以叠加游戏 Runtime 获得游戏时间与可靠状态，再按需加入记忆、目标、宿主证据校验的任务清单、产物、委派、外部工具、结构化交互和工作流图等扩展。 -输入是有大小限制的 JSON，可以表示对话、战斗观察、模拟 Tick、UI 事件、计划、传感状态或任意游戏数据，不要求是自然语言。项目不捆绑模型，同时支持云端和本地 API。 +输入是有大小限制的 JSON，并可携带持久化图片观察，可以表示对话、战斗观察、模拟 Tick、UI 事件、计划、传感状态、截图或任意游戏数据，不要求是自然语言。项目不捆绑模型，同时支持云端和本地 API。 ## 为游戏构建的可编程 Agent Runtime @@ -57,6 +57,7 @@ OpenGameAgent 不绑定任何模型或 Provider。角色通过开发者定义的 ```bash dotnet add package OpenGameAgent --version 0.3.0-alpha.2 dotnet add package OpenGameAgent.Memory --version 0.3.0-alpha.2 # 可选语义记忆 +dotnet add package OpenGameAgent.Attachments.Local --version 0.3.0-alpha.2 # 可选持久图片输入 ``` 内核、持久化、模型提供方和引擎兼容客户端也分别提供 `OpenGameAgent.*` 包。Godot、Unity 与可移植服务端压缩包可以从 [Releases](https://github.com/EricSun0218/OpenGameAgent/releases) 页面下载。接入游戏前请阅读[快速开始](docs/getting-started.md)和[引擎接入](docs/engine-integration.md)。 @@ -69,6 +70,7 @@ OpenGameAgent 不替游戏规定玩法，而是提供可复用的游戏坐标与 - 命名时间线、整数 Tick 和可选日历 JSON； - 保留浮点数的结构化观察与上下文； +- 经真实解码校验、内容寻址持久化、模型能力预检与会话授权读取的截图/图片输入； - 快速回复、完整 Agent、确定性 Workflow 三种路由； - 同一角色串行、不同角色有界并行； - 先记日志的动作意图与游戏权威回执； @@ -114,6 +116,7 @@ GameAgentRuntime | Agent 内核 | 流式类型化消息、工具循环、类型化工具中间结果、steering、follow-up、hooks、取消、严格会话校验、提供方错误结果化 | | 工具执行 | provider 请求前 schema 预检及执行期有界 JSON Schema 子集校验、每个已接受调用都有结果、安全并行读、冲突键串行、策略拦截/终止、超时与写入结果未知语义 | | 游戏 Runtime | 任意 JSON 输入、游戏时钟/时间线、快速/完整/Workflow 路由、乐观并发会话、输入去重、角色并发、运行中 steering/abort | +| 图片输入 | PNG/JPEG/WebP/GIF 准入、不可变内容寻址存储、仅引用会话、模型能力预检、工具结果图片与授权服务端读取 | | 扩展 API | 不可变构建器；提示词/上下文/工具/Skills/路由/Workflow/Hooks/提供方/服务注册；类型化生命周期事件与通道；命名空间持久状态 | | 官方扩展 | 工具策略与搜索、玩家结构化提问/推荐回复、目标、支持持久暂停/恢复且由宿主校验证据的有序任务清单、记忆、产物、外部知识、委派、追踪和可持久并行工作流图 | | 世界原语 | 可恢复动作、可续跑 Workflow、记忆、Skills、信号、游戏时间调度、支持批量只读待处理状态的角色邮箱 | @@ -223,6 +226,7 @@ dotnet test OpenGameAgent.sln -c Release --no-build --no-restore - [引擎集成](docs/engine-integration.md) - [部署与安全](docs/deployment-and-security.md) - [生成式媒体](docs/media.md) +- [图片输入与游戏感知](docs/image-input.md) ## 项目边界 diff --git a/docs/architecture.md b/docs/architecture.md index 647a2f7..a58083e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -17,7 +17,7 @@ The kernel owns one stateful model/tool loop: 7. apply steering or follow-up messages; 8. continue until the model stops, a hook stops the run, cancellation occurs, or a limit is reached. -It knows nothing about NPCs, worlds, inventories, or engines. Its canonical values are typed content parts (`text`, `json`, `resource`, `reasoning`, and `tool_call`), messages, model requests, tools, and events. +It knows nothing about NPCs, worlds, inventories, or engines. Its canonical values are typed content parts (`text`, `json`, `resource`, durable `image_attachment`, `reasoning`, and `tool_call`), messages, model requests, tools, and events. Inline image bytes are request-boundary input; canonical history stores only immutable attachment references. `Agent` owns mutable transcript and queue state. `AgentLoop` is the lower-level execution function. A host that already owns state can call the loop directly; most integrations should keep an `Agent` or use `GameAgentRuntime`. @@ -45,6 +45,7 @@ It does not own a universal world model. Context remains opaque JSON supplied by - `OpenGameAgent.Models` adds provider/model catalogs, capability-aware selection, reasoning levels, cost metadata, dynamic refresh, and replaceable authentication. - `OpenGameAgent.Models.BuiltIn` turns the bundled directory into an executable multi-provider model runtime; `OpenGameAgent.Models.Auth.BuiltIn` adds explicitly configured browser and device authorization flows. - `OpenGameAgent.ProviderTransport` centralizes bounded response observations, header guards, and retry metadata without adding HTTP concepts to the kernel. +- `OpenGameAgent.Attachments` defines immutable image references and storage admission; `OpenGameAgent.Attachments.Local` provides a content-addressed local implementation with real decode and integrity checks. - `OpenGameAgent.Media` routes image, audio, and video generation by provider/model capability while keeping generation jobs outside the text/tool protocol. - `OpenGameAgent.Connectors.Mcp` exposes external tool servers through one lazy, searchable tool by default. Direct tool exposure is an explicit opt-in. - Provider, persistence, engine, client, and server packages stay replaceable and do not change kernel semantics. @@ -106,6 +107,8 @@ Transcript compaction is also a provider-view operation. The included summarizin Context admission runs before the first request, after tool turns, and again after final request hooks. A hook therefore cannot accidentally bypass the configured context window. Large text or JSON tool results can be moved into the artifact store and replaced with a bounded handle and preview. This keeps canonical results recoverable without repeatedly paying their full context cost. +Image admission follows the same canonical/request-view split. Inline user or tool-result images are fully validated and persisted before they enter session history. The active provider/model is preflighted, then immutable references are resolved into bytes only for the outgoing model request. System and assistant images are rejected; generated assets use the media pipeline. See [Image input and game perception](image-input.md). + The system prompt keeps the most reusable bytes first: base instructions, then selected skills, then mutable authoritative game context. This ordering preserves the longest possible provider-cache prefix when world state changes, without moving dynamic state out of the game-owned context boundary. After a tool turn, `GameAgentRuntime` refreshes authoritative context, tools, and selected skills by default before the next model request. A configured next-turn hook can supply an explicit replacement context instead. Active game-layer runs can also be steered or aborted by `GameSessionKey`; messages never cross actor lanes. diff --git a/docs/deployment-and-security.md b/docs/deployment-and-security.md index ab71003..98483c9 100644 --- a/docs/deployment-and-security.md +++ b/docs/deployment-and-security.md @@ -30,6 +30,7 @@ OpenGameAgent__ApiKey=provider-secret OpenGameAgent__ServerApiKey=game-to-agent-secret OpenGameAgent__DataDirectory=/var/lib/opengameagent/sessions OpenGameAgent__ActionDirectory=/var/lib/opengameagent/actions +OpenGameAgent__AttachmentDirectory=/var/lib/opengameagent/attachments ``` The included service exposes: @@ -41,6 +42,7 @@ The included service exposes: - `POST /v1/control/steer` - `POST /v1/control/abort` - `POST /v1/usage` +- `POST /v1/attachments/read` - `POST /v1/actions/claim` - `POST /v1/actions/stream` (Server-Sent Events over a JSON POST request) - `POST /v1/actions/receipt` @@ -52,6 +54,8 @@ When `ServerApiKey` is set, run and control endpoints require `Authorization: Be Register an `IGameAgentOwnerAuthorizer` for player-facing or multi-tenant deployments. Every run, stream, steer, and abort request is then authorized against the authenticated principal and the parsed `(session, actor)` resource before the runtime, session store, or active actor is touched. Anonymous requests receive `401`; authenticated principals that do not own the resource receive `403`. The same operation contract reserves usage and durable-action operations so those endpoints use the identical ownership decision. Derive ownership from authenticated claims or an authoritative host store—never from an owner field supplied in the request payload. Without a registered authorizer the endpoint is suitable only for a trusted single-owner deployment. +Attachment reads use that same owner authorization before loading either the session or the content-addressed object. The requested attachment must also be referenced by the authorized session/actor transcript; knowing or guessing a SHA-256 ID is not sufficient. Inline upload bytes are validated and replaced with durable references before session persistence, and provider credentials never enter attachment metadata. + Control requests only address an already active `(session, actor)` loop; they cannot register tools or mutate game state directly. Put TLS, request-rate limits, tenant quotas, and abuse protection at the gateway. The included shared-secret gate identifies one deployment-wide subject; it is not a multi-user account system. ### Output audiences @@ -188,7 +192,7 @@ The external-tool connector defaults to one on-demand search/describe/call tool, ## Data and retention -The local stores are not encrypted. Put them in an access-controlled game save or service data directory. Decide which prompts, context, memories, artifacts, delegation records, generated assets, and provider identifiers may contain player data. Implement retention, export, deletion, consent, and regional handling for your product. The included stores retain completed records needed for deduplication and recovery and do not provide a generic purge policy; archive them only when the game can prove their replay-safety window has ended. +The local stores are not encrypted. Put them in an access-controlled game save or service data directory. Decide which prompts, context, memories, image observations, artifacts, delegation records, generated assets, and provider identifiers may contain player data. Implement retention, export, deletion, consent, and regional handling for your product. Back up sessions and their attachment objects together. Content-addressed images may be referenced by several actors or branches; an orphan collector must enumerate all authoritative references before deletion. The included stores retain completed records needed for deduplication and recovery and do not provide a generic purge policy; archive them only when the game can prove their replay-safety window has ended. Never log credentials. Avoid logging full prompts and tool payloads in production unless the player has consented and access is controlled. diff --git a/docs/features.md b/docs/features.md index c42e2c8..fc30301 100644 --- a/docs/features.md +++ b/docs/features.md @@ -44,6 +44,8 @@ This page maps product needs to the smallest reusable OpenGameAgent primitive. | Need | API | | --- | --- | | Submit non-language game data | `GameInput.PayloadJson` | +| Attach screenshots or other visual observations | `GameInput.Content`, `BinaryContent`, `GameImageAttachment` | +| Persist and resolve immutable image input | `IGameImageAttachmentStore`, `FileGameImageAttachmentStore` | | Express game time or save forks | `GameMoment` | | Supply current world state | `IGameContextProvider`, `GameContextSlice` | | Keep obvious dialogue fast | `AutomaticGameRoutePolicy`, `ModelGameRouteClassifier` | @@ -112,6 +114,7 @@ In-memory implementations are useful for tests and short-lived sessions. The `Op - delegation records; - directory-backed skills; - directory-backed prompt templates. +- content-addressed local image attachments (`OpenGameAgent.Attachments.Local`). File stores coordinate writers that use the same directory through cross-process leases, but they are not a distributed database. A multiplayer or multi-host service should implement the same interfaces using transactional shared storage and explicit actor ownership. Completed action, workflow, mailbox, and deduplication records are intentionally retained to preserve replay safety; long-running products should implement retention or archival in their game-owned stores rather than deleting evidence blindly. diff --git a/docs/game-integration-patterns.md b/docs/game-integration-patterns.md index ade47d0..51eaec4 100644 --- a/docs/game-integration-patterns.md +++ b/docs/game-integration-patterns.md @@ -14,6 +14,8 @@ The game emits an observation when goals, threats, resources, or player orders c Use steering to inject urgent state changes during a long run. A steering message should identify the new observation version so the model can abandon a stale plan. Use conflict keys to prevent two simultaneous writes to the same companion or resource. +For visual worlds, combine structured local state with a sparse BEV or topological map, then attach a screenshot or crop only when appearance or geometry matters. Do not serialize every voxel or pixel. Let the model select intent and targets, use read-only tools for exact follow-up queries, and leave pathfinding, placement, physics, and animation to deterministic game code. See [Image input and game perception](image-input.md). + ## Interactive world and many NPCs Keep world simulation deterministic and cheap. Invoke a model only when a character needs semantic judgment, dialogue, planning, or content generation. diff --git a/docs/getting-started.md b/docs/getting-started.md index d5733b0..000e56d 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -100,6 +100,39 @@ public ValueTask> GetContextAsync( Context is treated as data, not as a hidden state mutation channel. +## Add image observations + +For screenshots or visual tool results, mount an `IGameImageAttachmentStore` and pass inline image bytes through `GameInput.Content`. The runtime validates and persists the whole batch, saves only immutable references in the transcript, preflights the selected model, and resolves bytes just before provider dispatch. + +```powershell +dotnet add package OpenGameAgent.Attachments.Local --version 0.3.0-alpha.2 +``` + +```csharp +using OpenGameAgent.Attachments; +using OpenGameAgent.Attachments.Local; + +options.ImageAttachments = new FileGameImageAttachmentStore(attachmentDirectory); + +var input = new GameInput( + "save-42", + "npc-scout", + "scene_changed", + """{"region":"north-gate"}""", + new GameMoment("main", 900), + "scene-900-scout", + content: new AgentContent[] + { + new BinaryContent( + AgentMediaKind.Image, + Convert.ToBase64String(pngBytes), + GameImageMediaTypes.Png, + "scout-view.png"), + }); +``` + +Use a model whose catalog entry declares image input. Models that cannot consume the image fail explicitly; the runtime never drops it silently. For large voxel or open worlds, combine bounded structured state, a sparse BEV/topological summary, selective screenshots, and exact query tools instead of serializing every coordinate. See [Image input and game perception](image-input.md). + ## Expose actions Create tools per input so they can carry the stable input identity and actor scope. Prefer `GameActionTool.Create` for state changes. diff --git a/docs/image-input.md b/docs/image-input.md new file mode 100644 index 0000000..46fc9c8 --- /dev/null +++ b/docs/image-input.md @@ -0,0 +1,90 @@ +# Image input and game perception + +OpenGameAgent accepts images as durable, bounded input to an agent run. The framework does not bundle a vision model: use a cloud or local API model whose catalog entry declares image input. + +Image understanding and generated media are separate paths. Screenshots, crops, minimaps, and tool-produced observations enter the agent as image input. Assets generated by an image, audio, or video model use `OpenGameAgent.Media`. + +## Recommended perception stack + +Do not serialize every block, voxel, navmesh point, or screen pixel into a prompt. For a large world, give each NPC a layered observation: + +1. **Structured local state:** visible entities, relations, affordances, inventory, hazards, goals, game time, and authoritative IDs in bounded JSON. +2. **Sparse spatial view:** a local BEV, occupancy grid, room graph, chunk summary, or topological map at the coarsest resolution that preserves the current decision. +3. **Selective images:** a screenshot or crop when appearance, occlusion, terrain shape, an unknown object, or a player-created structure cannot be represented reliably by state alone. +4. **On-demand queries:** tools such as `inspect_entity`, `inspect_region`, `find_path`, or `measure_clearance` for exact facts after the model selects a target. +5. **Deterministic execution:** the model chooses an intent, target, or blueprint; ordinary game code performs exact pathfinding, placement, physics, collision, resource accounting, and animation. + +This keeps the prompt semantic and bounded. A block-building NPC can decide *what* to build from a screenshot, nearby materials, and a coarse local map without receiving millions of coordinates. Once it selects a design, a game-owned blueprint or building tool handles the exact blocks. + +Each actor should receive only its own visible scene. Apply fog-of-war, permissions, and secret filtering before creating JSON or image input. Capture images at decision boundaries or when the scene changed materially, not on every render frame. A game can route image-heavy decisions to a vision model and keep routine dialogue or deterministic ticks on a cheaper text model. + +## Durable attachment lifecycle + +Inline image bytes are admitted before the run: + +- PNG, JPEG, WebP, and GIF are supported; +- the complete batch is validated before any object is published; +- real image decoding verifies the declared media type, dimensions, byte limit, and pixel limit; +- the local store writes immutable SHA-256-addressed objects atomically; +- canonical transcripts and save files contain only bounded attachment references; +- bytes are resolved only immediately before a model request; +- the active provider/model is preflighted before attachment bytes are loaded; +- missing, corrupt, or mismatched objects fail closed; +- a server read is authorized against the referenced session and actor before the store is touched. + +Defaults are 5 MiB per image, 20 images per message, 100 MiB in aggregate, and 40 million decoded pixels per image. Configure lower limits for shipped games where appropriate. + +Only user input and tool results become model-visible image history. A tool can return a screenshot as an inline `BinaryContent`; `GameAgentRuntime` persists it before the next model turn and before the session checkpoint. Local tool-progress subscribers may receive bounded ephemeral binary previews, as used by generated-media progress. Progress content is neither canonical history nor part of the stock public JSON/SSE projection; persist a final result when the image must survive or cross the server boundary. + +## In-process example + +Install `OpenGameAgent.Attachments.Local` alongside the runtime, then mount one store below the game's save or application-data directory: + +```csharp +using OpenGameAgent; +using OpenGameAgent.Attachments; +using OpenGameAgent.Attachments.Local; +using OpenGameAgent.Kernel; + +var attachments = new FileGameImageAttachmentStore( + Path.Combine(saveRoot, "agent-attachments")); + +var runtime = new GameAgentRuntime(new GameAgentRuntimeOptions(provider, model) +{ + ImageAttachments = attachments, + ContextProvider = worldContext, + ToolProvider = gameTools, + SessionStore = sessionStore, +}); + +var screenshot = await File.ReadAllBytesAsync(framePath); +var input = new GameInput( + sessionId: "save-42", + actorId: "npc-builder", + type: "scene_changed", + payloadJson: """{"visibleEntities":["player","workbench"],"region":"base-east"}""", + moment: new GameMoment("main", tick: 18420), + inputId: "scene-18420-builder", + content: new AgentContent[] + { + new BinaryContent( + AgentMediaKind.Image, + Convert.ToBase64String(screenshot), + GameImageMediaTypes.Png, + "builder-view.png"), + }); + +var result = await runtime.RunAsync(input); +``` + +After admission, the runtime replaces the inline bytes with a `GameImageAttachment`. Replaying the session resolves the immutable object again; the transcript never stores base64 data. + +## Server placement + +The stock JSON/SSE server accepts inline image content in `GameInput.content` and uses the configured attachment directory. The client can call `ServerGameAgentClient.ReadImageAttachmentAsync` when an authorized UI needs to display an attachment referenced by that session. Multi-user hosts must install an identity-derived owner authorizer; an attachment ID alone grants no access. + +Back up and restore session state and its attachment directory together. Immutable content-addressed objects may be shared by multiple references. Retention and orphan collection remain a save/storage policy: never delete an object merely because one transcript branch no longer displays it unless the host has enumerated all authoritative references. + +## Multi-NPC performance + +Image admission does not change actor scheduling. Runs for the same `(sessionId, actorId)` remain serialized; different actors run concurrently up to `GameRuntimeLimits.MaxConcurrentActors`. Keep capture and preprocessing outside the engine's render-critical path, deduplicate identical frames, and apply actor importance/distance budgets before enqueuing runs. Shared-world writes still require game-owned revisions or transactions: per-actor ordering is not a global world lock. diff --git a/docs/nuget-package-readme.md b/docs/nuget-package-readme.md index 50ce8a5..e8505fb 100644 --- a/docs/nuget-package-readme.md +++ b/docs/nuget-package-readme.md @@ -4,6 +4,7 @@ Open-source C# agent runtime for AI-native games, autonomous NPCs, and interacti - Small streaming model/tool-loop kernel - Arbitrary structured game inputs and game time +- Durable screenshot/image input with content-addressed local storage and model-capability preflight - Durable game actions and workflows - Typed extension API plus official policy, catalog, interaction, goal, host-verified task-plan, memory, artifact, delegation, tracing, and workflow-graph extensions - Skills, scheduling, mailboxes, large-result spill, and multi-actor concurrency diff --git a/engines/godot/addons/open_game_agent/OpenGameAgent.Godot.props b/engines/godot/addons/open_game_agent/OpenGameAgent.Godot.props index b2ae3b4..72757d8 100644 --- a/engines/godot/addons/open_game_agent/OpenGameAgent.Godot.props +++ b/engines/godot/addons/open_game_agent/OpenGameAgent.Godot.props @@ -1,5 +1,9 @@ + + $(MSBuildThisFileDirectory)lib\OpenGameAgent.Attachments.dll + true + $(MSBuildThisFileDirectory)lib\OpenGameAgent.Kernel.dll true diff --git a/engines/godot/addons/open_game_agent/packages.lock.json b/engines/godot/addons/open_game_agent/packages.lock.json index e6ba2f6..a9d5ec0 100644 --- a/engines/godot/addons/open_game_agent/packages.lock.json +++ b/engines/godot/addons/open_game_agent/packages.lock.json @@ -14,16 +14,21 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.client": { "type": "Project", "dependencies": { "OpenGameAgent": "[0.3.0-alpha.2, )", + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } } diff --git a/engines/godot/build-package.ps1 b/engines/godot/build-package.ps1 index a671029..f13a3ce 100644 --- a/engines/godot/build-package.ps1 +++ b/engines/godot/build-package.ps1 @@ -51,7 +51,7 @@ $plugin = $plugin -replace 'version="[^"]+"', ('version="' + $Version + '"') $plugin | Set-Content -LiteralPath (Join-Path $packagedAddon 'plugin.cfg') -Encoding utf8NoBOM $buildOutput = Join-Path $addonRoot 'bin\Release\net8.0' -foreach ($assembly in @('OpenGameAgent.Kernel.dll', 'OpenGameAgent.dll', 'OpenGameAgent.Client.dll')) { +foreach ($assembly in @('OpenGameAgent.Attachments.dll', 'OpenGameAgent.Kernel.dll', 'OpenGameAgent.dll', 'OpenGameAgent.Client.dll')) { $source = Join-Path $buildOutput $assembly if (-not (Test-Path -LiteralPath $source -PathType Leaf)) { throw "Required Godot assembly '$assembly' is missing." diff --git a/engines/godot/test-package.ps1 b/engines/godot/test-package.ps1 index 8c84bb9..33aacce 100644 --- a/engines/godot/test-package.ps1 +++ b/engines/godot/test-package.ps1 @@ -18,6 +18,7 @@ $required = @( 'addons\open_game_agent\plugin.cfg', 'addons\open_game_agent\OpenGameAgent.Godot.props', 'addons\open_game_agent\runtime\OpenGameAgentNode.cs', + 'addons\open_game_agent\lib\OpenGameAgent.Attachments.dll', 'addons\open_game_agent\lib\OpenGameAgent.Kernel.dll', 'addons\open_game_agent\lib\OpenGameAgent.dll', 'addons\open_game_agent\lib\OpenGameAgent.Client.dll' diff --git a/engines/unity/build-package.ps1 b/engines/unity/build-package.ps1 index 1d43510..df44df4 100644 --- a/engines/unity/build-package.ps1 +++ b/engines/unity/build-package.ps1 @@ -63,6 +63,7 @@ DefaultImporter: $pluginFolderMeta | Set-Content -LiteralPath ($plugins + '.meta') -Encoding utf8NoBOM $buildOutput = Join-Path $engineRoot 'bin\Release\netstandard2.1' $assemblies = [ordered]@{ + 'OpenGameAgent.Attachments.dll' = '01b0c910bb244a64ba3d85bb66348656' 'OpenGameAgent.Kernel.dll' = 'b62e147235cd4b60bf2b3ec44621214f' 'OpenGameAgent.dll' = 'b8a23b387de04f5d8b59f0d59028cb6d' 'OpenGameAgent.Client.dll' = '36eb78a2d19d407fbe9308d746ed42fd' diff --git a/engines/unity/packages.lock.json b/engines/unity/packages.lock.json index 0d67273..8d12da7 100644 --- a/engines/unity/packages.lock.json +++ b/engines/unity/packages.lock.json @@ -70,16 +70,21 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.client": { "type": "Project", "dependencies": { "OpenGameAgent": "[0.3.0-alpha.2, )", + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } } diff --git a/engines/unity/test-package.ps1 b/engines/unity/test-package.ps1 index ec94287..a840dab 100644 --- a/engines/unity/test-package.ps1 +++ b/engines/unity/test-package.ps1 @@ -22,6 +22,8 @@ $required = @( 'Runtime\OpenGameAgentBehaviour.cs', 'Runtime\OpenGameAgentBehaviour.cs.meta', 'Runtime\Plugins.meta', + 'Runtime\Plugins\OpenGameAgent.Attachments.dll', + 'Runtime\Plugins\OpenGameAgent.Attachments.dll.meta', 'Runtime\Plugins\OpenGameAgent.Kernel.dll', 'Runtime\Plugins\OpenGameAgent.Kernel.dll.meta', 'Runtime\Plugins\OpenGameAgent.dll', diff --git a/examples/OpenGameAgent.Example/packages.lock.json b/examples/OpenGameAgent.Example/packages.lock.json index a0981f7..e017ba2 100644 --- a/examples/OpenGameAgent.Example/packages.lock.json +++ b/examples/OpenGameAgent.Example/packages.lock.json @@ -14,9 +14,13 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/src/OpenGameAgent.Attachments.Local/FileGameImageAttachmentStore.cs b/src/OpenGameAgent.Attachments.Local/FileGameImageAttachmentStore.cs new file mode 100644 index 0000000..e299098 --- /dev/null +++ b/src/OpenGameAgent.Attachments.Local/FileGameImageAttachmentStore.cs @@ -0,0 +1,512 @@ +using System; +using System.Buffers; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using OpenGameAgent.Attachments; +using SkiaSharp; + +namespace OpenGameAgent.Attachments.Local; + +public sealed class FileGameImageAttachmentStore : IGameImageAttachmentStore +{ + private const string IdPrefix = "sha256:"; + private readonly string _root; + + public FileGameImageAttachmentStore(string rootDirectory, GameImageAttachmentLimits? imageLimits = null) + { + if (string.IsNullOrWhiteSpace(rootDirectory)) + { + throw new ArgumentException("An attachment storage directory is required.", nameof(rootDirectory)); + } + + _root = Path.GetFullPath(rootDirectory); + ImageLimits = imageLimits ?? new GameImageAttachmentLimits(); + } + + public GameImageAttachmentLimits ImageLimits { get; } + + public async ValueTask ValidateImageAsync( + SaveGameImageAttachment input, + CancellationToken cancellationToken = default) + { + _ = await InspectAsync(input, cancellationToken).ConfigureAwait(false); + } + + public async ValueTask SaveImageAsync( + SaveGameImageAttachment input, + CancellationToken cancellationToken = default) + { + var data = input.Data.ToArray(); + var metadata = await InspectAsync(input, data, cancellationToken).ConfigureAwait(false); + var hash = ComputeSha256(data); + var bucket = Path.Combine(_root, "objects", hash.Substring(0, 2)); + var staging = Path.Combine(_root, "tmp"); + EnsurePrivateDirectory(bucket); + EnsurePrivateDirectory(staging); + var target = Path.Combine(bucket, hash); + var temporary = Path.Combine(staging, Guid.NewGuid().ToString("N", CultureInfo.InvariantCulture)); + try + { + await WriteDurablyAsync(temporary, data, cancellationToken).ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + try + { + File.Move(temporary, target); + } + catch (IOException) when (File.Exists(target)) + { + var existing = await ReadBoundedAsync(target, data.Length, cancellationToken).ConfigureAwait(false); + if (!FixedEquals(hash, ComputeSha256(existing))) + { + throw new GameAttachmentException( + "ATTACHMENT_CORRUPT", + "The stored attachment failed integrity verification."); + } + } + + FilePermissions.TryRestrictFile(target); + DirectoryDurability.TrySync(bucket); + DirectoryDurability.TrySync(Path.Combine(_root, "objects")); + } + catch (OperationCanceledException) + { + throw; + } + catch (GameAttachmentException) + { + throw; + } + catch (Exception exception) + { + throw new GameAttachmentException( + "ATTACHMENT_WRITE_FAILED", + "Unable to persist the image attachment.", + exception); + } + finally + { + TryDelete(temporary); + } + + return new GameImageAttachment( + IdPrefix + hash, + metadata.MediaType, + data.Length, + metadata.Width, + metadata.Height, + SanitizeName(input.Name)); + } + + public async ValueTask ReadImageAsync( + GameImageAttachment attachment, + CancellationToken cancellationToken = default) + { + if (attachment is null) + { + throw new ArgumentNullException(nameof(attachment)); + } + + cancellationToken.ThrowIfCancellationRequested(); + ValidateReference(attachment); + var hash = ParseAttachmentId(attachment.AttachmentId); + var path = Path.Combine(_root, "objects", hash.Substring(0, 2), hash); + byte[] data; + try + { + data = await ReadBoundedAsync(path, attachment.Bytes, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (FileNotFoundException exception) + { + throw new GameAttachmentException("ATTACHMENT_NOT_FOUND", "The attachment object is missing.", exception); + } + catch (DirectoryNotFoundException exception) + { + throw new GameAttachmentException("ATTACHMENT_NOT_FOUND", "The attachment object is missing.", exception); + } + catch (GameAttachmentException) + { + throw; + } + catch (Exception exception) + { + throw new GameAttachmentException("ATTACHMENT_READ_FAILED", "Unable to read the image attachment.", exception); + } + + cancellationToken.ThrowIfCancellationRequested(); + if (!FixedEquals(hash, ComputeSha256(data))) + { + throw new GameAttachmentException("ATTACHMENT_CORRUPT", "The stored attachment failed integrity verification."); + } + + var metadata = Probe(data); + cancellationToken.ThrowIfCancellationRequested(); + if (!string.Equals(metadata.MediaType, attachment.MediaType, StringComparison.Ordinal) + || data.Length != attachment.Bytes + || metadata.Width != attachment.Width + || metadata.Height != attachment.Height) + { + throw new GameAttachmentException("ATTACHMENT_CORRUPT", "Stored attachment metadata does not match its reference."); + } + + return new StoredGameImageAttachment(attachment, data); + } + + private async ValueTask InspectAsync( + SaveGameImageAttachment input, + CancellationToken cancellationToken) + { + var data = input?.Data.ToArray() ?? throw new ArgumentNullException(nameof(input)); + return await InspectAsync(input, data, cancellationToken).ConfigureAwait(false); + } + + private async ValueTask InspectAsync( + SaveGameImageAttachment input, + byte[] data, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (data.Length == 0) + { + throw new GameAttachmentException("INVALID_IMAGE", "The image is empty."); + } + + if (data.Length > ImageLimits.MaxImageBytes) + { + throw new GameAttachmentException("IMAGE_TOO_LARGE", "The image exceeds the configured byte limit."); + } + + if (!ImageLimits.MediaTypes.Contains(input.MediaType, StringComparer.Ordinal)) + { + throw new GameAttachmentException("UNSUPPORTED_IMAGE_TYPE", "The image media type is not accepted by this deployment."); + } + + var metadata = Probe(data); + if (!string.Equals(metadata.MediaType, input.MediaType, StringComparison.Ordinal)) + { + throw new GameAttachmentException("IMAGE_TYPE_MISMATCH", "The declared image type does not match its bytes."); + } + + if ((long)metadata.Width * metadata.Height > ImageLimits.MaxImagePixels) + { + throw new GameAttachmentException("IMAGE_TOO_MANY_PIXELS", "The image exceeds the configured decoded-pixel limit."); + } + + await Task.Run(() => DecodeFully(data, metadata), cancellationToken).ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + return metadata; + } + + private void ValidateReference(GameImageAttachment attachment) + { + if (attachment.Bytes > ImageLimits.MaxImageBytes + || (long)attachment.Width * attachment.Height > ImageLimits.MaxImagePixels + || !ImageLimits.MediaTypes.Contains(attachment.MediaType, StringComparer.Ordinal)) + { + throw new GameAttachmentException( + "INVALID_ATTACHMENT_REF", + "The attachment reference exceeds this store's admission policy."); + } + } + + private static ImageMetadata Probe(byte[] data) + { + try + { + using var skData = SKData.CreateCopy(data); + using var codec = SKCodec.Create(skData); + if (codec is null || codec.Info.Width <= 0 || codec.Info.Height <= 0) + { + throw new GameAttachmentException("INVALID_IMAGE", "Unsupported or malformed image data."); + } + + return new ImageMetadata(ToMediaType(codec.EncodedFormat), codec.Info.Width, codec.Info.Height); + } + catch (GameAttachmentException) + { + throw; + } + catch (Exception exception) + { + throw new GameAttachmentException("INVALID_IMAGE", "Unsupported or malformed image data.", exception); + } + } + + private static void DecodeFully(byte[] data, ImageMetadata metadata) + { + try + { + using var skData = SKData.CreateCopy(data); + using var codec = SKCodec.Create(skData); + if (codec is null) + { + throw new GameAttachmentException("INVALID_IMAGE", "Unsupported or malformed image data."); + } + + var info = new SKImageInfo(metadata.Width, metadata.Height, SKColorType.Rgba8888, SKAlphaType.Unpremul); + var byteCount = checked(info.RowBytes * metadata.Height); + var pixels = ArrayPool.Shared.Rent(byteCount); + try + { + var handle = GCHandle.Alloc(pixels, GCHandleType.Pinned); + try + { + var result = codec.GetPixels( + info, + handle.AddrOfPinnedObject(), + info.RowBytes, + new SKCodecOptions()); + if (result != SKCodecResult.Success) + { + throw new GameAttachmentException("INVALID_IMAGE", "Unsupported or malformed image data."); + } + } + finally + { + handle.Free(); + } + } + finally + { + ArrayPool.Shared.Return(pixels, clearArray: true); + } + } + catch (GameAttachmentException) + { + throw; + } + catch (Exception exception) + { + throw new GameAttachmentException("INVALID_IMAGE", "Unsupported or malformed image data.", exception); + } + } + + private static string ToMediaType(SKEncodedImageFormat format) => format switch + { + SKEncodedImageFormat.Png => GameImageMediaTypes.Png, + SKEncodedImageFormat.Jpeg => GameImageMediaTypes.Jpeg, + SKEncodedImageFormat.Webp => GameImageMediaTypes.WebP, + SKEncodedImageFormat.Gif => GameImageMediaTypes.Gif, + _ => throw new GameAttachmentException("INVALID_IMAGE", "Unsupported or malformed image data."), + }; + + private static string ComputeSha256(byte[] data) + { + using var sha256 = SHA256.Create(); + var hash = sha256.ComputeHash(data); + var builder = new StringBuilder(hash.Length * 2); + foreach (var value in hash) + { + _ = builder.Append(value.ToString("x2", CultureInfo.InvariantCulture)); + } + + return builder.ToString(); + } + + private static bool FixedEquals(string expected, string actual) + { + if (expected.Length != actual.Length) + { + return false; + } + + var difference = 0; + for (var index = 0; index < expected.Length; index++) + { + difference |= expected[index] ^ actual[index]; + } + + return difference == 0; + } + + private static string ParseAttachmentId(string value) + { + if (!value.StartsWith(IdPrefix, StringComparison.Ordinal) || value.Length != IdPrefix.Length + 64) + { + throw new GameAttachmentException("INVALID_ATTACHMENT_REF", "The attachment reference is invalid."); + } + + var hash = value.Substring(IdPrefix.Length); + foreach (var character in hash) + { + if (!((character >= '0' && character <= '9') || (character >= 'a' && character <= 'f'))) + { + throw new GameAttachmentException("INVALID_ATTACHMENT_REF", "The attachment reference is invalid."); + } + } + + return hash; + } + + private static async Task WriteDurablyAsync(string path, byte[] data, CancellationToken cancellationToken) + { + using var stream = new FileStream( + path, + FileMode.CreateNew, + FileAccess.Write, + FileShare.None, + 64 * 1024, + FileOptions.Asynchronous | FileOptions.WriteThrough); + await stream.WriteAsync(data, 0, data.Length, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + stream.Flush(flushToDisk: true); + FilePermissions.TryRestrictFile(path); + } + + private static async Task ReadBoundedAsync(string path, int expectedBytes, CancellationToken cancellationToken) + { + using var stream = new FileStream( + path, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + 64 * 1024, + FileOptions.Asynchronous | FileOptions.SequentialScan); + if (stream.Length != expectedBytes || stream.Length <= 0 || stream.Length > int.MaxValue) + { + throw new GameAttachmentException("ATTACHMENT_CORRUPT", "Stored attachment length does not match its reference."); + } + + var data = new byte[expectedBytes]; + var offset = 0; + while (offset < data.Length) + { + var read = await stream.ReadAsync(data, offset, data.Length - offset, cancellationToken).ConfigureAwait(false); + if (read == 0) + { + throw new GameAttachmentException("ATTACHMENT_CORRUPT", "The stored attachment ended unexpectedly."); + } + + offset += read; + } + + return data; + } + + private static void EnsurePrivateDirectory(string path) + { + Directory.CreateDirectory(path); + FilePermissions.TryRestrictDirectory(path); + } + + private static string? SanitizeName(string? value) + { + if (value is null) + { + return null; + } + + var slash = Math.Max(value.LastIndexOf('/'), value.LastIndexOf('\\')); + var leaf = slash >= 0 ? value.Substring(slash + 1) : value; + var builder = new StringBuilder(Math.Min(leaf.Length, 255)); + foreach (var character in leaf) + { + if (!char.IsControl(character) && builder.Length < 255) + { + _ = builder.Append(character); + } + } + + var clean = builder.ToString().Trim(); + return clean.Length == 0 ? null : clean; + } + + private static void TryDelete(string path) + { + try + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + catch + { + // A stale private staging object is harmless and can be removed by host maintenance. + } + } + + private readonly struct ImageMetadata + { + public ImageMetadata(string mediaType, int width, int height) + { + MediaType = mediaType; + Width = width; + Height = height; + } + + public string MediaType { get; } + + public int Width { get; } + + public int Height { get; } + } + + private static class FilePermissions + { + public static void TryRestrictDirectory(string path) + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + _ = Chmod(path, Convert.ToUInt32("700", 8)); + } + } + + public static void TryRestrictFile(string path) + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + _ = Chmod(path, Convert.ToUInt32("600", 8)); + } + } + + [DllImport("libc", EntryPoint = "chmod", SetLastError = true)] + private static extern int Chmod(string path, uint mode); + } + + private static class DirectoryDurability + { + private const int ReadOnly = 0; + + public static void TrySync(string path) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) || !Directory.Exists(path)) + { + return; + } + + var descriptor = Open(path, ReadOnly); + if (descriptor < 0) + { + return; + } + + try + { + _ = Fsync(descriptor); + } + finally + { + _ = Close(descriptor); + } + } + + [DllImport("libc", EntryPoint = "open", SetLastError = true)] + private static extern int Open(string path, int flags); + + [DllImport("libc", EntryPoint = "fsync", SetLastError = true)] + private static extern int Fsync(int descriptor); + + [DllImport("libc", EntryPoint = "close", SetLastError = true)] + private static extern int Close(int descriptor); + } +} diff --git a/src/OpenGameAgent.Attachments.Local/OpenGameAgent.Attachments.Local.csproj b/src/OpenGameAgent.Attachments.Local/OpenGameAgent.Attachments.Local.csproj new file mode 100644 index 0000000..39f53c4 --- /dev/null +++ b/src/OpenGameAgent.Attachments.Local/OpenGameAgent.Attachments.Local.csproj @@ -0,0 +1,14 @@ + + + netstandard2.1 + OpenGameAgent.Attachments.Local + Private content-addressed local image attachment storage for OpenGameAgent. + + + + + + + + + diff --git a/src/OpenGameAgent.Attachments.Local/packages.lock.json b/src/OpenGameAgent.Attachments.Local/packages.lock.json new file mode 100644 index 0000000..5954829 --- /dev/null +++ b/src/OpenGameAgent.Attachments.Local/packages.lock.json @@ -0,0 +1,42 @@ +{ + "version": 1, + "dependencies": { + ".NETStandard,Version=v2.1": { + "SkiaSharp": { + "type": "Direct", + "requested": "[4.150.1, )", + "resolved": "4.150.1", + "contentHash": "5v3T8X1N62Dp+AkPO70GNBNS/NRBPGMOTiN+Prg33sZAcm/Ug3YOAH+3RTj/jxJV8NGTJs2idGpC2Qdae2mGLQ==", + "dependencies": { + "SkiaSharp.NativeAssets.Win32": "4.150.1", + "SkiaSharp.NativeAssets.macOS": "4.150.1", + "System.Runtime.CompilerServices.Unsafe": "6.1.2" + } + }, + "SkiaSharp.NativeAssets.Linux.NoDependencies": { + "type": "Direct", + "requested": "[4.150.1, )", + "resolved": "4.150.1", + "contentHash": "2KVadgDky2xQw7lEMRT/u0ftX5K1u8X7MhCN47Em22Z6VM6JJg1c9cwfGuSPAt1oY6+GjGHH18vkeev+nqT9Kw==" + }, + "SkiaSharp.NativeAssets.macOS": { + "type": "Transitive", + "resolved": "4.150.1", + "contentHash": "r755HVwaHZhyf1clWjrM2/RoOZYCzkQEmE9pu/mVsebPejWu52niNPUwtfyf112qoF0PIk6OndqVOUoITj6TwQ==" + }, + "SkiaSharp.NativeAssets.Win32": { + "type": "Transitive", + "resolved": "4.150.1", + "contentHash": "qrLSL8OonbkMJdSH8heK0Jl39Y2xxZGdd7Ru4cyBXk2ITdx+Fu1sCFLmvCKmnMaYs//mlN/YSpSpOurEXErcQw==" + }, + "System.Runtime.CompilerServices.Unsafe": { + "type": "Transitive", + "resolved": "6.1.2", + "contentHash": "2hBr6zdbIBTDE3EhK7NSVNdX58uTK6iHW/P/Axmm9sl1xoGSLqDvMtpecn226TNwHByFokYwJmt/aQQNlO5CRw==" + }, + "opengameagent.attachments": { + "type": "Project" + } + } + } +} \ No newline at end of file diff --git a/src/OpenGameAgent.Attachments/ImageAttachments.cs b/src/OpenGameAgent.Attachments/ImageAttachments.cs new file mode 100644 index 0000000..c46d85f --- /dev/null +++ b/src/OpenGameAgent.Attachments/ImageAttachments.cs @@ -0,0 +1,255 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace OpenGameAgent.Attachments; + +public static class GameImageMediaTypes +{ + public const string Png = "image/png"; + public const string Jpeg = "image/jpeg"; + public const string WebP = "image/webp"; + public const string Gif = "image/gif"; + + public static IReadOnlyList Raster { get; } = Array.AsReadOnly(new[] + { + Png, + Jpeg, + WebP, + Gif, + }); + + public static bool IsRaster(string mediaType) => + string.Equals(mediaType, Png, StringComparison.Ordinal) + || string.Equals(mediaType, Jpeg, StringComparison.Ordinal) + || string.Equals(mediaType, WebP, StringComparison.Ordinal) + || string.Equals(mediaType, Gif, StringComparison.Ordinal); +} + +public sealed class GameImageAttachmentLimits +{ + public const int DefaultMaxImageBytes = 5 * 1024 * 1024; + public const int DefaultMaxImagesPerMessage = 20; + public const int DefaultMaxMessageImageBytes = 100 * 1024 * 1024; + public const long DefaultMaxImagePixels = 40_000_000; + + public GameImageAttachmentLimits( + int maxImageBytes = DefaultMaxImageBytes, + int maxImagesPerMessage = DefaultMaxImagesPerMessage, + int maxMessageImageBytes = DefaultMaxMessageImageBytes, + long maxImagePixels = DefaultMaxImagePixels, + IReadOnlyList? mediaTypes = null) + { + if (maxImageBytes <= 0 || maxImageBytes > 512 * 1024 * 1024) + { + throw new ArgumentOutOfRangeException(nameof(maxImageBytes)); + } + + if (maxImagesPerMessage <= 0 || maxImagesPerMessage > 1_024) + { + throw new ArgumentOutOfRangeException(nameof(maxImagesPerMessage)); + } + + if (maxMessageImageBytes < maxImageBytes || maxMessageImageBytes > 1024 * 1024 * 1024) + { + throw new ArgumentOutOfRangeException(nameof(maxMessageImageBytes)); + } + + if (maxImagePixels <= 0 || maxImagePixels > 1_000_000_000) + { + throw new ArgumentOutOfRangeException(nameof(maxImagePixels)); + } + + var accepted = mediaTypes is null ? GameImageMediaTypes.Raster : new ReadOnlyCollection(CopyMediaTypes(mediaTypes)); + MaxImageBytes = maxImageBytes; + MaxImagesPerMessage = maxImagesPerMessage; + MaxMessageImageBytes = maxMessageImageBytes; + MaxImagePixels = maxImagePixels; + MediaTypes = accepted; + } + + public int MaxImageBytes { get; } + + public int MaxImagesPerMessage { get; } + + public int MaxMessageImageBytes { get; } + + public long MaxImagePixels { get; } + + public IReadOnlyList MediaTypes { get; } + + private static string[] CopyMediaTypes(IReadOnlyList values) + { + if (values.Count == 0 || values.Count > 64) + { + throw new ArgumentOutOfRangeException(nameof(values)); + } + + var copy = new string[values.Count]; + var seen = new HashSet(StringComparer.Ordinal); + for (var index = 0; index < values.Count; index++) + { + var value = values[index]; + if (!GameImageMediaTypes.IsRaster(value) || !seen.Add(value)) + { + throw new ArgumentException("Image media types must be unique supported raster values.", nameof(values)); + } + + copy[index] = value; + } + + return copy; + } +} + +public sealed class GameImageAttachment +{ + public GameImageAttachment( + string attachmentId, + string mediaType, + int bytes, + int width, + int height, + string? name = null) + { + if (string.IsNullOrWhiteSpace(attachmentId) || attachmentId.Length > 256 || ContainsControl(attachmentId)) + { + throw new ArgumentException("A bounded opaque attachment ID is required.", nameof(attachmentId)); + } + + if (!GameImageMediaTypes.IsRaster(mediaType)) + { + throw new ArgumentException("The image media type is unsupported.", nameof(mediaType)); + } + + if (bytes <= 0) + { + throw new ArgumentOutOfRangeException(nameof(bytes)); + } + + if (width <= 0) + { + throw new ArgumentOutOfRangeException(nameof(width)); + } + + if (height <= 0) + { + throw new ArgumentOutOfRangeException(nameof(height)); + } + + if (name is { Length: > 255 } || ContainsControl(name)) + { + throw new ArgumentException("The attachment name is invalid.", nameof(name)); + } + + AttachmentId = attachmentId; + MediaType = mediaType; + Bytes = bytes; + Width = width; + Height = height; + Name = string.IsNullOrWhiteSpace(name) ? null : name; + } + + public string AttachmentId { get; } + + public string MediaType { get; } + + public int Bytes { get; } + + public int Width { get; } + + public int Height { get; } + + public string? Name { get; } + + private static bool ContainsControl(string? value) + { + if (value is null) + { + return false; + } + + foreach (var character in value) + { + if (char.IsControl(character)) + { + return true; + } + } + + return false; + } +} + +public sealed class SaveGameImageAttachment +{ + private readonly byte[] _data; + + public SaveGameImageAttachment(byte[] data, string mediaType, string? name = null) + { + _data = data is null ? throw new ArgumentNullException(nameof(data)) : (byte[])data.Clone(); + if (!GameImageMediaTypes.IsRaster(mediaType)) + { + throw new ArgumentException("The image media type is unsupported.", nameof(mediaType)); + } + + if (name is { Length: > 4096 } || name?.Any(char.IsControl) == true) + { + throw new ArgumentException("The source image name is invalid.", nameof(name)); + } + + MediaType = mediaType; + Name = name; + } + + public ReadOnlyMemory Data => _data; + + public string MediaType { get; } + + public string? Name { get; } +} + +public sealed class StoredGameImageAttachment +{ + private readonly byte[] _data; + + public StoredGameImageAttachment(GameImageAttachment attachment, byte[] data) + { + Attachment = attachment ?? throw new ArgumentNullException(nameof(attachment)); + _data = data is null ? throw new ArgumentNullException(nameof(data)) : (byte[])data.Clone(); + } + + public GameImageAttachment Attachment { get; } + + public ReadOnlyMemory Data => _data; +} + +public interface IGameImageAttachmentStore +{ + GameImageAttachmentLimits ImageLimits { get; } + + ValueTask ValidateImageAsync(SaveGameImageAttachment input, CancellationToken cancellationToken = default); + + ValueTask SaveImageAsync(SaveGameImageAttachment input, CancellationToken cancellationToken = default); + + ValueTask ReadImageAsync(GameImageAttachment attachment, CancellationToken cancellationToken = default); +} + +public sealed class GameAttachmentException : Exception +{ + public GameAttachmentException(string code, string message, Exception? innerException = null) + : base(message, innerException) + { + if (string.IsNullOrWhiteSpace(code) || code.Length > 128) + { + throw new ArgumentException("A bounded attachment error code is required.", nameof(code)); + } + + Code = code; + } + + public string Code { get; } +} diff --git a/src/OpenGameAgent.Attachments/OpenGameAgent.Attachments.csproj b/src/OpenGameAgent.Attachments/OpenGameAgent.Attachments.csproj new file mode 100644 index 0000000..d90ab6e --- /dev/null +++ b/src/OpenGameAgent.Attachments/OpenGameAgent.Attachments.csproj @@ -0,0 +1,7 @@ + + + netstandard2.1 + OpenGameAgent.Attachments + Durable, provider-neutral image attachment contracts for OpenGameAgent. + + diff --git a/src/OpenGameAgent.Attachments/packages.lock.json b/src/OpenGameAgent.Attachments/packages.lock.json new file mode 100644 index 0000000..034482b --- /dev/null +++ b/src/OpenGameAgent.Attachments/packages.lock.json @@ -0,0 +1,6 @@ +{ + "version": 1, + "dependencies": { + ".NETStandard,Version=v2.1": {} + } +} \ No newline at end of file diff --git a/src/OpenGameAgent.Client/OpenGameAgent.Client.csproj b/src/OpenGameAgent.Client/OpenGameAgent.Client.csproj index 87c3027..1932f8f 100644 --- a/src/OpenGameAgent.Client/OpenGameAgent.Client.csproj +++ b/src/OpenGameAgent.Client/OpenGameAgent.Client.csproj @@ -9,5 +9,6 @@ + diff --git a/src/OpenGameAgent.Client/ServerGameAgentClient.cs b/src/OpenGameAgent.Client/ServerGameAgentClient.cs index 782b005..5fec717 100644 --- a/src/OpenGameAgent.Client/ServerGameAgentClient.cs +++ b/src/OpenGameAgent.Client/ServerGameAgentClient.cs @@ -2,6 +2,7 @@ using System.Buffers; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Net.Http; using System.Net.Http.Headers; using System.Runtime.CompilerServices; @@ -9,6 +10,7 @@ using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using OpenGameAgent.Attachments; namespace OpenGameAgent.Client; @@ -99,6 +101,8 @@ public ServerGameAgentClientOptions(HttpClient httpClient, Uri serverBaseUri) public string AbortPath { get; set; } = "v1/control/abort"; + public string AttachmentReadPath { get; set; } = "v1/attachments/read"; + public string? ApiKey { get; set; } public string ApiKeyHeader { get; set; } = "Authorization"; @@ -121,6 +125,7 @@ public sealed class ServerGameAgentClient private readonly Uri _streamEndpoint; private readonly Uri _steerEndpoint; private readonly Uri _abortEndpoint; + private readonly Uri _attachmentReadEndpoint; private readonly string? _apiKey; private readonly string _apiKeyHeader; private readonly string _apiKeyScheme; @@ -198,6 +203,10 @@ public ServerGameAgentClient(ServerGameAgentClientOptions options) _streamEndpoint = CreateEndpoint(options.ServerBaseUri, options.StreamPath, nameof(options.StreamPath)); _steerEndpoint = CreateEndpoint(options.ServerBaseUri, options.SteerPath, nameof(options.SteerPath)); _abortEndpoint = CreateEndpoint(options.ServerBaseUri, options.AbortPath, nameof(options.AbortPath)); + _attachmentReadEndpoint = CreateEndpoint( + options.ServerBaseUri, + options.AttachmentReadPath, + nameof(options.AttachmentReadPath)); _apiKey = options.ApiKey; _apiKeyHeader = options.ApiKeyHeader; _apiKeyScheme = options.ApiKeyScheme ?? string.Empty; @@ -372,6 +381,114 @@ public Task AbortAsync( return SendControlAsync(_abortEndpoint, json, cancellationToken); } + public async Task ReadImageAttachmentAsync( + GameSessionKey key, + string attachmentId, + CancellationToken cancellationToken = default) + { + key.EnsureValidForClient(nameof(key)); + if (string.IsNullOrWhiteSpace(attachmentId) + || attachmentId.Length > 256 + || attachmentId.Any(static character => char.IsControl(character))) + { + throw new ArgumentException("A bounded attachment ID is required.", nameof(attachmentId)); + } + + var json = JsonSerializer.Serialize(new + { + sessionId = key.SessionId, + actorId = key.ActorId, + attachmentId, + }); + using var request = CreateJsonRequest(_attachmentReadEndpoint, json); + using var response = await _httpClient.SendAsync( + request, + HttpCompletionOption.ResponseHeadersRead, + cancellationToken).ConfigureAwait(false); + var body = await ReadBoundedAsync(response.Content, _maxResponseCharacters, cancellationToken).ConfigureAwait(false); + if (response.StatusCode == System.Net.HttpStatusCode.NotFound) + { + return null; + } + + EnsureSuccess(response, body); + return ParseAttachment(body); + } + + private static StoredGameImageAttachment ParseAttachment(string json) + { + using var document = RemoteJson.Parse(json, nameof(json)); + var root = document.RootElement; + if (root.ValueKind != JsonValueKind.Object + || !root.TryGetProperty("attachment", out var descriptor) + || descriptor.ValueKind != JsonValueKind.Object + || !root.TryGetProperty("data", out var dataElement) + || dataElement.ValueKind != JsonValueKind.String) + { + throw new InvalidDataException("The attachment response does not match the expected shape."); + } + + var attachmentId = RequireString(descriptor, "attachmentId"); + var mediaType = RequireString(descriptor, "mediaType"); + var bytes = RequireInt32(descriptor, "bytes"); + var width = RequireInt32(descriptor, "width"); + var height = RequireInt32(descriptor, "height"); + string? name = null; + if (descriptor.TryGetProperty("name", out var nameElement) + && nameElement.ValueKind != JsonValueKind.Null) + { + if (nameElement.ValueKind != JsonValueKind.String) + { + throw new InvalidDataException("The attachment name must be a string or null."); + } + + name = nameElement.GetString(); + } + + byte[] data; + try + { + data = Convert.FromBase64String(dataElement.GetString()!); + } + catch (FormatException exception) + { + throw new InvalidDataException("The attachment response contains invalid base64 data.", exception); + } + + if (data.Length != bytes) + { + throw new InvalidDataException("The attachment response length does not match its descriptor."); + } + + return new StoredGameImageAttachment( + new GameImageAttachment(attachmentId, mediaType, bytes, width, height, name), + data); + } + + private static string RequireString(JsonElement value, string propertyName) + { + if (!value.TryGetProperty(propertyName, out var property) + || property.ValueKind != JsonValueKind.String + || string.IsNullOrWhiteSpace(property.GetString())) + { + throw new InvalidDataException("The attachment response is missing '" + propertyName + "'."); + } + + return property.GetString()!; + } + + private static int RequireInt32(JsonElement value, string propertyName) + { + if (!value.TryGetProperty(propertyName, out var property) + || !property.TryGetInt32(out var result) + || result <= 0) + { + throw new InvalidDataException("The attachment response has an invalid '" + propertyName + "'."); + } + + return result; + } + private async Task SendControlAsync( Uri endpoint, string json, diff --git a/src/OpenGameAgent.Client/packages.lock.json b/src/OpenGameAgent.Client/packages.lock.json index 1a59cbd..780b8b1 100644 --- a/src/OpenGameAgent.Client/packages.lock.json +++ b/src/OpenGameAgent.Client/packages.lock.json @@ -71,9 +71,13 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } } diff --git a/src/OpenGameAgent.Connectors.Mcp/packages.lock.json b/src/OpenGameAgent.Connectors.Mcp/packages.lock.json index 5487085..2a16af2 100644 --- a/src/OpenGameAgent.Connectors.Mcp/packages.lock.json +++ b/src/OpenGameAgent.Connectors.Mcp/packages.lock.json @@ -150,6 +150,9 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.extensions": { "type": "Project", "dependencies": { @@ -160,6 +163,7 @@ "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/src/OpenGameAgent.Extensions/packages.lock.json b/src/OpenGameAgent.Extensions/packages.lock.json index 10391a2..c1ab240 100644 --- a/src/OpenGameAgent.Extensions/packages.lock.json +++ b/src/OpenGameAgent.Extensions/packages.lock.json @@ -70,9 +70,13 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/src/OpenGameAgent.Kernel/AgentOptions.cs b/src/OpenGameAgent.Kernel/AgentOptions.cs index 334292b..a8edf6f 100644 --- a/src/OpenGameAgent.Kernel/AgentOptions.cs +++ b/src/OpenGameAgent.Kernel/AgentOptions.cs @@ -37,6 +37,14 @@ public sealed class AgentLimits public int MaxBinaryDataCharactersPerPart { get; set; } = 16_000_000; + public int MaxImagesPerMessage { get; set; } = 20; + + public int MaxImageBytes { get; set; } = 5 * 1024 * 1024; + + public int MaxImageBytesPerMessage { get; set; } = 100 * 1024 * 1024; + + public long MaxImagePixels { get; set; } = 40_000_000; + public int MaxToolCallsPerTurn { get; set; } = 32; public int MaxTools { get; set; } = 256; @@ -91,6 +99,10 @@ internal void Validate() RequireRange(MaxJsonCharactersPerPart, 1, 100_000_000, nameof(MaxJsonCharactersPerPart)); RequireRange(MaxResourceUriCharacters, 1, 1_000_000, nameof(MaxResourceUriCharacters)); RequireRange(MaxBinaryDataCharactersPerPart, 1, 100_000_000, nameof(MaxBinaryDataCharactersPerPart)); + RequireRange(MaxImagesPerMessage, 1, 1_024, nameof(MaxImagesPerMessage)); + RequireRange(MaxImageBytes, 1, 512 * 1024 * 1024, nameof(MaxImageBytes)); + RequireRange(MaxImageBytesPerMessage, MaxImageBytes, 1024 * 1024 * 1024, nameof(MaxImageBytesPerMessage)); + RequireRange(MaxImagePixels, 1, 1_000_000_000, nameof(MaxImagePixels)); RequireRange(MaxToolCallsPerTurn, 1, 10_000, nameof(MaxToolCallsPerTurn)); RequireRange(MaxTools, 0, 100_000, nameof(MaxTools)); RequireRange(MaxToolNameCharacters, 1, 4096, nameof(MaxToolNameCharacters)); diff --git a/src/OpenGameAgent.Kernel/AgentValidator.cs b/src/OpenGameAgent.Kernel/AgentValidator.cs index 320d1b6..48a85d2 100644 --- a/src/OpenGameAgent.Kernel/AgentValidator.cs +++ b/src/OpenGameAgent.Kernel/AgentValidator.cs @@ -245,6 +245,15 @@ public static void ValidateMessage(AgentMessage message, AgentLimits limits) ValidateContent(content, limits); } + ValidateImageCollection(message.Content, limits); + if (message.Role is not AgentRole.User and not AgentRole.Tool + && message.Content.Any(IsImageContent)) + { + throw new ArgumentException( + "Images are supported only in user input and tool results.", + nameof(message)); + } + if (message.DetailsJson is { } details && details.Length > limits.MaxJsonCharactersPerPart) { throw new AgentLimitException(nameof(limits.MaxJsonCharactersPerPart), "Tool result details are too large."); @@ -323,6 +332,14 @@ public static void ValidateResponse(ModelResponse response, AgentLimits limits) ValidateContent(content, limits); } + ValidateImageCollection(response.Content, limits); + if (response.Content.Any(IsImageContent)) + { + throw new ArgumentException( + "Model responses cannot contain images. Use the media generation pipeline for generated assets.", + nameof(response)); + } + var calls = response.Content.Count(part => part is ToolCallContent); if (calls > limits.MaxToolCallsPerTurn) { @@ -399,6 +416,8 @@ public static void ValidateToolResult(ToolResult result, AgentLimits limits) ValidateContent(content, limits); } + ValidateImageCollection(result.Content, limits); + if (result.DetailsJson is { } details && details.Length > limits.MaxJsonCharactersPerPart) { throw new AgentLimitException(nameof(limits.MaxJsonCharactersPerPart), "Tool result details are too large."); @@ -448,6 +467,8 @@ public static void ValidateProgress(ToolProgress progress, AgentLimits limits) { ValidateContent(content, limits); } + + ValidateImageCollection(progress.Content, limits); } public static void ValidateRequest( @@ -533,6 +554,14 @@ private static void ValidateContent(AgentContent content, AgentLimits limits) throw new AgentLimitException(nameof(limits.MaxMetadataValueCharacters), "A resource media type is too large."); case ResourceContent resource when (resource.Name?.Length ?? 0) > limits.MaxTextCharactersPerPart: throw new AgentLimitException(nameof(limits.MaxTextCharactersPerPart), "A resource name is too large."); + case ImageAttachmentContent image when image.Attachment.AttachmentId.Length > limits.MaxResourceUriCharacters: + throw new AgentLimitException(nameof(limits.MaxResourceUriCharacters), "An image attachment ID is too large."); + case ImageAttachmentContent image when image.Attachment.Bytes > limits.MaxImageBytes: + throw new AgentLimitException(nameof(limits.MaxImageBytes), "An image attachment is too large."); + case ImageAttachmentContent image when (long)image.Attachment.Width * image.Attachment.Height > limits.MaxImagePixels: + throw new AgentLimitException(nameof(limits.MaxImagePixels), "An image attachment has too many pixels."); + case ImageAttachmentContent image when (image.Attachment.Name?.Length ?? 0) > 255: + throw new AgentLimitException(nameof(limits.MaxMetadataValueCharacters), "An image attachment name is too large."); case BinaryContent binary when binary.Data.Length > limits.MaxBinaryDataCharactersPerPart: throw new AgentLimitException(nameof(limits.MaxBinaryDataCharactersPerPart), "An inline media part is too large."); case BinaryContent binary when binary.MediaType.Length > limits.MaxMetadataValueCharacters: @@ -549,13 +578,63 @@ private static void ValidateContent(AgentContent content, AgentLimits limits) throw new AgentLimitException(nameof(limits.MaxTextCharactersPerPart), "A tool-call thought signature is too large."); case ToolCallContent call when (call.Namespace?.Length ?? 0) > limits.MaxToolNameCharacters: throw new AgentLimitException(nameof(limits.MaxToolNameCharacters), "A tool-call namespace is too large."); - case TextContent or ReasoningContent or JsonContent or ResourceContent or BinaryContent or ToolCallContent: + case TextContent or ReasoningContent or JsonContent or ResourceContent or ImageAttachmentContent or BinaryContent or ToolCallContent: break; default: throw new ArgumentException($"Unsupported content type '{content.GetType().FullName}'.", nameof(content)); } } + private static void ValidateImageCollection(IReadOnlyList content, AgentLimits limits) + { + var count = 0; + long bytes = 0; + foreach (var part in content) + { + long imageBytes; + switch (part) + { + case ImageAttachmentContent image: + imageBytes = image.Attachment.Bytes; + break; + case BinaryContent { MediaKind: AgentMediaKind.Image } image: + try + { + imageBytes = Convert.FromBase64String(image.Data).LongLength; + } + catch (FormatException exception) + { + throw new ArgumentException("Inline image data is not valid base64.", nameof(content), exception); + } + + if (imageBytes > limits.MaxImageBytes) + { + throw new AgentLimitException(nameof(limits.MaxImageBytes), "An inline image is too large."); + } + + break; + default: + continue; + } + + count++; + bytes += imageBytes; + if (count > limits.MaxImagesPerMessage) + { + throw new AgentLimitException(nameof(limits.MaxImagesPerMessage), "A message contains too many images."); + } + + if (bytes > limits.MaxImageBytesPerMessage) + { + throw new AgentLimitException(nameof(limits.MaxImageBytesPerMessage), "A message contains too many image bytes."); + } + } + } + + private static bool IsImageContent(AgentContent content) => + content is ImageAttachmentContent + or BinaryContent { MediaKind: AgentMediaKind.Image }; + private static void ValidateDiagnostic(ModelDiagnostic diagnostic, AgentLimits limits) { if (diagnostic.Code.Length > limits.MaxMetadataKeyCharacters diff --git a/src/OpenGameAgent.Kernel/AssemblyInfo.cs b/src/OpenGameAgent.Kernel/AssemblyInfo.cs new file mode 100644 index 0000000..742e787 --- /dev/null +++ b/src/OpenGameAgent.Kernel/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("OpenGameAgent.Kernel.Tests")] diff --git a/src/OpenGameAgent.Kernel/Content.cs b/src/OpenGameAgent.Kernel/Content.cs index 2ea7372..cf751db 100644 --- a/src/OpenGameAgent.Kernel/Content.cs +++ b/src/OpenGameAgent.Kernel/Content.cs @@ -1,5 +1,6 @@ using System; using System.Text.Json; +using OpenGameAgent.Attachments; namespace OpenGameAgent.Kernel; @@ -8,6 +9,7 @@ public enum AgentContentKind Text, Json, Resource, + ImageAttachment, Binary, Reasoning, ToolCall, @@ -99,6 +101,17 @@ public ResourceContent(string uri, string mediaType, string? name = null) public string? Name { get; } } +public sealed class ImageAttachmentContent : AgentContent +{ + public ImageAttachmentContent(GameImageAttachment attachment) + : base(AgentContentKind.ImageAttachment) + { + Attachment = attachment ?? throw new ArgumentNullException(nameof(attachment)); + } + + public GameImageAttachment Attachment { get; } +} + public sealed class BinaryContent : AgentContent { public BinaryContent( diff --git a/src/OpenGameAgent.Kernel/Models.cs b/src/OpenGameAgent.Kernel/Models.cs index beb5ed3..a103b44 100644 --- a/src/OpenGameAgent.Kernel/Models.cs +++ b/src/OpenGameAgent.Kernel/Models.cs @@ -772,6 +772,15 @@ public interface IModelProvider IAsyncEnumerable StreamAsync(ModelRequest request, CancellationToken cancellationToken); } +/// +/// Validates a request before durable attachment bytes are loaded. +/// Implementations must not dispatch a provider request or mutate external state. +/// +public interface IModelRequestPreflight +{ + ValueTask ValidateRequestAsync(ModelRequest request, CancellationToken cancellationToken); +} + public interface IDeferredModelProvider : IModelProvider { IAsyncEnumerable FetchDeferredAsync( diff --git a/src/OpenGameAgent.Kernel/OpenGameAgent.Kernel.csproj b/src/OpenGameAgent.Kernel/OpenGameAgent.Kernel.csproj index 7bf0dbb..6dfee38 100644 --- a/src/OpenGameAgent.Kernel/OpenGameAgent.Kernel.csproj +++ b/src/OpenGameAgent.Kernel/OpenGameAgent.Kernel.csproj @@ -7,4 +7,7 @@ + + + diff --git a/src/OpenGameAgent.Kernel/packages.lock.json b/src/OpenGameAgent.Kernel/packages.lock.json index c7b4c4d..af028fe 100644 --- a/src/OpenGameAgent.Kernel/packages.lock.json +++ b/src/OpenGameAgent.Kernel/packages.lock.json @@ -63,6 +63,9 @@ "dependencies": { "System.Runtime.CompilerServices.Unsafe": "4.5.3" } + }, + "opengameagent.attachments": { + "type": "Project" } } } diff --git a/src/OpenGameAgent.Media/packages.lock.json b/src/OpenGameAgent.Media/packages.lock.json index 10391a2..c1ab240 100644 --- a/src/OpenGameAgent.Media/packages.lock.json +++ b/src/OpenGameAgent.Media/packages.lock.json @@ -70,9 +70,13 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/src/OpenGameAgent.Memory/packages.lock.json b/src/OpenGameAgent.Memory/packages.lock.json index a0174ed..01e695f 100644 --- a/src/OpenGameAgent.Memory/packages.lock.json +++ b/src/OpenGameAgent.Memory/packages.lock.json @@ -70,9 +70,13 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } } diff --git a/src/OpenGameAgent.Models.Auth.BuiltIn/packages.lock.json b/src/OpenGameAgent.Models.Auth.BuiltIn/packages.lock.json index d4f2537..745ea0c 100644 --- a/src/OpenGameAgent.Models.Auth.BuiltIn/packages.lock.json +++ b/src/OpenGameAgent.Models.Auth.BuiltIn/packages.lock.json @@ -126,9 +126,13 @@ "System.Runtime.CompilerServices.Unsafe": "4.5.3" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/src/OpenGameAgent.Models.BuiltIn/BuiltInGameModelRuntime.cs b/src/OpenGameAgent.Models.BuiltIn/BuiltInGameModelRuntime.cs index 5795ae6..ff42ccf 100644 --- a/src/OpenGameAgent.Models.BuiltIn/BuiltInGameModelRuntime.cs +++ b/src/OpenGameAgent.Models.BuiltIn/BuiltInGameModelRuntime.cs @@ -1201,14 +1201,14 @@ private static AgentContent NormalizeInputPart( { if (content is BinaryContent binary && !Supports(binary.MediaKind, capabilities)) { - return UnsupportedMediaPlaceholder(binary.MediaKind); + throw UnsupportedMedia(binary.MediaKind); } if (content is ResourceContent resource && MediaKind(resource.MediaType) is { } mediaKind && !Supports(mediaKind, capabilities)) { - return UnsupportedMediaPlaceholder(mediaKind); + throw UnsupportedMedia(mediaKind); } if (content is JsonContent json @@ -1249,8 +1249,10 @@ private static AgentContent NormalizeInputPart( return null; } - private static TextContent UnsupportedMediaPlaceholder(AgentMediaKind kind) => - new($"[{kind.ToString().ToLowerInvariant()} omitted: model does not support this input]"); + private static ModelProviderException UnsupportedMedia(AgentMediaKind kind) => + new( + $"The selected model does not declare {kind.ToString().ToLowerInvariant()} input support.", + isTransient: false); private static AgentMessage CopyMessage(AgentMessage message, IReadOnlyList content) => new( message.Role, diff --git a/src/OpenGameAgent.Models.BuiltIn/packages.lock.json b/src/OpenGameAgent.Models.BuiltIn/packages.lock.json index 0d19e3c..298c6b5 100644 --- a/src/OpenGameAgent.Models.BuiltIn/packages.lock.json +++ b/src/OpenGameAgent.Models.BuiltIn/packages.lock.json @@ -126,9 +126,13 @@ "System.Runtime.CompilerServices.Unsafe": "4.5.3" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/src/OpenGameAgent.Models/ProviderCatalog.cs b/src/OpenGameAgent.Models/ProviderCatalog.cs index 86aa155..8b8d106 100644 --- a/src/OpenGameAgent.Models/ProviderCatalog.cs +++ b/src/OpenGameAgent.Models/ProviderCatalog.cs @@ -1075,7 +1075,7 @@ private static bool Equivalent( && left.All(pair => right.TryGetValue(pair.Key, out var value) && EqualityComparer.Default.Equals(pair.Value, value)); - private sealed class CatalogDispatchProvider : IModelProvider + private sealed class CatalogDispatchProvider : IModelProvider, IModelRequestPreflight { private readonly GameModelCatalog _catalog; private readonly string _providerId; @@ -1090,6 +1090,48 @@ public IAsyncEnumerable StreamAsync( ModelRequest request, CancellationToken cancellationToken) => _catalog.StreamAsync(_providerId, request, cancellationToken); + + public ValueTask ValidateRequestAsync( + ModelRequest request, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var model = _catalog.GetModel(_providerId, request.Model) + ?? throw new ModelProviderException( + $"Model '{_providerId}/{request.Model}' is not registered.", + isTransient: false); + foreach (var part in request.Messages.SelectMany(message => message.Content)) + { + var required = RequiredInput(part); + if (required != GameModelInputCapabilities.None + && !model.Supports(required, GameModelOutputCapabilities.None)) + { + throw new ModelProviderException( + $"Model '{_providerId}/{request.Model}' does not support {required.ToString().ToLowerInvariant()} input.", + isTransient: false); + } + } + + return default; + } + + private static GameModelInputCapabilities RequiredInput(AgentContent part) => part switch + { + ImageAttachmentContent => GameModelInputCapabilities.Image, + BinaryContent { MediaKind: AgentMediaKind.Image } => GameModelInputCapabilities.Image, + BinaryContent { MediaKind: AgentMediaKind.Audio } => GameModelInputCapabilities.Audio, + BinaryContent { MediaKind: AgentMediaKind.Video } => GameModelInputCapabilities.Video, + ResourceContent resource when resource.MediaType.StartsWith( + "image/", + StringComparison.OrdinalIgnoreCase) => GameModelInputCapabilities.Image, + ResourceContent resource when resource.MediaType.StartsWith( + "audio/", + StringComparison.OrdinalIgnoreCase) => GameModelInputCapabilities.Audio, + ResourceContent resource when resource.MediaType.StartsWith( + "video/", + StringComparison.OrdinalIgnoreCase) => GameModelInputCapabilities.Video, + _ => GameModelInputCapabilities.None, + }; } private sealed class Entry diff --git a/src/OpenGameAgent.Models/packages.lock.json b/src/OpenGameAgent.Models/packages.lock.json index 1e82da3..6c7f5f1 100644 --- a/src/OpenGameAgent.Models/packages.lock.json +++ b/src/OpenGameAgent.Models/packages.lock.json @@ -63,9 +63,13 @@ "System.Runtime.CompilerServices.Unsafe": "4.5.3" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } } diff --git a/src/OpenGameAgent.Persistence/AgentMessageCodec.cs b/src/OpenGameAgent.Persistence/AgentMessageCodec.cs index 9c96d3e..ae88b2b 100644 --- a/src/OpenGameAgent.Persistence/AgentMessageCodec.cs +++ b/src/OpenGameAgent.Persistence/AgentMessageCodec.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using System.Text.Json; +using OpenGameAgent.Attachments; using OpenGameAgent.Kernel; namespace OpenGameAgent.Persistence; @@ -86,6 +87,16 @@ public static AgentMessage Decode(MessageDocument document) Redacted = reasoning.Redacted, }, ResourceContent resource => new ContentDocument { Kind = "resource", Text = resource.Name, Reference = resource.Uri, Detail = resource.MediaType }, + ImageAttachmentContent image => new ContentDocument + { + Kind = "image", + Text = image.Attachment.Name, + Reference = image.Attachment.AttachmentId, + Detail = image.Attachment.MediaType, + Bytes = image.Attachment.Bytes, + Width = image.Attachment.Width, + Height = image.Attachment.Height, + }, ToolCallContent call => new ContentDocument { Kind = "tool_call", Text = call.Name, Reference = call.Id, Json = call.ArgumentsJson }, _ => throw new InvalidOperationException("Unsupported agent content type."), }; @@ -99,6 +110,13 @@ public static AgentMessage Decode(MessageDocument document) document.Reference ?? throw new PersistenceException("Persisted resource URI is missing."), document.Detail ?? throw new PersistenceException("Persisted resource media type is missing."), document.Text), + "image" => new ImageAttachmentContent(new GameImageAttachment( + document.Reference ?? throw new PersistenceException("Persisted image attachment ID is missing."), + document.Detail ?? throw new PersistenceException("Persisted image media type is missing."), + document.Bytes, + document.Width, + document.Height, + document.Text)), "tool_call" => new ToolCallContent( document.Reference ?? throw new PersistenceException("Persisted tool call ID is missing."), document.Text ?? throw new PersistenceException("Persisted tool call name is missing."), @@ -149,6 +167,12 @@ internal sealed class ContentDocument public string? Detail { get; set; } public bool Redacted { get; set; } + + public int Bytes { get; set; } + + public int Width { get; set; } + + public int Height { get; set; } } internal sealed class UsageDocument diff --git a/src/OpenGameAgent.Persistence/packages.lock.json b/src/OpenGameAgent.Persistence/packages.lock.json index 08fe12b..a7b42a8 100644 --- a/src/OpenGameAgent.Persistence/packages.lock.json +++ b/src/OpenGameAgent.Persistence/packages.lock.json @@ -71,6 +71,9 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.extensions": { "type": "Project", "dependencies": { @@ -81,6 +84,7 @@ "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/src/OpenGameAgent.Plugins/packages.lock.json b/src/OpenGameAgent.Plugins/packages.lock.json index f1e6d27..b074ab9 100644 --- a/src/OpenGameAgent.Plugins/packages.lock.json +++ b/src/OpenGameAgent.Plugins/packages.lock.json @@ -150,6 +150,9 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.connectors.mcp": { "type": "Project", "dependencies": { @@ -167,6 +170,7 @@ "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/src/OpenGameAgent.Providers.Anthropic/packages.lock.json b/src/OpenGameAgent.Providers.Anthropic/packages.lock.json index 775e1fc..942f674 100644 --- a/src/OpenGameAgent.Providers.Anthropic/packages.lock.json +++ b/src/OpenGameAgent.Providers.Anthropic/packages.lock.json @@ -64,9 +64,13 @@ "System.Runtime.CompilerServices.Unsafe": "4.5.3" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/src/OpenGameAgent.Providers.Bedrock/packages.lock.json b/src/OpenGameAgent.Providers.Bedrock/packages.lock.json index 2ee7d58..1218f5f 100644 --- a/src/OpenGameAgent.Providers.Bedrock/packages.lock.json +++ b/src/OpenGameAgent.Providers.Bedrock/packages.lock.json @@ -84,9 +84,13 @@ "System.Runtime.CompilerServices.Unsafe": "4.5.3" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/src/OpenGameAgent.Providers.Google/packages.lock.json b/src/OpenGameAgent.Providers.Google/packages.lock.json index 9ceaf03..d3aed47 100644 --- a/src/OpenGameAgent.Providers.Google/packages.lock.json +++ b/src/OpenGameAgent.Providers.Google/packages.lock.json @@ -109,9 +109,13 @@ "System.Runtime.CompilerServices.Unsafe": "4.5.3" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/src/OpenGameAgent.Providers.MediaHttp/packages.lock.json b/src/OpenGameAgent.Providers.MediaHttp/packages.lock.json index 1a59cbd..780b8b1 100644 --- a/src/OpenGameAgent.Providers.MediaHttp/packages.lock.json +++ b/src/OpenGameAgent.Providers.MediaHttp/packages.lock.json @@ -71,9 +71,13 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } } diff --git a/src/OpenGameAgent.Providers.MessageGateway/packages.lock.json b/src/OpenGameAgent.Providers.MessageGateway/packages.lock.json index 9268a26..f7dd392 100644 --- a/src/OpenGameAgent.Providers.MessageGateway/packages.lock.json +++ b/src/OpenGameAgent.Providers.MessageGateway/packages.lock.json @@ -63,9 +63,13 @@ "System.Runtime.CompilerServices.Unsafe": "4.5.3" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/src/OpenGameAgent.Providers.Mistral/packages.lock.json b/src/OpenGameAgent.Providers.Mistral/packages.lock.json index 775e1fc..942f674 100644 --- a/src/OpenGameAgent.Providers.Mistral/packages.lock.json +++ b/src/OpenGameAgent.Providers.Mistral/packages.lock.json @@ -64,9 +64,13 @@ "System.Runtime.CompilerServices.Unsafe": "4.5.3" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/src/OpenGameAgent.Providers.OpenAI/packages.lock.json b/src/OpenGameAgent.Providers.OpenAI/packages.lock.json index 775e1fc..942f674 100644 --- a/src/OpenGameAgent.Providers.OpenAI/packages.lock.json +++ b/src/OpenGameAgent.Providers.OpenAI/packages.lock.json @@ -64,9 +64,13 @@ "System.Runtime.CompilerServices.Unsafe": "4.5.3" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/src/OpenGameAgent.Providers.OpenAICompatible/packages.lock.json b/src/OpenGameAgent.Providers.OpenAICompatible/packages.lock.json index 775e1fc..942f674 100644 --- a/src/OpenGameAgent.Providers.OpenAICompatible/packages.lock.json +++ b/src/OpenGameAgent.Providers.OpenAICompatible/packages.lock.json @@ -64,9 +64,13 @@ "System.Runtime.CompilerServices.Unsafe": "4.5.3" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/src/OpenGameAgent.Providers.OpenRouter/packages.lock.json b/src/OpenGameAgent.Providers.OpenRouter/packages.lock.json index 9cb3548..68b9f4f 100644 --- a/src/OpenGameAgent.Providers.OpenRouter/packages.lock.json +++ b/src/OpenGameAgent.Providers.OpenRouter/packages.lock.json @@ -71,9 +71,13 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/src/OpenGameAgent.Providers.Remote/packages.lock.json b/src/OpenGameAgent.Providers.Remote/packages.lock.json index ef5d71f..e03df57 100644 --- a/src/OpenGameAgent.Providers.Remote/packages.lock.json +++ b/src/OpenGameAgent.Providers.Remote/packages.lock.json @@ -64,9 +64,13 @@ "System.Runtime.CompilerServices.Unsafe": "4.5.3" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } } diff --git a/src/OpenGameAgent.Server/OpenGameAgent.Server.csproj b/src/OpenGameAgent.Server/OpenGameAgent.Server.csproj index 9d88596..cb988c9 100644 --- a/src/OpenGameAgent.Server/OpenGameAgent.Server.csproj +++ b/src/OpenGameAgent.Server/OpenGameAgent.Server.csproj @@ -7,6 +7,8 @@ + + diff --git a/src/OpenGameAgent.Server/Program.cs b/src/OpenGameAgent.Server/Program.cs index fd7def7..0af93f6 100644 --- a/src/OpenGameAgent.Server/Program.cs +++ b/src/OpenGameAgent.Server/Program.cs @@ -1,4 +1,6 @@ using OpenGameAgent; +using OpenGameAgent.Attachments; +using OpenGameAgent.Attachments.Local; using OpenGameAgent.Persistence; using OpenGameAgent.Providers.OpenAICompatible; using OpenGameAgent.Server; @@ -6,6 +8,13 @@ var builder = WebApplication.CreateBuilder(args); builder.Services.AddProblemDetails(); builder.Services.AddHttpClient("model"); +builder.Services.AddSingleton(serviceProvider => +{ + var configuration = serviceProvider.GetRequiredService(); + var attachmentDirectory = configuration["OpenGameAgent:AttachmentDirectory"] + ?? Path.Combine(AppContext.BaseDirectory, "data", "attachments"); + return new FileGameImageAttachmentStore(attachmentDirectory); +}); builder.Services.AddSingleton(serviceProvider => { var configuration = serviceProvider.GetRequiredService(); @@ -46,6 +55,7 @@ } runtimeOptions.Instructions = configuration["OpenGameAgent:Instructions"] ?? string.Empty; + runtimeOptions.ImageAttachments = serviceProvider.GetRequiredService(); runtimeOptions.SessionStore = new FileGameSessionStore( configuration["OpenGameAgent:DataDirectory"] ?? Path.Combine(AppContext.BaseDirectory, "data", "sessions")); return new GameAgentRuntime(runtimeOptions); diff --git a/src/OpenGameAgent.Server/ServerAudience.cs b/src/OpenGameAgent.Server/ServerAudience.cs index d343cd9..8892ea3 100644 --- a/src/OpenGameAgent.Server/ServerAudience.cs +++ b/src/OpenGameAgent.Server/ServerAudience.cs @@ -393,7 +393,13 @@ private static bool SanitizeMessage(JsonObject message) var kind = part?["kind"]?.GetValue(); if (kind is not ("reasoning" or "tool_call")) { - safe.Add(part?.DeepClone()); + var visible = part?.DeepClone(); + if (visible is JsonObject visibleObject) + { + visibleObject["signature"] = null; + } + + safe.Add(visible); } } diff --git a/src/OpenGameAgent.Server/ServerAuthorization.cs b/src/OpenGameAgent.Server/ServerAuthorization.cs index f39467b..4dadbae 100644 --- a/src/OpenGameAgent.Server/ServerAuthorization.cs +++ b/src/OpenGameAgent.Server/ServerAuthorization.cs @@ -16,6 +16,7 @@ public enum GameAgentServerOperation StreamActions = 6, SubmitActionReceipt = 7, ReconcileAction = 8, + ReadAttachment = 9, } /// diff --git a/src/OpenGameAgent.Server/ServerEndpoints.cs b/src/OpenGameAgent.Server/ServerEndpoints.cs index 8f848fc..00a326a 100644 --- a/src/OpenGameAgent.Server/ServerEndpoints.cs +++ b/src/OpenGameAgent.Server/ServerEndpoints.cs @@ -56,7 +56,8 @@ public static IApplicationBuilder UseOpenGameAgentApiKey( if (!context.Request.Path.StartsWithSegments("/v1/run") && !context.Request.Path.StartsWithSegments("/v1/control") && !context.Request.Path.StartsWithSegments("/v1/actions") - && !context.Request.Path.StartsWithSegments("/v1/usage")) + && !context.Request.Path.StartsWithSegments("/v1/usage") + && !context.Request.Path.StartsWithSegments("/v1/attachments")) { await next(context); return; @@ -115,13 +116,14 @@ public static IEndpointRouteBuilder MapOpenGameAgent( name = "OpenGameAgent", protocolVersion = "1", transports = new[] { "json", "sse" }, - input = new[] { "text", "json", "resource-reference" }, + input = new[] { "text", "json", "resource-reference", "image" }, routes = new[] { "quick", "agent", "workflow" }, execution = new[] { "in-process", "server" }, control = new[] { "steer", "abort" }, audience = new[] { "internal", "owner", "public", "recipient" }, actions = new[] { "claim", "stream", "receipt", "reconcile" }, usage = new[] { "session-ledger", "by-cause", "itemized-cost" }, + attachments = new[] { "content-addressed-images", "session-authorized-read" }, })); endpoints.MapPost( "/v1/run", @@ -143,10 +145,84 @@ public static IEndpointRouteBuilder MapOpenGameAgent( "/v1/usage", (HttpRequest request, GameAgentRuntime runtime, CancellationToken cancellationToken) => ReadUsageAsync(request, runtime, maximumRequestBodyBytes, cancellationToken)); + endpoints.MapPost( + "/v1/attachments/read", + (HttpRequest request, GameAgentRuntime runtime, CancellationToken cancellationToken) => + ReadAttachmentAsync(request, runtime, maximumRequestBodyBytes, cancellationToken)); MapGameActionExchangeEndpoints(endpoints, maximumRequestBodyBytes); return endpoints; } + private static async Task ReadAttachmentAsync( + HttpRequest httpRequest, + GameAgentRuntime runtime, + int maximumRequestBodyBytes, + CancellationToken cancellationToken) + { + AttachmentReadRequest request; + GameSessionKey key; + try + { + using var requestDocument = await ReadRequestDocumentAsync( + httpRequest, + maximumRequestBodyBytes, + cancellationToken); + request = ParseRequest(requestDocument.RootElement); + key = request.ToKey(); + request.EnsureValid(); + } + catch (RequestBodyTooLargeException exception) + { + return RequestError(StatusCodes.Status413PayloadTooLarge, "request_too_large", exception.Message); + } + catch (UnsupportedRequestContentTypeException exception) + { + return RequestError(StatusCodes.Status415UnsupportedMediaType, "unsupported_media_type", exception.Message); + } + catch (Exception exception) when (exception is ArgumentException or JsonException) + { + return RequestError(StatusCodes.Status400BadRequest, "invalid_request", exception.Message); + } + + var authenticationFailure = await AuthenticatePresentedCredentialAsync( + httpRequest.HttpContext, + request.Credential, + key, + GameAgentServerOperation.ReadAttachment, + cancellationToken); + if (authenticationFailure is not null) + { + return authenticationFailure; + } + + var authorizationFailure = await GetAuthorizationFailureAsync( + httpRequest.HttpContext, + key, + GameAgentServerOperation.ReadAttachment, + cancellationToken); + if (authorizationFailure is not null) + { + return authorizationFailure; + } + + var stored = await runtime.ReadImageAttachmentAsync(key, request.AttachmentId, cancellationToken); + return stored is null + ? Results.NotFound(new { error = "attachment_not_found" }) + : Results.Json(new + { + attachment = new + { + attachmentId = stored.Attachment.AttachmentId, + mediaType = stored.Attachment.MediaType, + bytes = stored.Attachment.Bytes, + width = stored.Attachment.Width, + height = stored.Attachment.Height, + name = stored.Attachment.Name, + }, + data = Convert.ToBase64String(stored.Data.ToArray()), + }); + } + private static async Task ReadUsageAsync( HttpRequest httpRequest, GameAgentRuntime runtime, @@ -828,3 +904,27 @@ public sealed class ControlRequest public string GetPayloadJson() => Payload.ValueKind == JsonValueKind.Undefined ? "{}" : Payload.GetRawText(); } + +public sealed class AttachmentReadRequest +{ + public string? Credential { get; set; } + + public string SessionId { get; set; } = string.Empty; + + public string ActorId { get; set; } = string.Empty; + + public string AttachmentId { get; set; } = string.Empty; + + public GameSessionKey ToKey() => new(SessionId, ActorId); + + public void EnsureValid() + { + _ = ToKey(); + if (string.IsNullOrWhiteSpace(AttachmentId) + || AttachmentId.Length > 256 + || AttachmentId.Any(static character => char.IsControl(character))) + { + throw new ArgumentException("A bounded attachment ID is required.", nameof(AttachmentId)); + } + } +} diff --git a/src/OpenGameAgent.Server/packages.lock.json b/src/OpenGameAgent.Server/packages.lock.json index 4e24e0b..19f3a70 100644 --- a/src/OpenGameAgent.Server/packages.lock.json +++ b/src/OpenGameAgent.Server/packages.lock.json @@ -2,6 +2,30 @@ "version": 1, "dependencies": { "net8.0": { + "SkiaSharp": { + "type": "Transitive", + "resolved": "4.150.1", + "contentHash": "5v3T8X1N62Dp+AkPO70GNBNS/NRBPGMOTiN+Prg33sZAcm/Ug3YOAH+3RTj/jxJV8NGTJs2idGpC2Qdae2mGLQ==", + "dependencies": { + "SkiaSharp.NativeAssets.Win32": "4.150.1", + "SkiaSharp.NativeAssets.macOS": "4.150.1" + } + }, + "SkiaSharp.NativeAssets.Linux.NoDependencies": { + "type": "Transitive", + "resolved": "4.150.1", + "contentHash": "2KVadgDky2xQw7lEMRT/u0ftX5K1u8X7MhCN47Em22Z6VM6JJg1c9cwfGuSPAt1oY6+GjGHH18vkeev+nqT9Kw==" + }, + "SkiaSharp.NativeAssets.macOS": { + "type": "Transitive", + "resolved": "4.150.1", + "contentHash": "r755HVwaHZhyf1clWjrM2/RoOZYCzkQEmE9pu/mVsebPejWu52niNPUwtfyf112qoF0PIk6OndqVOUoITj6TwQ==" + }, + "SkiaSharp.NativeAssets.Win32": { + "type": "Transitive", + "resolved": "4.150.1", + "contentHash": "qrLSL8OonbkMJdSH8heK0Jl39Y2xxZGdd7Ru4cyBXk2ITdx+Fu1sCFLmvCKmnMaYs//mlN/YSpSpOurEXErcQw==" + }, "System.Text.Json": { "type": "Transitive", "resolved": "8.0.6", @@ -14,6 +38,17 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, + "opengameagent.attachments.local": { + "type": "Project", + "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", + "SkiaSharp": "[4.150.1, )", + "SkiaSharp.NativeAssets.Linux.NoDependencies": "[4.150.1, )" + } + }, "opengameagent.extensions": { "type": "Project", "dependencies": { @@ -24,6 +59,7 @@ "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/src/OpenGameAgent/GameAgentRuntime.cs b/src/OpenGameAgent/GameAgentRuntime.cs index 9eb8e47..aeb4311 100644 --- a/src/OpenGameAgent/GameAgentRuntime.cs +++ b/src/OpenGameAgent/GameAgentRuntime.cs @@ -4,6 +4,7 @@ using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using OpenGameAgent.Attachments; using OpenGameAgent.Kernel; namespace OpenGameAgent; @@ -254,6 +255,8 @@ public GameAgentRuntimeOptions(IModelProvider provider, string model) public IGameSessionStore SessionStore { get; set; } = new InMemoryGameSessionStore(); + public IGameImageAttachmentStore? ImageAttachments { get; set; } + public IGameContextProvider? ContextProvider { get; set; } public IGameSkillSource? SkillSource { get; set; } @@ -329,6 +332,7 @@ public sealed class GameAgentRuntime : IDisposable, IAsyncDisposable private readonly string _instructions; private readonly IGameRoutePolicy _routePolicy; private readonly IGameSessionStore _sessionStore; + private readonly IGameImageAttachmentStore? _imageAttachments; private readonly IGameContextProvider? _contextProvider; private readonly IGameSkillSource? _skillSource; private readonly GameToolProvider? _toolProvider; @@ -368,6 +372,7 @@ public GameAgentRuntime(GameAgentRuntimeOptions options) ?? throw new ArgumentException("A route policy is required.", nameof(options)); _sessionStore = options.SessionStore ?? throw new ArgumentException("A session store is required.", nameof(options)); + _imageAttachments = options.ImageAttachments; _contextProvider = options.ContextProvider; _skillSource = options.SkillSource; _toolProvider = options.ToolProvider; @@ -501,6 +506,54 @@ public Task RunAsync(GameInput input, CancellationToken canc : new GameSessionUsageSnapshot(snapshot.Key, snapshot.Revision, snapshot.UsageLedger); } + /// + /// Reads an image only when its durable reference belongs to the requested session actor. + /// Server hosts must authorize the caller before invoking this method. + /// + public async ValueTask ReadImageAttachmentAsync( + GameSessionKey key, + string attachmentId, + CancellationToken cancellationToken = default) + { + key.EnsureValid(nameof(key)); + if (string.IsNullOrWhiteSpace(attachmentId) + || attachmentId.Length > 256 + || attachmentId.Any(static character => char.IsControl(character))) + { + throw new ArgumentException("A bounded attachment ID is required.", nameof(attachmentId)); + } + + if (Volatile.Read(ref _disposed) != 0) + { + throw new ObjectDisposedException(nameof(GameAgentRuntime)); + } + + var store = _imageAttachments + ?? throw new InvalidOperationException("This runtime does not have an image attachment store."); + var snapshot = await _sessionStore.LoadAsync(key, cancellationToken).ConfigureAwait(false); + if (snapshot is null) + { + return null; + } + + if (!snapshot.Key.Equals(key)) + { + throw new InvalidOperationException("The game session store returned a snapshot for a different session key."); + } + + var attachment = snapshot.Messages + .SelectMany(static message => message.Content) + .OfType() + .Select(static content => content.Attachment) + .FirstOrDefault(candidate => string.Equals( + candidate.AttachmentId, + attachmentId, + StringComparison.Ordinal)); + return attachment is null + ? null + : await store.ReadImageAsync(attachment, cancellationToken).ConfigureAwait(false); + } + public Task RunAsync( GameInput input, GameAgentEventHandler? observer, @@ -592,6 +645,7 @@ private async ValueTask RunCoreAsync( GameAgentExtensionRunContext? failureContext = null; try { + input = await PersistInputImagesAsync(input, cancellationToken).ConfigureAwait(false); var key = new GameSessionKey(input.SessionId, input.ActorId); var loaded = await _sessionStore.LoadAsync(key, cancellationToken).ConfigureAwait(false) ?? new GameSessionSnapshot(key, 0); @@ -838,6 +892,10 @@ await _extensions.PublishAsync( IReadOnlyList? checkpointConflictUsageRecords = null; GameSessionUsageLedger? checkpointConflictUsageLedger = null; var recoverySafety = new GameModelRecoverySafety(resumingCheckpoint); + Func wrapProvider = candidate => + candidate is ImageResolvingModelProvider + ? candidate + : new ImageResolvingModelProvider(candidate, ResolveModelImagesAsync); Func? wrapRecoveryProvider = null; if (_transcriptCompactor is not null && contextWindowTokens > 0) { @@ -856,9 +914,13 @@ await _extensions.PublishAsync( usageAccounting.Record, usageAccounting.Record, usageAccounting.ClearAssistantSuppression); - provider = wrapRecoveryProvider(provider); + var wrapImages = wrapProvider; + var wrapRecovery = wrapRecoveryProvider; + wrapProvider = candidate => wrapRecovery(wrapImages(candidate)); } + provider = wrapProvider(provider); + var runHooks = CreateRunHooks( route.Route, input, @@ -868,22 +930,19 @@ await _extensions.PublishAsync( contextWindowTokens, maximumOutputTokens, usageAccounting); - if (wrapRecoveryProvider is not null) + var configuredProviderUpdate = runHooks.PrepareNextTurnAsync; + runHooks.PrepareNextTurnAsync = async (turnContext, token) => { - var configured = runHooks.PrepareNextTurnAsync; - runHooks.PrepareNextTurnAsync = async (turnContext, token) => + var update = configuredProviderUpdate is null + ? null + : await configuredProviderUpdate(turnContext, token).ConfigureAwait(false); + if (update?.Provider is not null) { - var update = configured is null - ? null - : await configured(turnContext, token).ConfigureAwait(false); - if (update?.Provider is not null) - { - update.Provider = wrapRecoveryProvider(update.Provider); - } + update.Provider = wrapProvider(update.Provider); + } - return update; - }; - } + return update; + }; if (_persistToolTurnCheckpoints && route.Route == GameRouteKind.Agent) { @@ -1381,11 +1440,11 @@ private static AgentMessage CreateInputMessage(GameInput input) ["game.timeline_id"] = input.Moment.TimelineId, ["game.tick"] = input.Moment.Tick.ToString(System.Globalization.CultureInfo.InvariantCulture), }; - var content = new List(input.Resources.Count + 1) + var content = new List(input.Content.Count + 1) { new JsonContent(payload), }; - content.AddRange(input.Resources); + content.AddRange(input.Content); return new AgentMessage(AgentRole.User, content, DateTimeOffset.UtcNow, metadata: metadata); } @@ -1427,6 +1486,261 @@ private static bool InputMessageEquals(AgentMessage left, AgentMessage right) => && left.Content.Count == right.Content.Count && left.Content.Zip(right.Content, GameAgentValueComparer.ContentEquals).All(equal => equal); + private async ValueTask PersistInputImagesAsync( + GameInput input, + CancellationToken cancellationToken) + { + if (!input.Content.Any(part => part is BinaryContent { MediaKind: AgentMediaKind.Image })) + { + return input; + } + + var store = _imageAttachments + ?? throw new GameAttachmentException( + "ATTACHMENT_STORE_REQUIRED", + "Image input requires a durable image attachment store."); + var prepared = PrepareImageBatch(input.Content, store.ImageLimits); + foreach (var upload in prepared.Uploads) + { + await store.ValidateImageAsync(upload, cancellationToken).ConfigureAwait(false); + } + + var replacements = new Queue(); + foreach (var upload in prepared.Uploads) + { + var attachment = await store.SaveImageAsync(upload, cancellationToken).ConfigureAwait(false); + replacements.Enqueue(new ImageAttachmentContent(attachment)); + } + + var content = input.Content.Select(part => + part is BinaryContent { MediaKind: AgentMediaKind.Image } + ? (AgentContent)replacements.Dequeue() + : part) + .ToArray(); + return input.WithPersistedContent(content); + } + + private async ValueTask PersistToolImagesAsync( + ToolResult result, + CancellationToken cancellationToken) + { + if (!result.Content.Any(part => part is BinaryContent { MediaKind: AgentMediaKind.Image } + or ImageAttachmentContent)) + { + return result; + } + + var store = _imageAttachments + ?? throw new GameAttachmentException( + "ATTACHMENT_STORE_REQUIRED", + "Image tool results require a durable image attachment store."); + var prepared = PrepareImageBatch(result.Content, store.ImageLimits); + foreach (var upload in prepared.Uploads) + { + await store.ValidateImageAsync(upload, cancellationToken).ConfigureAwait(false); + } + + var replacements = new Queue(); + foreach (var upload in prepared.Uploads) + { + var attachment = await store.SaveImageAsync(upload, cancellationToken).ConfigureAwait(false); + replacements.Enqueue(new ImageAttachmentContent(attachment)); + } + + var content = result.Content.Select(part => + part is BinaryContent { MediaKind: AgentMediaKind.Image } + ? (AgentContent)replacements.Dequeue() + : part) + .ToArray(); + return new ToolResult( + content, + result.IsError, + result.DetailsJson, + result.Terminate, + result.Usage, + result.OutcomeUncertain, + result.AddedToolNames); + } + + private async ValueTask ResolveModelImagesAsync( + ModelRequest request, + CancellationToken cancellationToken) + { + if (!request.Messages.Any(message => message.Content.Any(part => part is ImageAttachmentContent))) + { + return request; + } + + var store = _imageAttachments + ?? throw new GameAttachmentException( + "ATTACHMENT_STORE_REQUIRED", + "Image history requires a durable image attachment store."); + var resolved = new Dictionary(StringComparer.Ordinal); + var messages = new List(request.Messages.Count); + foreach (var message in request.Messages) + { + if (!message.Content.Any(part => part is ImageAttachmentContent)) + { + messages.Add(message); + continue; + } + + var content = new List(message.Content.Count); + foreach (var part in message.Content) + { + if (part is not ImageAttachmentContent image) + { + content.Add(part); + continue; + } + + if (!resolved.TryGetValue(image.Attachment.AttachmentId, out var stored)) + { + stored = await store.ReadImageAsync(image.Attachment, cancellationToken).ConfigureAwait(false); + resolved.Add(image.Attachment.AttachmentId, stored); + } + content.Add(new BinaryContent( + AgentMediaKind.Image, + Convert.ToBase64String(stored.Data.ToArray()), + stored.Attachment.MediaType, + stored.Attachment.Name)); + } + + messages.Add(CloneMessageWithContent(message, content)); + } + + return new ModelRequest( + request.Model, + request.SystemPrompt, + messages, + request.Tools, + request.Parameters, + request.SessionId, + request.RunId, + request.Turn); + } + + private static PreparedImageBatch PrepareImageBatch( + IReadOnlyList content, + GameImageAttachmentLimits limits) + { + var uploads = new List(); + var imageCount = 0; + long imageBytes = 0; + foreach (var part in content) + { + switch (part) + { + case ImageAttachmentContent image: + imageCount++; + imageBytes += image.Attachment.Bytes; + break; + case BinaryContent { MediaKind: AgentMediaKind.Image } binary: + byte[] data; + try + { + data = Convert.FromBase64String(binary.Data); + } + catch (FormatException exception) + { + throw new GameAttachmentException( + "INVALID_IMAGE_ENCODING", + "Inline image data is not valid base64.", + exception); + } + + imageCount++; + imageBytes += data.Length; + uploads.Add(new SaveGameImageAttachment(data, binary.MediaType, binary.Name)); + break; + } + + if (imageCount > limits.MaxImagesPerMessage) + { + throw new GameAttachmentException( + "TOO_MANY_IMAGES", + "The message contains too many images."); + } + + if (imageBytes > limits.MaxMessageImageBytes) + { + throw new GameAttachmentException( + "TOO_MANY_IMAGE_BYTES", + "The message contains too many image bytes."); + } + } + + return new PreparedImageBatch(uploads); + } + + private static AgentMessage CloneMessageWithContent( + AgentMessage message, + IReadOnlyList content) => new( + message.Role, + content, + message.Timestamp, + customRole: message.Role == AgentRole.Custom ? message.CustomRole : null, + toolCallId: message.Role == AgentRole.Tool ? message.ToolCallId : null, + toolName: message.Role == AgentRole.Tool ? message.ToolName : null, + isError: message.Role == AgentRole.Tool && message.IsError, + detailsJson: message.Role == AgentRole.Tool ? message.DetailsJson : null, + metadata: message.Metadata, + model: message.Role == AgentRole.Assistant ? message.Model : null, + stopReason: message.Role == AgentRole.Assistant ? message.StopReason : null, + usage: message.Usage, + errorMessage: message.Role == AgentRole.Assistant ? message.ErrorMessage : null, + provider: message.Role == AgentRole.Assistant ? message.Provider : null, + api: message.Role == AgentRole.Assistant ? message.Api : null, + responseModel: message.Role == AgentRole.Assistant ? message.ResponseModel : null, + responseId: message.Role == AgentRole.Assistant ? message.ResponseId : null, + rawStopReason: message.Role == AgentRole.Assistant ? message.RawStopReason : null, + endTurn: message.Role == AgentRole.Assistant ? message.EndTurn : null, + diagnostics: message.Role == AgentRole.Assistant ? message.Diagnostics : null, + deferred: message.Role == AgentRole.Assistant ? message.Deferred : null, + addedToolNames: message.Role == AgentRole.Tool ? message.AddedToolNames : null); + + private sealed class PreparedImageBatch + { + public PreparedImageBatch(IReadOnlyList uploads) + { + Uploads = uploads; + } + + public IReadOnlyList Uploads { get; } + } + + private sealed class ImageResolvingModelProvider : IModelProvider + { + private readonly IModelProvider _inner; + private readonly Func> _resolve; + + public ImageResolvingModelProvider( + IModelProvider inner, + Func> resolve) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + _resolve = resolve ?? throw new ArgumentNullException(nameof(resolve)); + } + + public async IAsyncEnumerable StreamAsync( + ModelRequest request, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + if (_inner is IModelRequestPreflight preflight) + { + await preflight.ValidateRequestAsync(request, cancellationToken).ConfigureAwait(false); + } + + var resolved = await _resolve(request, cancellationToken).ConfigureAwait(false); + await foreach (var streamEvent in _inner.StreamAsync(resolved, cancellationToken) + .WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + yield return streamEvent; + } + } + } + private static AgentLimits CopyAgentLimits(AgentLimits value) => new() { MaxSystemPromptCharacters = value.MaxSystemPromptCharacters, @@ -1439,6 +1753,11 @@ private static bool InputMessageEquals(AgentMessage left, AgentMessage right) => MaxTextCharactersPerPart = value.MaxTextCharactersPerPart, MaxJsonCharactersPerPart = value.MaxJsonCharactersPerPart, MaxResourceUriCharacters = value.MaxResourceUriCharacters, + MaxBinaryDataCharactersPerPart = value.MaxBinaryDataCharactersPerPart, + MaxImagesPerMessage = value.MaxImagesPerMessage, + MaxImageBytes = value.MaxImageBytes, + MaxImageBytesPerMessage = value.MaxImageBytesPerMessage, + MaxImagePixels = value.MaxImagePixels, MaxToolCallsPerTurn = value.MaxToolCallsPerTurn, MaxTools = value.MaxTools, MaxToolNameCharacters = value.MaxToolNameCharacters, @@ -1483,6 +1802,34 @@ private AgentHooks CreateRunHooks( hooks = _extensions.ComposeHooks(extensionContext, hooks); + var configuredAfterToolCall = hooks.AfterToolCallAsync; + hooks.AfterToolCallAsync = async (context, cancellationToken) => + { + var result = context.Result; + OperationCanceledException? cancellation = null; + if (configuredAfterToolCall is not null) + { + try + { + result = await configuredAfterToolCall(context, cancellationToken).ConfigureAwait(false) + ?? context.Result; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + cancellation = new OperationCanceledException(cancellationToken); + } + } + + using var settlementCancellation = new CancellationTokenSource(_sessionCommitTimeoutMilliseconds); + var persisted = await PersistToolImagesAsync(result, settlementCancellation.Token).ConfigureAwait(false); + if (cancellation is not null) + { + throw cancellation; + } + + return persisted; + }; + if (route == GameRouteKind.Agent && _refreshContextAfterToolTurns) { var configured = hooks.PrepareNextTurnAsync; diff --git a/src/OpenGameAgent/GameAgentValueComparer.cs b/src/OpenGameAgent/GameAgentValueComparer.cs index 9811992..4d2bc25 100644 --- a/src/OpenGameAgent/GameAgentValueComparer.cs +++ b/src/OpenGameAgent/GameAgentValueComparer.cs @@ -31,6 +31,13 @@ public static bool ContentEquals(AgentContent? left, AgentContent? right) string.Equals(first.Uri, second.Uri, StringComparison.Ordinal) && string.Equals(first.MediaType, second.MediaType, StringComparison.Ordinal) && string.Equals(first.Name, second.Name, StringComparison.Ordinal), + (ImageAttachmentContent first, ImageAttachmentContent second) => + string.Equals(first.Attachment.AttachmentId, second.Attachment.AttachmentId, StringComparison.Ordinal) + && string.Equals(first.Attachment.MediaType, second.Attachment.MediaType, StringComparison.Ordinal) + && first.Attachment.Bytes == second.Attachment.Bytes + && first.Attachment.Width == second.Attachment.Width + && first.Attachment.Height == second.Attachment.Height + && string.Equals(first.Attachment.Name, second.Attachment.Name, StringComparison.Ordinal), (ToolCallContent first, ToolCallContent second) => string.Equals(first.Id, second.Id, StringComparison.Ordinal) && string.Equals(first.Name, second.Name, StringComparison.Ordinal) diff --git a/src/OpenGameAgent/GameAgentWire.cs b/src/OpenGameAgent/GameAgentWire.cs index 011e5a4..b827b60 100644 --- a/src/OpenGameAgent/GameAgentWire.cs +++ b/src/OpenGameAgent/GameAgentWire.cs @@ -28,12 +28,7 @@ public static string SerializeInput(GameInput input) ? (JsonElement?)null : ParseElement(input.Moment.CalendarJson), metadata = input.Metadata, - resources = input.Resources.Select(resource => new - { - uri = resource.Uri, - mediaType = resource.MediaType, - name = resource.Name, - }).ToArray(), + content = input.Content.Select(ProjectInputContent).ToArray(), }, JsonOptions); } @@ -276,13 +271,79 @@ private static bool IsDeltaOnlyUpdate(AgentEvent agentEvent) => or ModelStreamEventKind.ReasoningDelta or ModelStreamEventKind.ToolCallDelta; + private static object ProjectInputContent(AgentContent content) => content switch + { + TextContent text => new { kind = "text", text = text.Text }, + JsonContent json => new { kind = "json", data = (object)ParseElement(json.Json) }, + ResourceContent resource => new + { + kind = "resource", + uri = resource.Uri, + mediaType = resource.MediaType, + name = resource.Name, + }, + BinaryContent { MediaKind: AgentMediaKind.Image } image => new + { + kind = "image", + data = image.Data, + mediaType = image.MediaType, + name = image.Name, + }, + ImageAttachmentContent => throw new ArgumentException( + "Durable image references cannot be submitted on the input wire; send the image bytes instead.", + nameof(content)), + _ => throw new ArgumentException($"Unsupported game input content type '{content.GetType().FullName}'.", nameof(content)), + }; + private static object ProjectContent(AgentContent content) => content switch { - TextContent text => new ContentDocument("text", text.Text, null, null, null), - ReasoningContent reasoning => new ContentDocument("reasoning", reasoning.Text, null, null, reasoning.Signature), - JsonContent json => new ContentDocument("json", null, ParseElement(json.Json), null, null), - ResourceContent resource => new ContentDocument("resource", resource.Name, null, resource.Uri, resource.MediaType), - ToolCallContent call => new ContentDocument("tool_call", call.Name, ParseElement(call.ArgumentsJson), call.Id, null), + TextContent text => new + { + kind = "text", + text = text.Text, + signature = text.Signature, + phase = text.Phase?.ToString(), + }, + ReasoningContent reasoning => new + { + kind = "reasoning", + text = reasoning.Text, + signature = reasoning.Signature, + redacted = reasoning.Redacted, + }, + JsonContent json => new { kind = "json", data = (object)ParseElement(json.Json) }, + ResourceContent resource => new + { + kind = "resource", + uri = resource.Uri, + mediaType = resource.MediaType, + name = resource.Name, + }, + ImageAttachmentContent image => new + { + kind = "image", + attachment = new + { + attachmentId = image.Attachment.AttachmentId, + mediaType = image.Attachment.MediaType, + bytes = image.Attachment.Bytes, + width = image.Attachment.Width, + height = image.Attachment.Height, + name = image.Attachment.Name, + }, + }, + BinaryContent => throw new ArgumentException( + "Inline binary content cannot be projected to a public event; persist it as an attachment first.", + nameof(content)), + ToolCallContent call => new + { + kind = "tool_call", + id = call.Id, + name = call.Name, + arguments = (object)ParseElement(call.ArgumentsJson), + thoughtSignature = call.ThoughtSignature, + toolNamespace = call.Namespace, + }, _ => throw new ArgumentException($"Unsupported agent content type '{content.GetType().FullName}'.", nameof(content)), }; @@ -314,7 +375,7 @@ private sealed class InputDocument public Dictionary? Metadata { get; set; } - public List? Resources { get; set; } + public List? Content { get; set; } public GameInput ToInput() => new( SessionId, @@ -329,41 +390,54 @@ private sealed class InputDocument : null), InputId, Metadata, - (Resources ?? new List()) - .Select(resource => resource.ToResource()) + (Content ?? new List()) + .Select(part => part.ToContent()) .ToArray()); } - private sealed class InputResourceDocument + private sealed class InputContentDocument { + public string Kind { get; set; } = string.Empty; + + public string? Text { get; set; } + + public JsonElement Data { get; set; } + public string Uri { get; set; } = string.Empty; public string MediaType { get; set; } = string.Empty; public string? Name { get; set; } - public ResourceContent ToResource() => new(Uri, MediaType, Name); - } - - private sealed class ContentDocument - { - public ContentDocument(string kind, string? text, JsonElement? data, string? reference, string? detail) + public AgentContent ToContent() { - Kind = kind; - Text = text; - Data = data; - Reference = reference; - Detail = detail; + switch (Kind) + { + case "text": + return new TextContent(Text ?? throw new ArgumentException("Text input content requires text.")); + case "json": + if (Data.ValueKind == JsonValueKind.Undefined) + { + throw new ArgumentException("JSON input content requires data."); + } + + return new JsonContent(Data.GetRawText()); + case "resource": + return new ResourceContent(Uri, MediaType, Name); + case "image": + if (Data.ValueKind != JsonValueKind.String) + { + throw new ArgumentException("Image input content requires base64 string data."); + } + + return new BinaryContent( + AgentMediaKind.Image, + Data.GetString() ?? string.Empty, + MediaType, + Name); + default: + throw new ArgumentException($"Unsupported game input content kind '{Kind}'."); + } } - - public string Kind { get; } - - public string? Text { get; } - - public JsonElement? Data { get; } - - public string? Reference { get; } - - public string? Detail { get; } } } diff --git a/src/OpenGameAgent/GameData.cs b/src/OpenGameAgent/GameData.cs index f015d63..6433864 100644 --- a/src/OpenGameAgent/GameData.cs +++ b/src/OpenGameAgent/GameData.cs @@ -102,7 +102,21 @@ public GameInput( GameMoment moment, string? inputId = null, IReadOnlyDictionary? metadata = null, - IReadOnlyList? resources = null) + IReadOnlyList? content = null) + : this(sessionId, actorId, type, payloadJson, moment, inputId, metadata, content, allowStoredImages: false) + { + } + + private GameInput( + string sessionId, + string actorId, + string type, + string payloadJson, + GameMoment moment, + string? inputId, + IReadOnlyDictionary? metadata, + IReadOnlyList? content, + bool allowStoredImages) { SessionId = GameJson.RequireId(sessionId, nameof(sessionId)); ActorId = GameJson.RequireId(actorId, nameof(actorId)); @@ -122,15 +136,38 @@ public GameInput( } Metadata = new ReadOnlyDictionary(copiedMetadata); - var copiedResources = (resources ?? Array.Empty()).ToArray(); - if (copiedResources.Any(resource => resource is null)) + var copiedContent = (content ?? Array.Empty()).ToArray(); + if (copiedContent.Any(part => part is null)) + { + throw new ArgumentException("Input content cannot contain null values.", nameof(content)); + } + + if (copiedContent.Any(part => + part is not TextContent + and not JsonContent + and not ResourceContent + and not BinaryContent { MediaKind: AgentMediaKind.Image } + && !(allowStoredImages && part is ImageAttachmentContent))) { - throw new ArgumentException("Input resources cannot contain null values.", nameof(resources)); + throw new ArgumentException( + "Game input content supports text, JSON, resources, and unsaved inline images only.", + nameof(content)); } - Resources = Array.AsReadOnly(copiedResources); + Content = Array.AsReadOnly(copiedContent); } + internal GameInput WithPersistedContent(IReadOnlyList content) => new( + SessionId, + ActorId, + Type, + PayloadJson, + Moment, + InputId, + Metadata, + content, + allowStoredImages: true); + public string InputId { get; } public string SessionId { get; } @@ -145,7 +182,7 @@ public GameInput( public IReadOnlyDictionary Metadata { get; } - public IReadOnlyList Resources { get; } + public IReadOnlyList Content { get; } } public sealed class GameContextSlice @@ -179,7 +216,7 @@ public sealed class GameRuntimeLimits public int MaxMetadataEntries { get; set; } = 64; - public int MaxInputResources { get; set; } = 16; + public int MaxInputContentParts { get; set; } = 32; public int MaxMetadataKeyCharacters { get; set; } = 256; @@ -221,7 +258,7 @@ internal GameRuntimeLimits CopyAndValidate() RequireRange(copy.MaxContextSlices, 0, 100_000, nameof(MaxContextSlices)); RequireRange(copy.MaxContextJsonCharacters, 2, 100_000_000, nameof(MaxContextJsonCharacters)); RequireRange(copy.MaxMetadataEntries, 0, 100_000, nameof(MaxMetadataEntries)); - RequireRange(copy.MaxInputResources, 0, 10_000, nameof(MaxInputResources)); + RequireRange(copy.MaxInputContentParts, 0, 10_000, nameof(MaxInputContentParts)); RequireRange(copy.MaxMetadataKeyCharacters, 1, 100_000, nameof(MaxMetadataKeyCharacters)); RequireRange(copy.MaxMetadataValueCharacters, 0, 100_000_000, nameof(MaxMetadataValueCharacters)); RequireRange(copy.MaxIdentifierCharacters, 1, 16_384, nameof(MaxIdentifierCharacters)); @@ -258,9 +295,9 @@ internal void Validate(GameInput input) throw new GameRuntimeLimitException(nameof(MaxMetadataEntries), "The input has too many metadata entries."); } - if (input.Resources.Count > MaxInputResources) + if (input.Content.Count > MaxInputContentParts) { - throw new GameRuntimeLimitException(nameof(MaxInputResources), "The input has too many attached resources."); + throw new GameRuntimeLimitException(nameof(MaxInputContentParts), "The input has too many content parts."); } foreach (var value in new[] { input.InputId, input.SessionId, input.ActorId, input.Type, input.Moment.TimelineId }) diff --git a/src/OpenGameAgent/ModelProviders.cs b/src/OpenGameAgent/ModelProviders.cs index d3b4b44..2a11789 100644 --- a/src/OpenGameAgent/ModelProviders.cs +++ b/src/OpenGameAgent/ModelProviders.cs @@ -8,7 +8,7 @@ namespace OpenGameAgent; -public sealed class RetryingModelProvider : IModelProvider +public sealed class RetryingModelProvider : IModelProvider, IModelRequestPreflight { private readonly IModelProvider _inner; private readonly int _maximumAttempts; @@ -40,6 +40,11 @@ public RetryingModelProvider( } } + public ValueTask ValidateRequestAsync(ModelRequest request, CancellationToken cancellationToken) => + _inner is IModelRequestPreflight preflight + ? preflight.ValidateRequestAsync(request, cancellationToken) + : default; + public async IAsyncEnumerable StreamAsync( ModelRequest request, [EnumeratorCancellation] CancellationToken cancellationToken) @@ -158,7 +163,7 @@ public async IAsyncEnumerable StreamAsync( } } -public sealed class FallbackModelProvider : IModelProvider +public sealed class FallbackModelProvider : IModelProvider, IModelRequestPreflight { private readonly IReadOnlyList _providers; private readonly Func _canFallback; @@ -183,6 +188,17 @@ public FallbackModelProvider( exception is not ModelProviderException providerFailure || providerFailure.IsTransient); } + public async ValueTask ValidateRequestAsync(ModelRequest request, CancellationToken cancellationToken) + { + foreach (var provider in _providers) + { + if (provider is IModelRequestPreflight preflight) + { + await preflight.ValidateRequestAsync(request, cancellationToken).ConfigureAwait(false); + } + } + } + public async IAsyncEnumerable StreamAsync( ModelRequest request, [EnumeratorCancellation] CancellationToken cancellationToken) diff --git a/src/OpenGameAgent/Transcripts.cs b/src/OpenGameAgent/Transcripts.cs index 79744d7..01b15ff 100644 --- a/src/OpenGameAgent/Transcripts.cs +++ b/src/OpenGameAgent/Transcripts.cs @@ -176,6 +176,21 @@ public static long EstimateMessages(IReadOnlyList messages) + (resource.Name?.Length ?? 0)); tokens = checked(tokens + ResourceTokenEstimate); break; + case ImageAttachmentContent image: + characters = checked(characters + + image.Attachment.AttachmentId.Length + + image.Attachment.MediaType.Length + + (image.Attachment.Name?.Length ?? 0)); + tokens = checked(tokens + EstimateImageTokens( + image.Attachment.Width, + image.Attachment.Height)); + break; + case BinaryContent binary: + characters = checked(characters + + binary.MediaType.Length + + (binary.Name?.Length ?? 0)); + tokens = checked(tokens + ResourceTokenEstimate); + break; default: throw new InvalidOperationException( $"Unsupported agent content type '{content.GetType().FullName}'."); @@ -190,6 +205,13 @@ public static long EstimateMessages(IReadOnlyList messages) private static long DivideRoundUp(long value, long divisor) => checked((value + divisor - 1) / divisor); + + private static long EstimateImageTokens(int width, int height) + { + var horizontalTiles = Math.Max(1L, DivideRoundUp(width, 512)); + var verticalTiles = Math.Max(1L, DivideRoundUp(height, 512)); + return checked(85L + (170L * horizontalTiles * verticalTiles)); + } } internal sealed class GameModelRecoverySafety @@ -240,7 +262,7 @@ public GameModelRecoveryCompaction( public GameTranscriptCompactionResult Compaction { get; } } -internal sealed class ContextOverflowRecoveryModelProvider : IModelProvider +internal sealed class ContextOverflowRecoveryModelProvider : IModelProvider, IModelRequestPreflight { private readonly IModelProvider _inner; private readonly GameModelRecoverySafety _safety; @@ -275,6 +297,11 @@ public ContextOverflowRecoveryModelProvider( ?? throw new ArgumentNullException(nameof(clearAssistantSuppression)); } + public ValueTask ValidateRequestAsync(ModelRequest request, CancellationToken cancellationToken) => + _inner is IModelRequestPreflight preflight + ? preflight.ValidateRequestAsync(request, cancellationToken) + : default; + public async IAsyncEnumerable StreamAsync( ModelRequest request, [EnumeratorCancellation] CancellationToken cancellationToken) diff --git a/src/OpenGameAgent/packages.lock.json b/src/OpenGameAgent/packages.lock.json index ef5d71f..e03df57 100644 --- a/src/OpenGameAgent/packages.lock.json +++ b/src/OpenGameAgent/packages.lock.json @@ -64,9 +64,13 @@ "System.Runtime.CompilerServices.Unsafe": "4.5.3" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } } diff --git a/tests/OpenGameAgent.Attachments.Tests/ImageAttachmentTests.cs b/tests/OpenGameAgent.Attachments.Tests/ImageAttachmentTests.cs new file mode 100644 index 0000000..9e1c16e --- /dev/null +++ b/tests/OpenGameAgent.Attachments.Tests/ImageAttachmentTests.cs @@ -0,0 +1,216 @@ +using OpenGameAgent.Attachments; +using OpenGameAgent.Attachments.Local; +using SkiaSharp; +using Xunit; + +namespace OpenGameAgent.Attachments.Tests; + +public sealed class ImageAttachmentTests : IDisposable +{ + private readonly string _root = Path.Combine(Path.GetTempPath(), "oga-attachments-" + Guid.NewGuid().ToString("N")); + + [Fact] + public async Task SavesReadsDeduplicatesAndSanitizesNames() + { + var store = new FileGameImageAttachmentStore(_root); + var png = CreateImage(SKEncodedImageFormat.Png, 3, 2); + var token = TestContext.Current.CancellationToken; + var first = await store.SaveImageAsync(new SaveGameImageAttachment( + png, + GameImageMediaTypes.Png, + "C:\\private\\capture.png"), token); + var second = await store.SaveImageAsync(new SaveGameImageAttachment( + png, + GameImageMediaTypes.Png), token); + + Assert.Equal(first.AttachmentId, second.AttachmentId); + Assert.Matches("^sha256:[0-9a-f]{64}$", first.AttachmentId); + Assert.Equal("capture.png", first.Name); + Assert.Equal(3, first.Width); + Assert.Equal(2, first.Height); + Assert.Equal(png.Length, first.Bytes); + var stored = await store.ReadImageAsync(first, token); + Assert.Equal(png, stored.Data.ToArray()); + Assert.Same(first, stored.Attachment); + } + + [Theory] + [InlineData(SKEncodedImageFormat.Png, GameImageMediaTypes.Png)] + [InlineData(SKEncodedImageFormat.Jpeg, GameImageMediaTypes.Jpeg)] + [InlineData(SKEncodedImageFormat.Webp, GameImageMediaTypes.WebP)] + public async Task FullyDecodesSupportedEncodedFormats(SKEncodedImageFormat format, string mediaType) + { + var store = new FileGameImageAttachmentStore(_root); + var data = CreateImage(format, 4, 3); + var token = TestContext.Current.CancellationToken; + var attachment = await store.SaveImageAsync(new SaveGameImageAttachment(data, mediaType), token); + + Assert.Equal(mediaType, attachment.MediaType); + Assert.Equal(4, attachment.Width); + Assert.Equal(3, attachment.Height); + Assert.Equal(data, (await store.ReadImageAsync(attachment, token)).Data.ToArray()); + } + + [Fact] + public async Task FullyDecodesGif() + { + var store = new FileGameImageAttachmentStore(_root); + var gif = Convert.FromBase64String("R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw=="); + var token = TestContext.Current.CancellationToken; + var attachment = await store.SaveImageAsync(new SaveGameImageAttachment(gif, GameImageMediaTypes.Gif), token); + + Assert.Equal(1, attachment.Width); + Assert.Equal(1, attachment.Height); + Assert.Equal(gif, (await store.ReadImageAsync(attachment, token)).Data.ToArray()); + } + + [Fact] + public async Task RejectsMalformedMismatchedOversizedAndPixelBombImages() + { + var png = CreateImage(SKEncodedImageFormat.Png, 3, 3); + var store = new FileGameImageAttachmentStore( + _root, + new GameImageAttachmentLimits( + maxImageBytes: png.Length, + maxImagesPerMessage: 1, + maxMessageImageBytes: png.Length, + maxImagePixels: 4)); + var token = TestContext.Current.CancellationToken; + + await AssertCodeAsync("INVALID_IMAGE", () => store.SaveImageAsync( + new SaveGameImageAttachment(new byte[] { 1, 2, 3 }, GameImageMediaTypes.Png), token).AsTask()); + await AssertCodeAsync("IMAGE_TYPE_MISMATCH", () => store.SaveImageAsync( + new SaveGameImageAttachment(png, GameImageMediaTypes.Jpeg), token).AsTask()); + await AssertCodeAsync("IMAGE_TOO_LARGE", () => new FileGameImageAttachmentStore( + _root, + new GameImageAttachmentLimits( + maxImageBytes: png.Length - 1, + maxImagesPerMessage: 1, + maxMessageImageBytes: png.Length - 1, + maxImagePixels: 100)).SaveImageAsync( + new SaveGameImageAttachment(png, GameImageMediaTypes.Png), token).AsTask()); + await AssertCodeAsync("IMAGE_TOO_MANY_PIXELS", () => store.SaveImageAsync( + new SaveGameImageAttachment(png, GameImageMediaTypes.Png), token).AsTask()); + } + + [Fact] + public async Task ValidationDoesNotCreateStorage() + { + var store = new FileGameImageAttachmentStore(_root); + await store.ValidateImageAsync(new SaveGameImageAttachment( + CreateImage(SKEncodedImageFormat.Png, 1, 1), + GameImageMediaTypes.Png), TestContext.Current.CancellationToken); + + Assert.False(Directory.Exists(_root)); + } + + [Fact] + public async Task ReadsFailClosedForMissingCorruptAndMismatchedReferences() + { + var store = new FileGameImageAttachmentStore(_root); + var data = CreateImage(SKEncodedImageFormat.Png, 2, 2); + var token = TestContext.Current.CancellationToken; + var reference = await store.SaveImageAsync(new SaveGameImageAttachment(data, GameImageMediaTypes.Png), token); + var hash = reference.AttachmentId.Substring("sha256:".Length); + var path = Path.Combine(_root, "objects", hash.Substring(0, 2), hash); + await File.WriteAllBytesAsync(path, new byte[] { 1, 2, 3 }, token); + + await AssertCodeAsync("ATTACHMENT_CORRUPT", () => store.ReadImageAsync(reference, token).AsTask()); + await AssertCodeAsync("INVALID_ATTACHMENT_REF", () => store.ReadImageAsync(new GameImageAttachment( + "opaque-but-not-content-addressed", + GameImageMediaTypes.Png, + 1, + 1, + 1), token).AsTask()); + var missing = new GameImageAttachment( + "sha256:" + new string('a', 64), + GameImageMediaTypes.Png, + 1, + 1, + 1); + await AssertCodeAsync("ATTACHMENT_NOT_FOUND", () => store.ReadImageAsync(missing, token).AsTask()); + var oversized = new GameImageAttachment( + "sha256:" + new string('b', 64), + GameImageMediaTypes.Png, + store.ImageLimits.MaxImageBytes + 1, + 1, + 1); + await AssertCodeAsync("INVALID_ATTACHMENT_REF", () => store.ReadImageAsync(oversized, token).AsTask()); + } + + [Fact] + public async Task ConcurrentEqualWritesPublishOneVerifiedObject() + { + var store = new FileGameImageAttachmentStore(_root); + var data = CreateImage(SKEncodedImageFormat.Png, 8, 8); + var token = TestContext.Current.CancellationToken; + var tasks = Enumerable.Range(0, 16) + .Select(_ => store.SaveImageAsync(new SaveGameImageAttachment(data, GameImageMediaTypes.Png), token).AsTask()) + .ToArray(); + + var results = await Task.WhenAll(tasks); + Assert.Single(results.Select(value => value.AttachmentId).Distinct(StringComparer.Ordinal)); + Assert.Equal(data, (await store.ReadImageAsync(results[0], token)).Data.ToArray()); + } + + [Fact] + public async Task CancellationIsPreserved() + { + var store = new FileGameImageAttachmentStore(_root); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync(() => store.ValidateImageAsync( + new SaveGameImageAttachment(CreateImage(SKEncodedImageFormat.Png, 1, 1), GameImageMediaTypes.Png), + cancellation.Token).AsTask()); + } + + [Fact] + public void PublicContractsRejectUnsupportedFormatsAndUnsafeSourceNames() + { + Assert.Throws(() => new GameImageAttachmentLimits( + mediaTypes: new[] { "image/svg+xml" })); + Assert.Throws(() => new SaveGameImageAttachment( + new byte[] { 1 }, + GameImageMediaTypes.Png, + "bad\nname.png")); + } + + [Fact] + public void BytePayloadsAreDefensivelyCopiedAndExposedReadOnly() + { + var source = new byte[] { 1, 2, 3 }; + var pending = new SaveGameImageAttachment(source, GameImageMediaTypes.Png); + var stored = new StoredGameImageAttachment( + new GameImageAttachment("sha256:" + new string('c', 64), GameImageMediaTypes.Png, 3, 1, 1), + source); + + source[0] = 9; + + Assert.Equal(new byte[] { 1, 2, 3 }, pending.Data.ToArray()); + Assert.Equal(new byte[] { 1, 2, 3 }, stored.Data.ToArray()); + } + + private static byte[] CreateImage(SKEncodedImageFormat format, int width, int height) + { + using var bitmap = new SKBitmap(width, height); + bitmap.Erase(SKColors.CornflowerBlue); + using var image = SKImage.FromBitmap(bitmap); + using var encoded = image.Encode(format, 90); + return encoded.ToArray(); + } + + private static async Task AssertCodeAsync(string code, Func action) + { + var exception = await Assert.ThrowsAsync(action); + Assert.Equal(code, exception.Code); + } + + public void Dispose() + { + if (Directory.Exists(_root)) + { + Directory.Delete(_root, recursive: true); + } + } +} diff --git a/tests/OpenGameAgent.Attachments.Tests/OpenGameAgent.Attachments.Tests.csproj b/tests/OpenGameAgent.Attachments.Tests/OpenGameAgent.Attachments.Tests.csproj new file mode 100644 index 0000000..2b6c75f --- /dev/null +++ b/tests/OpenGameAgent.Attachments.Tests/OpenGameAgent.Attachments.Tests.csproj @@ -0,0 +1,19 @@ + + + Exe + net8.0 + false + true + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + diff --git a/tests/OpenGameAgent.Attachments.Tests/packages.lock.json b/tests/OpenGameAgent.Attachments.Tests/packages.lock.json new file mode 100644 index 0000000..4ae0259 --- /dev/null +++ b/tests/OpenGameAgent.Attachments.Tests/packages.lock.json @@ -0,0 +1,237 @@ +{ + "version": 1, + "dependencies": { + "net8.0": { + "Microsoft.NET.Test.Sdk": { + "type": "Direct", + "requested": "[18.8.1, )", + "resolved": "18.8.1", + "contentHash": "dknJL3/9Y3t4XuCBqnc0PevPxgLsUMmVhjwup/b1HNovA8zWcj3XsfIf7c6p05363DWcqL7X/YhDL9B+Zymv1w==", + "dependencies": { + "Microsoft.CodeCoverage": "18.8.1", + "Microsoft.TestPlatform.TestHost": "18.8.1" + } + }, + "System.Diagnostics.DiagnosticSource": { + "type": "Direct", + "requested": "[10.0.10, )", + "resolved": "10.0.10", + "contentHash": "cjtKi6ERMYWp6b9UTVPcwDT29PjKDtlM3W9OwnWL5abRsI8ku42Q2wqZoLIIXJnT/XF2s2CjuK8Nl4a3mmTxQQ==" + }, + "System.Security.AccessControl": { + "type": "Direct", + "requested": "[6.0.1, )", + "resolved": "6.0.1", + "contentHash": "IQ4NXP/B3Ayzvw0rDQzVTYsCKyy0Jp9KI6aYcK7UnGVlR9+Awz++TIPCQtPYfLJfOpm8ajowMR09V7quD3sEHw==" + }, + "xunit.runner.visualstudio": { + "type": "Direct", + "requested": "[3.1.5, )", + "resolved": "3.1.5", + "contentHash": "tKi7dSTwP4m5m9eXPM2Ime4Kn7xNf4x4zT9sdLO/G4hZVnQCRiMTWoSZqI/pYTVeI27oPPqHBKYI/DjJ9GsYgA==" + }, + "xunit.v3": { + "type": "Direct", + "requested": "[3.2.2, )", + "resolved": "3.2.2", + "contentHash": "L+4/4y0Uqcg8/d6hfnxhnwh4j9FaeULvefTwrk30rr1o4n/vdPfyUQ8k0yzH8VJx7bmFEkDdcRfbtbjEHlaYcA==", + "dependencies": { + "xunit.v3.mtp-v1": "[3.2.2]" + } + }, + "Microsoft.ApplicationInsights": { + "type": "Transitive", + "resolved": "2.23.0", + "contentHash": "nWArUZTdU7iqZLycLKWe0TDms48KKGE6pONH2terYNa8REXiqixrMOkf1sk5DHGMaUTqONU2YkS4SAXBhLStgw==", + "dependencies": { + "System.Diagnostics.DiagnosticSource": "5.0.0" + } + }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "UcSjPsst+DfAdJGVDsu346FX0ci0ah+lw3WRtn18NUwEqRt70HaOQ7lI72vy3+1LxtqI3T5GWwV39rQSrCzAeg==" + }, + "Microsoft.CodeCoverage": { + "type": "Transitive", + "resolved": "18.8.1", + "contentHash": "Eclse/ZZjr4lmWzZFNN9h/OluhKL+SK/QbUyKUewgX139aGeyMEO/DkMPwuFs2MixvanTnz6891rF8UHDg+W4Q==" + }, + "Microsoft.Testing.Extensions.Telemetry": { + "type": "Transitive", + "resolved": "1.9.1", + "contentHash": "No5AudZMmSb+uNXjlgL2y3/stHD2IT4uxqc5yHwkE+/nNux9jbKcaJMvcp9SwgP4DVD8L9/P3OUz8mmmcvEIdQ==", + "dependencies": { + "Microsoft.ApplicationInsights": "2.23.0", + "Microsoft.Testing.Platform": "1.9.1" + } + }, + "Microsoft.Testing.Extensions.TrxReport.Abstractions": { + "type": "Transitive", + "resolved": "1.9.1", + "contentHash": "AL46Xe1WBi85Ntd4mNPvat5ZSsZ2uejiVqoKCypr8J3wK0elA5xJ3AN4G/Q4GIwzUFnggZoH/DBjnr9J18IO/g==", + "dependencies": { + "Microsoft.Testing.Platform": "1.9.1" + } + }, + "Microsoft.Testing.Platform": { + "type": "Transitive", + "resolved": "1.9.1", + "contentHash": "QafNtNSmEI0zazdebnsIkDKmFtTSpmx/5PLOjURWwozcPb3tvRxzosQSL8xwYNM1iPhhKiBksXZyRSE2COisrA==" + }, + "Microsoft.Testing.Platform.MSBuild": { + "type": "Transitive", + "resolved": "1.9.1", + "contentHash": "oTUtyR4X/s9ytuiNA29FGsNCCH0rNmY5Wdm14NCKLjTM1cT9edVSlA+rGS/mVmusPqcP0l/x9qOnMXg16v87RQ==", + "dependencies": { + "Microsoft.Testing.Platform": "1.9.1" + } + }, + "Microsoft.TestPlatform.ObjectModel": { + "type": "Transitive", + "resolved": "18.8.1", + "contentHash": "qLbktNB1+b1XZLNJBTzaWVVJAd6PEzD7cgD406geMb6PcFZhp3EDNa1tctWx1+mtMU6MP/6ozVvFPC9vs2a9rw==", + "dependencies": { + "System.Reflection.Metadata": "8.0.0" + } + }, + "Microsoft.TestPlatform.TestHost": { + "type": "Transitive", + "resolved": "18.8.1", + "contentHash": "FaQHPDTUOcE+SFTjssNPfrub2lT9Zyon4J2W/KLHt/efLJACb1TCeWXyOgh0D/4Q1e4n+S3E6mOKud+9nLZlEA==", + "dependencies": { + "Microsoft.TestPlatform.ObjectModel": "18.8.1" + } + }, + "Microsoft.Win32.Registry": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "dDoKi0PnDz31yAyETfRntsLArTlVAVzUzCIvvEDsDsucrl33Dl8pIJG06ePTJTI3tGpeyHS9Cq7Foc/s4EeKcg==", + "dependencies": { + "System.Security.AccessControl": "5.0.0", + "System.Security.Principal.Windows": "5.0.0" + } + }, + "SkiaSharp": { + "type": "Transitive", + "resolved": "4.150.1", + "contentHash": "5v3T8X1N62Dp+AkPO70GNBNS/NRBPGMOTiN+Prg33sZAcm/Ug3YOAH+3RTj/jxJV8NGTJs2idGpC2Qdae2mGLQ==", + "dependencies": { + "SkiaSharp.NativeAssets.Win32": "4.150.1", + "SkiaSharp.NativeAssets.macOS": "4.150.1" + } + }, + "SkiaSharp.NativeAssets.Linux.NoDependencies": { + "type": "Transitive", + "resolved": "4.150.1", + "contentHash": "2KVadgDky2xQw7lEMRT/u0ftX5K1u8X7MhCN47Em22Z6VM6JJg1c9cwfGuSPAt1oY6+GjGHH18vkeev+nqT9Kw==" + }, + "SkiaSharp.NativeAssets.macOS": { + "type": "Transitive", + "resolved": "4.150.1", + "contentHash": "r755HVwaHZhyf1clWjrM2/RoOZYCzkQEmE9pu/mVsebPejWu52niNPUwtfyf112qoF0PIk6OndqVOUoITj6TwQ==" + }, + "SkiaSharp.NativeAssets.Win32": { + "type": "Transitive", + "resolved": "4.150.1", + "contentHash": "qrLSL8OonbkMJdSH8heK0Jl39Y2xxZGdd7Ru4cyBXk2ITdx+Fu1sCFLmvCKmnMaYs//mlN/YSpSpOurEXErcQw==" + }, + "System.Collections.Immutable": { + "type": "Transitive", + "resolved": "8.0.0", + "contentHash": "AurL6Y5BA1WotzlEvVaIDpqzpIPvYnnldxru8oXJU2yFxFUy3+pNXjXd1ymO+RA0rq0+590Q8gaz2l3Sr7fmqg==" + }, + "System.Reflection.Metadata": { + "type": "Transitive", + "resolved": "8.0.0", + "contentHash": "ptvgrFh7PvWI8bcVqG5rsA/weWM09EnthFHR5SCnS6IN+P4mj6rE1lBDC4U8HL9/57htKAqy4KQ3bBj84cfYyQ==", + "dependencies": { + "System.Collections.Immutable": "8.0.0" + } + }, + "System.Security.Principal.Windows": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "t0MGLukB5WAVU9bO3MGzvlGnyJPgUlcwerXn1kzBRjwLKixT96XV0Uza41W49gVd8zEMFu9vQEFlv0IOrytICA==" + }, + "xunit.analyzers": { + "type": "Transitive", + "resolved": "1.27.0", + "contentHash": "y/pxIQaLvk/kxAoDkZW9GnHLCEqzwl5TW0vtX3pweyQpjizB9y3DXhb9pkw2dGeUqhLjsxvvJM1k89JowU6z3g==" + }, + "xunit.v3.assert": { + "type": "Transitive", + "resolved": "3.2.2", + "contentHash": "BPciBghgEEaJN/JG00QfCYDfEfnLgQhfnYEy+j1izoeHVNYd5+3Wm8GJ6JgYysOhpBPYGE+sbf75JtrRc7jrdA==" + }, + "xunit.v3.common": { + "type": "Transitive", + "resolved": "3.2.2", + "contentHash": "Hj775PEH6GTbbg0wfKRvG2hNspDCvTH9irXhH4qIWgdrOSV1sQlqPie+DOvFeigsFg2fxSM3ZAaaCDQs+KreFA==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "6.0.0" + } + }, + "xunit.v3.core.mtp-v1": { + "type": "Transitive", + "resolved": "3.2.2", + "contentHash": "Ga5aA2Ca9ktz+5k3g5ukzwfexwoqwDUpV6z7atSEUvqtd6JuybU1XopHqg1oFd78QdTfZgZE9h5sHpO4qYIi5w==", + "dependencies": { + "Microsoft.Testing.Extensions.Telemetry": "1.9.1", + "Microsoft.Testing.Extensions.TrxReport.Abstractions": "1.9.1", + "Microsoft.Testing.Platform": "1.9.1", + "Microsoft.Testing.Platform.MSBuild": "1.9.1", + "xunit.v3.extensibility.core": "[3.2.2]", + "xunit.v3.runner.inproc.console": "[3.2.2]" + } + }, + "xunit.v3.extensibility.core": { + "type": "Transitive", + "resolved": "3.2.2", + "contentHash": "srY8z/oMPvh/t8axtO2DwrHajhFMH7tnqKildvYrVQIfICi8fOn3yIBWkVPAcrKmHMwvXRJ/XsQM3VMR6DOYfQ==", + "dependencies": { + "xunit.v3.common": "[3.2.2]" + } + }, + "xunit.v3.mtp-v1": { + "type": "Transitive", + "resolved": "3.2.2", + "contentHash": "O41aAzYKBT5PWqATa1oEWVNCyEUypFQ4va6K0kz37dduV3EKzXNMaV2UnEhufzU4Cce1I33gg0oldS8tGL5I0A==", + "dependencies": { + "xunit.analyzers": "1.27.0", + "xunit.v3.assert": "[3.2.2]", + "xunit.v3.core.mtp-v1": "[3.2.2]" + } + }, + "xunit.v3.runner.common": { + "type": "Transitive", + "resolved": "3.2.2", + "contentHash": "/hkHkQCzGrugelOAehprm7RIWdsUFVmIVaD6jDH/8DNGCymTlKKPTbGokD5czbAfqfex47mBP0sb0zbHYwrO/g==", + "dependencies": { + "Microsoft.Win32.Registry": "[5.0.0]", + "xunit.v3.common": "[3.2.2]" + } + }, + "xunit.v3.runner.inproc.console": { + "type": "Transitive", + "resolved": "3.2.2", + "contentHash": "ulWOdSvCk+bPXijJZ73bth9NyoOHsAs1ZOvamYbCkD4DNLX/Bd29Ve2ZNUwBbK0MqfIYWXHZViy/HKrdEC/izw==", + "dependencies": { + "xunit.v3.extensibility.core": "[3.2.2]", + "xunit.v3.runner.common": "[3.2.2]" + } + }, + "opengameagent.attachments": { + "type": "Project" + }, + "opengameagent.attachments.local": { + "type": "Project", + "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", + "SkiaSharp": "[4.150.1, )", + "SkiaSharp.NativeAssets.Linux.NoDependencies": "[4.150.1, )" + } + } + } + } +} \ No newline at end of file diff --git a/tests/OpenGameAgent.Connectors.Mcp.Tests/packages.lock.json b/tests/OpenGameAgent.Connectors.Mcp.Tests/packages.lock.json index 2d9ec2b..d938423 100644 --- a/tests/OpenGameAgent.Connectors.Mcp.Tests/packages.lock.json +++ b/tests/OpenGameAgent.Connectors.Mcp.Tests/packages.lock.json @@ -218,6 +218,9 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.connectors.mcp": { "type": "Project", "dependencies": { @@ -235,6 +238,7 @@ "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/tests/OpenGameAgent.Extensions.Tests/packages.lock.json b/tests/OpenGameAgent.Extensions.Tests/packages.lock.json index ecd7682..9675a6d 100644 --- a/tests/OpenGameAgent.Extensions.Tests/packages.lock.json +++ b/tests/OpenGameAgent.Extensions.Tests/packages.lock.json @@ -209,6 +209,9 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.extensions": { "type": "Project", "dependencies": { @@ -219,6 +222,7 @@ "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/tests/OpenGameAgent.Kernel.Tests/ImageValidationTests.cs b/tests/OpenGameAgent.Kernel.Tests/ImageValidationTests.cs new file mode 100644 index 0000000..3614504 --- /dev/null +++ b/tests/OpenGameAgent.Kernel.Tests/ImageValidationTests.cs @@ -0,0 +1,103 @@ +using OpenGameAgent.Attachments; +using Xunit; + +namespace OpenGameAgent.Kernel.Tests; + +public sealed class ImageValidationTests +{ + [Fact] + public void MessageImageCountAndAggregateBytesAreBounded() + { + var message = Message( + Reference("one", bytes: 4, width: 1, height: 1), + Reference("two", bytes: 4, width: 1, height: 1)); + + Assert.Throws(() => AgentValidation.ValidateMessages( + new[] { message }, + new AgentLimits { MaxImagesPerMessage = 1 })); + Assert.Throws(() => AgentValidation.ValidateMessages( + new[] { message }, + new AgentLimits { MaxImageBytes = 4, MaxImageBytesPerMessage = 7 })); + } + + [Fact] + public void AttachmentPixelsAndBytesAreCheckedWithoutLoadingTheObject() + { + var message = Message(Reference("large", bytes: 11, width: 4, height: 3)); + + Assert.Throws(() => AgentValidation.ValidateMessages( + new[] { message }, + new AgentLimits { MaxImageBytes = 10 })); + Assert.Throws(() => AgentValidation.ValidateMessages( + new[] { message }, + new AgentLimits { MaxImagePixels = 11 })); + } + + [Fact] + public void InlineImageLimitsUseDecodedBytesAndApplyToToolResults() + { + var result = new ToolResult(new AgentContent[] + { + new BinaryContent(AgentMediaKind.Image, "AQIDBA==", GameImageMediaTypes.Png), + }); + + var call = new ToolCallContent("call", "inspect", "{}"); + var message = AgentMessage.ToolResult(call, result, DateTimeOffset.UnixEpoch); + Assert.Throws(() => AgentValidation.ValidateMessages( + new[] { message }, + new AgentLimits { MaxImageBytes = 3 })); + AgentValidation.ValidateMessages(new[] { message }, new AgentLimits { MaxImageBytes = 4 }); + } + + [Fact] + public void ImagesAreLimitedToUserAndToolResultMessages() + { + var image = Reference("role", bytes: 4, width: 1, height: 1); + var assistant = new AgentMessage( + AgentRole.Assistant, + new AgentContent[] { image }, + DateTimeOffset.UnixEpoch, + model: "model", + stopReason: ModelStopReason.Stop); + + Assert.Throws(() => AgentValidation.ValidateMessages( + new[] { assistant }, + new AgentLimits())); + Assert.Throws(() => AgentValidator.ValidateResponse( + new ModelResponse(new AgentContent[] { image }, ModelStopReason.Stop), + new AgentLimits())); + } + + [Fact] + public void InlineImageProgressIsStillBounded() + { + var progress = new ToolProgress(content: new AgentContent[] + { + new BinaryContent(AgentMediaKind.Image, "AQI=", GameImageMediaTypes.Png), + }); + + AgentValidator.ValidateProgress(progress, new AgentLimits()); + Assert.Throws(() => AgentValidator.ValidateProgress( + progress, + new AgentLimits { MaxImageBytes = 1, MaxImageBytesPerMessage = 1 })); + AgentValidator.ValidateProgress( + new ToolProgress(content: new AgentContent[] { Reference("progress", 1, 1, 1) }), + new AgentLimits()); + } + + private static AgentMessage Message(params AgentContent[] content) => new( + AgentRole.User, + content, + DateTimeOffset.UnixEpoch); + + private static ImageAttachmentContent Reference( + string suffix, + int bytes, + int width, + int height) => new(new GameImageAttachment( + "sha256:" + suffix, + GameImageMediaTypes.Png, + bytes, + width, + height)); +} diff --git a/tests/OpenGameAgent.Kernel.Tests/ProjectDependencyBoundaryTests.cs b/tests/OpenGameAgent.Kernel.Tests/ProjectDependencyBoundaryTests.cs index e4fe5ae..fb24db9 100644 --- a/tests/OpenGameAgent.Kernel.Tests/ProjectDependencyBoundaryTests.cs +++ b/tests/OpenGameAgent.Kernel.Tests/ProjectDependencyBoundaryTests.cs @@ -40,11 +40,14 @@ public void ReusableAiFoundationDoesNotDependOnGameHostPackages() } [Fact] - public void KernelIsDependencyFreeAndModelsOnlyDependsOnKernel() + public void AttachmentContractsAreDependencyFreeAndKernelUsesOnlyAttachmentContracts() { var root = FindRepositoryRoot(); - Assert.Empty(ReadProjectReferences(root, "OpenGameAgent.Kernel")); + Assert.Empty(ReadProjectReferences(root, "OpenGameAgent.Attachments")); + Assert.Equal( + new[] { "OpenGameAgent.Attachments" }, + ReadProjectReferences(root, "OpenGameAgent.Kernel")); Assert.Equal( new[] { "OpenGameAgent.Kernel" }, ReadProjectReferences(root, "OpenGameAgent.Models")); diff --git a/tests/OpenGameAgent.Kernel.Tests/PublicApiCompatibilityTests.cs b/tests/OpenGameAgent.Kernel.Tests/PublicApiCompatibilityTests.cs index eb0d519..381cce1 100644 --- a/tests/OpenGameAgent.Kernel.Tests/PublicApiCompatibilityTests.cs +++ b/tests/OpenGameAgent.Kernel.Tests/PublicApiCompatibilityTests.cs @@ -6,7 +6,7 @@ namespace OpenGameAgent.Kernel.Tests; public sealed class PublicApiCompatibilityTests { - private const string ApprovedApiHash = "7EC92B92D13B764CB0D3D3E8F71985220DBD6CF8C9D5DFC44C65C3D940CEEBBD"; + private const string ApprovedApiHash = "BD392AD6C0BCD5EE9209A96783278EA87C41CAF0CC9AAF1D30B78F1D7577EA71"; [Fact] public void KernelPublicApiMatchesTheApprovedStableSurface() diff --git a/tests/OpenGameAgent.Kernel.Tests/packages.lock.json b/tests/OpenGameAgent.Kernel.Tests/packages.lock.json index 520f91a..905dfec 100644 --- a/tests/OpenGameAgent.Kernel.Tests/packages.lock.json +++ b/tests/OpenGameAgent.Kernel.Tests/packages.lock.json @@ -202,9 +202,13 @@ "xunit.v3.runner.common": "[3.2.2]" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } } diff --git a/tests/OpenGameAgent.Media.Tests/packages.lock.json b/tests/OpenGameAgent.Media.Tests/packages.lock.json index cfeb704..0adb73c 100644 --- a/tests/OpenGameAgent.Media.Tests/packages.lock.json +++ b/tests/OpenGameAgent.Media.Tests/packages.lock.json @@ -209,9 +209,13 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/tests/OpenGameAgent.Memory.Tests/packages.lock.json b/tests/OpenGameAgent.Memory.Tests/packages.lock.json index 62ce6c9..4b28fa8 100644 --- a/tests/OpenGameAgent.Memory.Tests/packages.lock.json +++ b/tests/OpenGameAgent.Memory.Tests/packages.lock.json @@ -209,6 +209,9 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.extensions": { "type": "Project", "dependencies": { @@ -219,6 +222,7 @@ "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/tests/OpenGameAgent.Models.Auth.BuiltIn.Tests/packages.lock.json b/tests/OpenGameAgent.Models.Auth.BuiltIn.Tests/packages.lock.json index 709cba2..5af3a1a 100644 --- a/tests/OpenGameAgent.Models.Auth.BuiltIn.Tests/packages.lock.json +++ b/tests/OpenGameAgent.Models.Auth.BuiltIn.Tests/packages.lock.json @@ -259,9 +259,13 @@ "xunit.v3.runner.common": "[3.2.2]" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/tests/OpenGameAgent.Models.BuiltIn.Tests/BuiltInGameModelRuntimeTests.cs b/tests/OpenGameAgent.Models.BuiltIn.Tests/BuiltInGameModelRuntimeTests.cs index de4cf85..7ebe406 100644 --- a/tests/OpenGameAgent.Models.BuiltIn.Tests/BuiltInGameModelRuntimeTests.cs +++ b/tests/OpenGameAgent.Models.BuiltIn.Tests/BuiltInGameModelRuntimeTests.cs @@ -410,7 +410,7 @@ public void CatalogDescriptorsExposeOnlyMediaCapabilitiesImplementedByTextProvid } [Fact] - public async Task BundledAudioCapabilityIsDowngradedBeforeGoogleWireSerialization() + public async Task BundledModelRejectsUnsupportedAudioBeforeGoogleWireSerialization() { var handler = new RecordingHandler(BuiltInGameModelApis.GoogleGenerativeAi); using var client = new HttpClient(handler); @@ -445,8 +445,10 @@ public async Task BundledAudioCapabilityIsDowngradedBeforeGoogleWireSerializatio request, TestContext.Current.CancellationToken)); - Assert.Equal(ModelStreamEventKind.Completed, Assert.Single(events, item => item.IsTerminal).Kind); - Assert.Contains("[audio omitted: model does not support this input]", handler.Body, StringComparison.Ordinal); + var terminal = Assert.Single(events, item => item.IsTerminal); + Assert.Equal(ModelStreamEventKind.Failed, terminal.Kind); + Assert.Contains("does not declare audio input support", terminal.Response!.ErrorMessage, StringComparison.Ordinal); + Assert.Empty(handler.Body); Assert.DoesNotContain("YXVkaW8=", handler.Body, StringComparison.Ordinal); } @@ -869,7 +871,7 @@ async IAsyncEnumerable Transport( } [Fact] - public async Task UnsupportedUserAndToolImagesBecomeStableTextBeforeProviderSerialization() + public async Task UnsupportedUserAndToolImagesFailBeforeProviderSerialization() { var handler = new RecordingHandler(BuiltInGameModelApis.OpenAiCompletions); using var client = new HttpClient(handler); @@ -924,10 +926,9 @@ public async Task UnsupportedUserAndToolImagesBecomeStableTextBeforeProviderSeri TestContext.Current.CancellationToken)); var terminal = Assert.Single(events, item => item.IsTerminal); - Assert.True( - terminal.Kind == ModelStreamEventKind.Completed, - terminal.Response?.ErrorMessage ?? "The provider did not complete."); - Assert.Equal(2, Occurrences(handler.Body, "[image omitted: model does not support this input]")); + Assert.Equal(ModelStreamEventKind.Failed, terminal.Kind); + Assert.Contains("does not declare image input support", terminal.Response!.ErrorMessage, StringComparison.Ordinal); + Assert.Empty(handler.Body); Assert.DoesNotContain("aW1hZ2U=", handler.Body, StringComparison.Ordinal); Assert.DoesNotContain("dG9vbA==", handler.Body, StringComparison.Ordinal); } diff --git a/tests/OpenGameAgent.Models.BuiltIn.Tests/packages.lock.json b/tests/OpenGameAgent.Models.BuiltIn.Tests/packages.lock.json index 9c3a75c..cd05dd6 100644 --- a/tests/OpenGameAgent.Models.BuiltIn.Tests/packages.lock.json +++ b/tests/OpenGameAgent.Models.BuiltIn.Tests/packages.lock.json @@ -259,9 +259,13 @@ "xunit.v3.runner.common": "[3.2.2]" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/tests/OpenGameAgent.Models.Tests/ModelCatalogTests.cs b/tests/OpenGameAgent.Models.Tests/ModelCatalogTests.cs index 490236e..d961a32 100644 --- a/tests/OpenGameAgent.Models.Tests/ModelCatalogTests.cs +++ b/tests/OpenGameAgent.Models.Tests/ModelCatalogTests.cs @@ -1,5 +1,6 @@ using System.Collections.Concurrent; using System.Runtime.CompilerServices; +using OpenGameAgent.Attachments; using OpenGameAgent.Extensions; using OpenGameAgent.Kernel; using Xunit; @@ -70,6 +71,43 @@ public void DescriptorClampsReasoningAndResolutionBoundsParametersAndCost() requiredInput: GameModelInputCapabilities.Video)); } + [Fact] + public async Task DispatchProviderPreflightsImageCapabilityWithoutProviderIo() + { + var provider = new ScriptedProvider(); + var catalog = Catalog(Registration("provider", provider, Model("provider", "text-only"))); + var dispatch = catalog.CreateProvider("provider"); + var preflight = Assert.IsAssignableFrom(dispatch); + var request = new ModelRequest( + "text-only", + "", + new[] + { + new AgentMessage( + AgentRole.User, + new AgentContent[] + { + new ImageAttachmentContent(new GameImageAttachment( + "sha256:" + new string('a', 64), + GameImageMediaTypes.Png, + 1, + 1, + 1)), + }, + DateTimeOffset.UnixEpoch), + }, + Array.Empty(), + new ModelParameters(), + "session", + "run", + 1); + + await Assert.ThrowsAsync(() => preflight.ValidateRequestAsync( + request, + TestContext.Current.CancellationToken).AsTask()); + Assert.Empty(provider.Requests); + } + [Fact] public void PricingDistinguishesUnknownFromKnownFreeAndEstimatesItemizedUsage() { diff --git a/tests/OpenGameAgent.Models.Tests/packages.lock.json b/tests/OpenGameAgent.Models.Tests/packages.lock.json index 9d772d3..9851e65 100644 --- a/tests/OpenGameAgent.Models.Tests/packages.lock.json +++ b/tests/OpenGameAgent.Models.Tests/packages.lock.json @@ -266,6 +266,9 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.extensions": { "type": "Project", "dependencies": { @@ -276,6 +279,7 @@ "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/tests/OpenGameAgent.Persistence.Tests/PersistenceTests.cs b/tests/OpenGameAgent.Persistence.Tests/PersistenceTests.cs index 58b80d9..71f7057 100644 --- a/tests/OpenGameAgent.Persistence.Tests/PersistenceTests.cs +++ b/tests/OpenGameAgent.Persistence.Tests/PersistenceTests.cs @@ -1,4 +1,5 @@ using System.Text.Json.Nodes; +using OpenGameAgent.Attachments; using OpenGameAgent.Extensions; using OpenGameAgent.Kernel; using Xunit; @@ -22,6 +23,13 @@ public async Task SessionRoundTripsEveryCanonicalContentKindAcrossRestart() new TextContent("hello"), new JsonContent("{\"value\":1.25}"), new ResourceContent("game://asset/1", "application/game-object", "object"), + new ImageAttachmentContent(new GameImageAttachment( + "sha256:0123456789abcdef", + GameImageMediaTypes.Png, + 123, + 32, + 16, + "frame.png")), }, DateTimeOffset.UnixEpoch, metadata: new Dictionary { ["kind"] = "input" }), @@ -58,6 +66,10 @@ public async Task SessionRoundTripsEveryCanonicalContentKindAcrossRestart() Assert.NotNull(loaded); Assert.Equal(3, loaded.Messages.Count); Assert.Equal("{\"value\":1.25}", Assert.IsType(loaded.Messages[0].Content[1]).Json); + var image = Assert.IsType(loaded.Messages[0].Content[3]).Attachment; + Assert.Equal("sha256:0123456789abcdef", image.AttachmentId); + Assert.Equal(32, image.Width); + Assert.Equal("frame.png", image.Name); Assert.Equal("signature", Assert.IsType(loaded.Messages[1].Content[0]).Signature); Assert.True(Assert.IsType(loaded.Messages[1].Content[0]).Redacted); Assert.Equal(10, loaded.Messages[1].Usage!.InputTokens); diff --git a/tests/OpenGameAgent.Persistence.Tests/packages.lock.json b/tests/OpenGameAgent.Persistence.Tests/packages.lock.json index cd8632c..1213b06 100644 --- a/tests/OpenGameAgent.Persistence.Tests/packages.lock.json +++ b/tests/OpenGameAgent.Persistence.Tests/packages.lock.json @@ -209,6 +209,9 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.extensions": { "type": "Project", "dependencies": { @@ -219,6 +222,7 @@ "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/tests/OpenGameAgent.Plugins.Tests/packages.lock.json b/tests/OpenGameAgent.Plugins.Tests/packages.lock.json index 1c77817..713fa42 100644 --- a/tests/OpenGameAgent.Plugins.Tests/packages.lock.json +++ b/tests/OpenGameAgent.Plugins.Tests/packages.lock.json @@ -261,6 +261,9 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.connectors.mcp": { "type": "Project", "dependencies": { @@ -278,6 +281,7 @@ "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/tests/OpenGameAgent.Providers.Anthropic.Tests/packages.lock.json b/tests/OpenGameAgent.Providers.Anthropic.Tests/packages.lock.json index 6b6a4cb..c3cdfdd 100644 --- a/tests/OpenGameAgent.Providers.Anthropic.Tests/packages.lock.json +++ b/tests/OpenGameAgent.Providers.Anthropic.Tests/packages.lock.json @@ -202,9 +202,13 @@ "xunit.v3.runner.common": "[3.2.2]" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/tests/OpenGameAgent.Providers.Bedrock.Tests/packages.lock.json b/tests/OpenGameAgent.Providers.Bedrock.Tests/packages.lock.json index 4bb6cd1..32901a5 100644 --- a/tests/OpenGameAgent.Providers.Bedrock.Tests/packages.lock.json +++ b/tests/OpenGameAgent.Providers.Bedrock.Tests/packages.lock.json @@ -215,9 +215,13 @@ "xunit.v3.runner.common": "[3.2.2]" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/tests/OpenGameAgent.Providers.Google.Tests/packages.lock.json b/tests/OpenGameAgent.Providers.Google.Tests/packages.lock.json index 5c2e29b..cfc6a37 100644 --- a/tests/OpenGameAgent.Providers.Google.Tests/packages.lock.json +++ b/tests/OpenGameAgent.Providers.Google.Tests/packages.lock.json @@ -246,9 +246,13 @@ "xunit.v3.runner.common": "[3.2.2]" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/tests/OpenGameAgent.Providers.MediaHttp.Tests/packages.lock.json b/tests/OpenGameAgent.Providers.MediaHttp.Tests/packages.lock.json index 66d2fc5..e4d6b91 100644 --- a/tests/OpenGameAgent.Providers.MediaHttp.Tests/packages.lock.json +++ b/tests/OpenGameAgent.Providers.MediaHttp.Tests/packages.lock.json @@ -209,9 +209,13 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/tests/OpenGameAgent.Providers.MessageGateway.Tests/packages.lock.json b/tests/OpenGameAgent.Providers.MessageGateway.Tests/packages.lock.json index a4378fa..fdaa49f 100644 --- a/tests/OpenGameAgent.Providers.MessageGateway.Tests/packages.lock.json +++ b/tests/OpenGameAgent.Providers.MessageGateway.Tests/packages.lock.json @@ -209,9 +209,13 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/tests/OpenGameAgent.Providers.Mistral.Tests/packages.lock.json b/tests/OpenGameAgent.Providers.Mistral.Tests/packages.lock.json index d4d5b4a..e94ebb6 100644 --- a/tests/OpenGameAgent.Providers.Mistral.Tests/packages.lock.json +++ b/tests/OpenGameAgent.Providers.Mistral.Tests/packages.lock.json @@ -202,9 +202,13 @@ "xunit.v3.runner.common": "[3.2.2]" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/tests/OpenGameAgent.Providers.OpenAI.Tests/packages.lock.json b/tests/OpenGameAgent.Providers.OpenAI.Tests/packages.lock.json index 86c6861..ce88602 100644 --- a/tests/OpenGameAgent.Providers.OpenAI.Tests/packages.lock.json +++ b/tests/OpenGameAgent.Providers.OpenAI.Tests/packages.lock.json @@ -202,9 +202,13 @@ "xunit.v3.runner.common": "[3.2.2]" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/tests/OpenGameAgent.Providers.OpenAICompatible.Tests/packages.lock.json b/tests/OpenGameAgent.Providers.OpenAICompatible.Tests/packages.lock.json index 97e1ddc..1079bc2 100644 --- a/tests/OpenGameAgent.Providers.OpenAICompatible.Tests/packages.lock.json +++ b/tests/OpenGameAgent.Providers.OpenAICompatible.Tests/packages.lock.json @@ -202,9 +202,13 @@ "xunit.v3.runner.common": "[3.2.2]" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/tests/OpenGameAgent.Providers.OpenRouter.Tests/packages.lock.json b/tests/OpenGameAgent.Providers.OpenRouter.Tests/packages.lock.json index 67b37fe..e96eddf 100644 --- a/tests/OpenGameAgent.Providers.OpenRouter.Tests/packages.lock.json +++ b/tests/OpenGameAgent.Providers.OpenRouter.Tests/packages.lock.json @@ -189,9 +189,13 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/tests/OpenGameAgent.Providers.Remote.Tests/packages.lock.json b/tests/OpenGameAgent.Providers.Remote.Tests/packages.lock.json index b5d39fb..f84dfa8 100644 --- a/tests/OpenGameAgent.Providers.Remote.Tests/packages.lock.json +++ b/tests/OpenGameAgent.Providers.Remote.Tests/packages.lock.json @@ -202,9 +202,13 @@ "xunit.v3.runner.common": "[3.2.2]" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, diff --git a/tests/OpenGameAgent.Server.Tests/ServerTests.cs b/tests/OpenGameAgent.Server.Tests/ServerTests.cs index f633019..586f9b4 100644 --- a/tests/OpenGameAgent.Server.Tests/ServerTests.cs +++ b/tests/OpenGameAgent.Server.Tests/ServerTests.cs @@ -7,6 +7,7 @@ using Microsoft.AspNetCore.TestHost; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using OpenGameAgent.Attachments; using OpenGameAgent.Client; using OpenGameAgent.Kernel; using OpenGameAgent.Persistence; @@ -96,7 +97,7 @@ public async Task ServerRunPreservesResourceReferencesFromTheEngineWireFormat() "{}", new GameMoment("world", 1), "resource-input", - resources: new[] + content: new AgentContent[] { new ResourceContent("game://capture/frame", "image/png", "frame"), }); @@ -376,13 +377,13 @@ public async Task WireRoundTripPreservesAbsentCalendarAndStreamsToolIdentity() "event", "{}", new GameMoment("world", 1), - resources: new[] + content: new AgentContent[] { new ResourceContent("https://assets.example.test/frame.png", "image/png", "frame"), }); var roundTrip = GameAgentWire.ParseInput(GameAgentWire.SerializeInput(input)); Assert.Null(roundTrip.Moment.CalendarJson); - var roundTripResource = Assert.Single(roundTrip.Resources); + var roundTripResource = Assert.Single(roundTrip.Content.OfType()); Assert.Equal("https://assets.example.test/frame.png", roundTripResource.Uri); Assert.Equal("image/png", roundTripResource.MediaType); @@ -498,7 +499,7 @@ public async Task EngineCompatibleClientConsumesServerSse() }, TestContext.Current.CancellationToken); - Assert.True(result.Succeeded); + Assert.True(result.Succeeded, result.Error); Assert.Contains(events, item => item.Name == "agent" && item.Json.Contains("TextDelta", StringComparison.Ordinal)); Assert.Equal("result", events.Last().Name); } @@ -827,6 +828,7 @@ public async Task AudienceProjectionProtectsReasoningAndToolDetailsForOwnerAndPu Assert.Contains("visible-answer", ownerJson, StringComparison.Ordinal); Assert.DoesNotContain("private-reasoning", ownerJson, StringComparison.Ordinal); Assert.DoesNotContain("reasoning-signature", ownerJson, StringComparison.Ordinal); + Assert.DoesNotContain("text-signature", ownerJson, StringComparison.Ordinal); Assert.DoesNotContain("private-tool-result", ownerJson, StringComparison.Ordinal); Assert.DoesNotContain("private-tool-details", ownerJson, StringComparison.Ordinal); Assert.DoesNotContain("secret-argument", ownerJson, StringComparison.Ordinal); @@ -851,6 +853,7 @@ public async Task AudienceProjectionProtectsReasoningAndToolDetailsForOwnerAndPu internalResponse.EnsureSuccessStatusCode(); Assert.Contains("private-reasoning", internalStream, StringComparison.Ordinal); Assert.Contains("reasoning-signature", internalStream, StringComparison.Ordinal); + Assert.Contains("text-signature", internalStream, StringComparison.Ordinal); Assert.Contains("private-tool-result", internalStream, StringComparison.Ordinal); Assert.Contains("private-tool-details", internalStream, StringComparison.Ordinal); Assert.Contains("secret-argument", internalStream, StringComparison.Ordinal); @@ -874,6 +877,7 @@ public async Task AudienceProjectionProtectsReasoningAndToolDetailsForOwnerAndPu Assert.Contains("visible-answer", publicStream, StringComparison.Ordinal); Assert.DoesNotContain("private-reasoning", publicStream, StringComparison.Ordinal); Assert.DoesNotContain("reasoning-signature", publicStream, StringComparison.Ordinal); + Assert.DoesNotContain("text-signature", publicStream, StringComparison.Ordinal); Assert.DoesNotContain("private-tool-result", publicStream, StringComparison.Ordinal); Assert.DoesNotContain("private-tool-details", publicStream, StringComparison.Ordinal); Assert.DoesNotContain("secret-argument", publicStream, StringComparison.Ordinal); @@ -1159,6 +1163,117 @@ public void ServerRejectsWhitespaceOnlyApiKeyConfiguration() })); } + [Fact] + public async Task AttachmentReadIsSessionAuthorizedBeforeTheRuntimeOrStoreIsTouched() + { + var attachments = new ServerAttachmentStore(); + var key = new GameSessionKey("image-session", "image-actor"); + var runtime = new GameAgentRuntime(new GameAgentRuntimeOptions(new StreamingProvider(), "vision-model") + { + ImageAttachments = attachments, + }); + var authorizer = new TestOwnerAuthorizer((subject, requested, _) => + string.Equals(subject, "owner-a", StringComparison.Ordinal) + && requested.Equals(key)); + await using var app = await CreateAppAsync(runtime, authorizer: authorizer); + using var client = app.GetTestClient(); + var input = new GameInput( + key.SessionId, + key.ActorId, + "observe", + "{}", + new GameMoment("world", 1), + "image-input", + content: new AgentContent[] + { + new BinaryContent(AgentMediaKind.Image, "AQID", GameImageMediaTypes.Png, "frame.png"), + }); + using var run = CreateOwnedRequest( + HttpMethod.Post, + "/v1/run", + "owner-a", + GameAgentWire.SerializeInput(input)); + using var runResponse = await client.SendAsync(run, TestContext.Current.CancellationToken); + runResponse.EnsureSuccessStatusCode(); + var attachment = Assert.IsType(attachments.LastAttachment); + var readsAfterRun = attachments.ReadCount; + var requestJson = JsonSerializer.Serialize(new + { + sessionId = key.SessionId, + actorId = key.ActorId, + attachmentId = attachment.AttachmentId, + }); + + using var forbidden = CreateOwnedRequest( + HttpMethod.Post, + "/v1/attachments/read", + "owner-b", + requestJson); + using var forbiddenResponse = await client.SendAsync(forbidden, TestContext.Current.CancellationToken); + + Assert.Equal(System.Net.HttpStatusCode.Forbidden, forbiddenResponse.StatusCode); + Assert.Equal(readsAfterRun, attachments.ReadCount); + Assert.Contains(authorizer.Calls, call => call.Operation == GameAgentServerOperation.ReadAttachment); + + using var allowed = CreateOwnedRequest( + HttpMethod.Post, + "/v1/attachments/read", + "owner-a", + requestJson); + using var allowedResponse = await client.SendAsync(allowed, TestContext.Current.CancellationToken); + var allowedJson = await allowedResponse.Content.ReadAsStringAsync(TestContext.Current.CancellationToken); + + allowedResponse.EnsureSuccessStatusCode(); + using var document = JsonDocument.Parse(allowedJson); + Assert.Equal("AQID", document.RootElement.GetProperty("data").GetString()); + Assert.Equal(attachment.AttachmentId, document.RootElement + .GetProperty("attachment") + .GetProperty("attachmentId") + .GetString()); + } + + [Fact] + public async Task ServerClientReadsOnlyAttachmentsReferencedByTheSession() + { + var attachments = new ServerAttachmentStore(); + var runtime = new GameAgentRuntime(new GameAgentRuntimeOptions(new StreamingProvider(), "vision-model") + { + ImageAttachments = attachments, + }); + await using var app = await CreateAppAsync(runtime); + using var http = app.GetTestClient(); + var remote = new ServerGameAgentClient(new ServerGameAgentClientOptions( + http, + new Uri("http://localhost/"))); + var input = new GameInput( + "client-image-session", + "client-image-actor", + "observe", + "{}", + new GameMoment("world", 1), + "client-image-input", + content: new AgentContent[] + { + new BinaryContent(AgentMediaKind.Image, "BAUG", GameImageMediaTypes.Png), + }); + + var result = await remote.RunAsync(input, TestContext.Current.CancellationToken); + var attachment = Assert.IsType(attachments.LastAttachment); + var stored = await remote.ReadImageAttachmentAsync( + new GameSessionKey(input.SessionId, input.ActorId), + attachment.AttachmentId, + TestContext.Current.CancellationToken); + var missing = await remote.ReadImageAttachmentAsync( + new GameSessionKey(input.SessionId, "another-actor"), + attachment.AttachmentId, + TestContext.Current.CancellationToken); + + Assert.True(result.Succeeded); + Assert.NotNull(stored); + Assert.Equal(new byte[] { 4, 5, 6 }, stored.Data.ToArray()); + Assert.Null(missing); + } + private static async Task CreateAppAsync( string? apiKey = null, int maximumRequestBodyBytes = ServerEndpoints.DefaultMaximumRequestBodyBytes) @@ -1388,6 +1503,49 @@ public async IAsyncEnumerable StreamAsync( } } + private sealed class ServerAttachmentStore : IGameImageAttachmentStore + { + private readonly Dictionary _objects = new(StringComparer.Ordinal); + private int _readCount; + + public GameImageAttachmentLimits ImageLimits { get; } = new(); + + public GameImageAttachment? LastAttachment { get; private set; } + + public int ReadCount => Volatile.Read(ref _readCount); + + public ValueTask ValidateImageAsync( + SaveGameImageAttachment input, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return default; + } + + public ValueTask SaveImageAsync( + SaveGameImageAttachment input, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + var data = input.Data.ToArray(); + var id = "sha256:" + Convert.ToHexString( + System.Security.Cryptography.SHA256.HashData(data)).ToLowerInvariant(); + var attachment = new GameImageAttachment(id, input.MediaType, input.Data.Length, 1, 1, input.Name); + _objects[id] = new StoredGameImageAttachment(attachment, data); + LastAttachment = attachment; + return new ValueTask(attachment); + } + + public ValueTask ReadImageAsync( + GameImageAttachment attachment, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + Interlocked.Increment(ref _readCount); + return new ValueTask(_objects[attachment.AttachmentId]); + } + } + private sealed class ResourceCaptureProvider : IModelProvider { public System.Collections.Concurrent.ConcurrentQueue Requests { get; } = new(); @@ -1559,7 +1717,7 @@ public async IAsyncEnumerable StreamAsync( } yield return ModelStreamEvent.Terminal(new ModelResponse( - new AgentContent[] { new TextContent("visible-answer") }, + new AgentContent[] { new TextContent("visible-answer", "text-signature") }, ModelStopReason.Stop)); } } diff --git a/tests/OpenGameAgent.Server.Tests/packages.lock.json b/tests/OpenGameAgent.Server.Tests/packages.lock.json index 3962b74..034edea 100644 --- a/tests/OpenGameAgent.Server.Tests/packages.lock.json +++ b/tests/OpenGameAgent.Server.Tests/packages.lock.json @@ -121,6 +121,30 @@ "System.Security.Principal.Windows": "5.0.0" } }, + "SkiaSharp": { + "type": "Transitive", + "resolved": "4.150.1", + "contentHash": "5v3T8X1N62Dp+AkPO70GNBNS/NRBPGMOTiN+Prg33sZAcm/Ug3YOAH+3RTj/jxJV8NGTJs2idGpC2Qdae2mGLQ==", + "dependencies": { + "SkiaSharp.NativeAssets.Win32": "4.150.1", + "SkiaSharp.NativeAssets.macOS": "4.150.1" + } + }, + "SkiaSharp.NativeAssets.Linux.NoDependencies": { + "type": "Transitive", + "resolved": "4.150.1", + "contentHash": "2KVadgDky2xQw7lEMRT/u0ftX5K1u8X7MhCN47Em22Z6VM6JJg1c9cwfGuSPAt1oY6+GjGHH18vkeev+nqT9Kw==" + }, + "SkiaSharp.NativeAssets.macOS": { + "type": "Transitive", + "resolved": "4.150.1", + "contentHash": "r755HVwaHZhyf1clWjrM2/RoOZYCzkQEmE9pu/mVsebPejWu52niNPUwtfyf112qoF0PIk6OndqVOUoITj6TwQ==" + }, + "SkiaSharp.NativeAssets.Win32": { + "type": "Transitive", + "resolved": "4.150.1", + "contentHash": "qrLSL8OonbkMJdSH8heK0Jl39Y2xxZGdd7Ru4cyBXk2ITdx+Fu1sCFLmvCKmnMaYs//mlN/YSpSpOurEXErcQw==" + }, "System.Collections.Immutable": { "type": "Transitive", "resolved": "8.0.0", @@ -223,10 +247,22 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, + "opengameagent.attachments.local": { + "type": "Project", + "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", + "SkiaSharp": "[4.150.1, )", + "SkiaSharp.NativeAssets.Linux.NoDependencies": "[4.150.1, )" + } + }, "opengameagent.client": { "type": "Project", "dependencies": { "OpenGameAgent": "[0.3.0-alpha.2, )", + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, @@ -240,6 +276,7 @@ "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } }, @@ -272,6 +309,8 @@ "type": "Project", "dependencies": { "OpenGameAgent": "[0.3.0-alpha.2, )", + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", + "OpenGameAgent.Attachments.Local": "[0.3.0-alpha.2, )", "OpenGameAgent.Persistence": "[0.3.0-alpha.2, )", "OpenGameAgent.Providers.OpenAICompatible": "[0.3.0-alpha.2, )" } diff --git a/tests/OpenGameAgent.Tests/ImageRuntimeTests.cs b/tests/OpenGameAgent.Tests/ImageRuntimeTests.cs new file mode 100644 index 0000000..286f29c --- /dev/null +++ b/tests/OpenGameAgent.Tests/ImageRuntimeTests.cs @@ -0,0 +1,308 @@ +using System.Collections.Concurrent; +using System.Runtime.CompilerServices; +using System.Security.Cryptography; +using OpenGameAgent.Attachments; +using OpenGameAgent.Kernel; +using Xunit; + +namespace OpenGameAgent.Tests; + +public sealed class ImageRuntimeTests +{ + [Fact] + public async Task InputImagesArePersistedAsReferencesAndResolvedOnlyForTheProvider() + { + var bytes = new byte[] { 1, 2, 3, 4 }; + var attachments = new RecordingAttachmentStore(); + var sessions = new InMemoryGameSessionStore(); + var provider = new RecordingProvider(request => + { + var image = Assert.Single(request.Messages.SelectMany(message => message.Content).OfType()); + Assert.Equal(Convert.ToBase64String(bytes), image.Data); + return Text("seen"); + }); + var runtime = new GameAgentRuntime(new GameAgentRuntimeOptions(provider, "vision-model") + { + ImageAttachments = attachments, + SessionStore = sessions, + }); + var input = Input( + "input-image", + new BinaryContent(AgentMediaKind.Image, Convert.ToBase64String(bytes), GameImageMediaTypes.Png, "frame.png")); + + var result = await runtime.RunAsync(input, TestContext.Current.CancellationToken); + + Assert.True(result.Succeeded, result.Error); + Assert.Equal(1, attachments.SaveCount); + var key = new GameSessionKey(input.SessionId, input.ActorId); + var snapshot = Assert.IsType(await sessions.LoadAsync(key, TestContext.Current.CancellationToken)); + var reference = Assert.Single(snapshot.Messages.SelectMany(message => message.Content).OfType()); + Assert.Empty(snapshot.Messages.SelectMany(message => message.Content).OfType()); + var stored = Assert.IsType(await runtime.ReadImageAttachmentAsync( + key, + reference.Attachment.AttachmentId, + TestContext.Current.CancellationToken)); + Assert.Equal(bytes, stored.Data.ToArray()); + Assert.Null(await runtime.ReadImageAttachmentAsync( + new GameSessionKey(input.SessionId, "other-actor"), + reference.Attachment.AttachmentId, + TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task ImageBatchIsFullyValidatedBeforeAnyObjectIsSaved() + { + var attachments = new RecordingAttachmentStore { FailValidationCall = 2 }; + var provider = new RecordingProvider(_ => Text("must-not-run")); + var runtime = new GameAgentRuntime(new GameAgentRuntimeOptions(provider, "vision-model") + { + ImageAttachments = attachments, + }); + var input = Input( + "invalid-batch", + new BinaryContent(AgentMediaKind.Image, "AQ==", GameImageMediaTypes.Png), + new BinaryContent(AgentMediaKind.Image, "Ag==", GameImageMediaTypes.Png)); + + await Assert.ThrowsAsync( + () => runtime.RunAsync(input, TestContext.Current.CancellationToken)); + + Assert.Equal(2, attachments.ValidateCount); + Assert.Equal(0, attachments.SaveCount); + Assert.Equal(0, provider.CallCount); + } + + [Fact] + public async Task ToolImagesArePersistedBeforeTheNextModelTurnAndSessionCommit() + { + var attachments = new RecordingAttachmentStore(); + var sessions = new InMemoryGameSessionStore(); + var provider = new RecordingProvider(request => + { + if (request.Turn == 1) + { + return new ModelResponse( + new AgentContent[] { new ToolCallContent("call-1", "inspect_scene", "{}") }, + ModelStopReason.ToolUse); + } + + var toolImage = Assert.Single(request.Messages + .Where(message => message.Role == AgentRole.Tool) + .SelectMany(message => message.Content) + .OfType()); + Assert.Equal("CQgH", toolImage.Data); + return Text("done"); + }); + var runtime = new GameAgentRuntime(new GameAgentRuntimeOptions(provider, "vision-model") + { + ImageAttachments = attachments, + SessionStore = sessions, + ToolProvider = (_, _) => new ValueTask>(new[] + { + new AgentTool( + new ToolDefinition("inspect_scene", "capture", "{\"type\":\"object\",\"additionalProperties\":false}"), + (_, _, _) => new ValueTask(new ToolResult(new AgentContent[] + { + new BinaryContent(AgentMediaKind.Image, "CQgH", GameImageMediaTypes.Png, "tool.png"), + })), + ToolRisk.ReadOnly), + }), + }); + var input = Input("tool-image"); + + var result = await runtime.RunAsync(input, TestContext.Current.CancellationToken); + + Assert.True(result.Succeeded, result.Error); + Assert.Equal(1, attachments.SaveCount); + var snapshot = Assert.IsType(await sessions.LoadAsync( + new GameSessionKey(input.SessionId, input.ActorId), + TestContext.Current.CancellationToken)); + Assert.Single(snapshot.Messages + .Where(message => message.Role == AgentRole.Tool) + .SelectMany(message => message.Content) + .OfType()); + Assert.Empty(snapshot.Messages.SelectMany(message => message.Content).OfType()); + } + + [Fact] + public async Task ProviderPreflightRejectsBeforeAttachmentBytesAreReadOrProviderIsCalled() + { + var attachments = new RecordingAttachmentStore(); + var provider = new RejectingPreflightProvider(); + var runtime = new GameAgentRuntime(new GameAgentRuntimeOptions(provider, "text-model") + { + ImageAttachments = attachments, + }); + + var result = await runtime.RunAsync( + Input( + "preflight", + new BinaryContent(AgentMediaKind.Image, "AQ==", GameImageMediaTypes.Png)), + TestContext.Current.CancellationToken); + + Assert.False(result.Succeeded); + Assert.Equal(1, attachments.SaveCount); + Assert.Equal(0, attachments.ReadCount); + Assert.Equal(1, provider.PreflightCount); + Assert.Equal(0, provider.CallCount); + } + + [Fact] + public async Task RetryWrapperPreservesPreflightBeforeAttachmentRead() + { + var attachments = new RecordingAttachmentStore(); + var inner = new RejectingPreflightProvider(); + var runtime = new GameAgentRuntime(new GameAgentRuntimeOptions( + new RetryingModelProvider(inner), + "text-model") + { + ImageAttachments = attachments, + }); + + var result = await runtime.RunAsync( + Input( + "wrapped-preflight", + new BinaryContent(AgentMediaKind.Image, "AQ==", GameImageMediaTypes.Png)), + TestContext.Current.CancellationToken); + + Assert.False(result.Succeeded); + Assert.Equal(0, attachments.ReadCount); + Assert.Equal(1, inner.PreflightCount); + Assert.Equal(0, inner.CallCount); + } + + [Fact] + public void GameInputRejectsCallerSuppliedDurableImageReferences() + { + var attachment = new GameImageAttachment( + "sha256:" + new string('a', 64), + GameImageMediaTypes.Png, + 1, + 1, + 1); + + Assert.Throws(() => Input( + "forged-reference", + new ImageAttachmentContent(attachment))); + } + + private static GameInput Input(string inputId, params AgentContent[] content) => new( + "image-session", + "image-actor", + "observe", + "{}", + new GameMoment("world", 1), + inputId, + content: content); + + private static ModelResponse Text(string text) => new( + new AgentContent[] { new TextContent(text) }, + ModelStopReason.Stop); + + private sealed class RecordingProvider : IModelProvider + { + private readonly Func _handler; + private int _calls; + + public RecordingProvider(Func handler) + { + _handler = handler; + } + + public int CallCount => Volatile.Read(ref _calls); + + public async IAsyncEnumerable StreamAsync( + ModelRequest request, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Interlocked.Increment(ref _calls); + await Task.Yield(); + yield return ModelStreamEvent.Terminal(_handler(request)); + } + } + + private sealed class RejectingPreflightProvider : IModelProvider, IModelRequestPreflight + { + private int _preflightCount; + private int _callCount; + + public int PreflightCount => Volatile.Read(ref _preflightCount); + + public int CallCount => Volatile.Read(ref _callCount); + + public ValueTask ValidateRequestAsync(ModelRequest request, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Interlocked.Increment(ref _preflightCount); + throw new ModelProviderException("image input is unsupported", isTransient: false); + } + + public async IAsyncEnumerable StreamAsync( + ModelRequest request, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + Interlocked.Increment(ref _callCount); + await Task.Yield(); + yield return ModelStreamEvent.Terminal(Text("unexpected")); + } + } + + private sealed class RecordingAttachmentStore : IGameImageAttachmentStore + { + private readonly ConcurrentDictionary _objects = new(StringComparer.Ordinal); + private int _validateCount; + private int _saveCount; + private int _readCount; + + public GameImageAttachmentLimits ImageLimits { get; } = new(); + + public int? FailValidationCall { get; set; } + + public int ValidateCount => Volatile.Read(ref _validateCount); + + public int SaveCount => Volatile.Read(ref _saveCount); + + public int ReadCount => Volatile.Read(ref _readCount); + + public ValueTask ValidateImageAsync( + SaveGameImageAttachment input, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + var call = Interlocked.Increment(ref _validateCount); + if (FailValidationCall == call) + { + throw new GameAttachmentException("INVALID_IMAGE", "simulated invalid image"); + } + + return default; + } + + public ValueTask SaveImageAsync( + SaveGameImageAttachment input, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + Interlocked.Increment(ref _saveCount); + var data = input.Data.ToArray(); + var id = "sha256:" + Convert.ToHexString(SHA256.HashData(data)).ToLowerInvariant(); + var attachment = new GameImageAttachment(id, input.MediaType, input.Data.Length, 1, 1, input.Name); + _objects.TryAdd(id, new StoredGameImageAttachment(attachment, data)); + return new ValueTask(attachment); + } + + public ValueTask ReadImageAsync( + GameImageAttachment attachment, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + Interlocked.Increment(ref _readCount); + if (!_objects.TryGetValue(attachment.AttachmentId, out var stored)) + { + throw new GameAttachmentException("ATTACHMENT_NOT_FOUND", "missing attachment"); + } + + return new ValueTask(stored); + } + } +} diff --git a/tests/OpenGameAgent.Tests/PublicApiCompatibilityTests.cs b/tests/OpenGameAgent.Tests/PublicApiCompatibilityTests.cs index 03f14e1..22a9a08 100644 --- a/tests/OpenGameAgent.Tests/PublicApiCompatibilityTests.cs +++ b/tests/OpenGameAgent.Tests/PublicApiCompatibilityTests.cs @@ -5,7 +5,7 @@ namespace OpenGameAgent.Tests; public sealed class PublicApiCompatibilityTests { - private const string ApprovedApiHash = "D57DE3663DE8EA3EAB266C4A586B2A10C9E03676D554F4E70F5C31F2F7E9C19D"; + private const string ApprovedApiHash = "436EAE0B0E6367F7532232DD799B22DB6CA3C2A01EC0E9D4009EE9B157427AEB"; [Fact] public void RuntimePublicApiMatchesTheApprovedStableSurface() diff --git a/tests/OpenGameAgent.Tests/RuntimeTests.cs b/tests/OpenGameAgent.Tests/RuntimeTests.cs index aab6cb4..c35839e 100644 --- a/tests/OpenGameAgent.Tests/RuntimeTests.cs +++ b/tests/OpenGameAgent.Tests/RuntimeTests.cs @@ -106,7 +106,7 @@ public async Task StructuredGameInputForwardsAttachedModelResources() "observation", "{\"question\":\"what is visible?\"}", new GameMoment("world", 10), - resources: new[] + content: new AgentContent[] { new ResourceContent("https://assets.example.test/frame.png", "image/png", "camera"), }); diff --git a/tests/OpenGameAgent.Tests/packages.lock.json b/tests/OpenGameAgent.Tests/packages.lock.json index f55ba4f..a7cb7ab 100644 --- a/tests/OpenGameAgent.Tests/packages.lock.json +++ b/tests/OpenGameAgent.Tests/packages.lock.json @@ -209,9 +209,13 @@ "System.Text.Json": "[8.0.6, )" } }, + "opengameagent.attachments": { + "type": "Project" + }, "opengameagent.kernel": { "type": "Project", "dependencies": { + "OpenGameAgent.Attachments": "[0.3.0-alpha.2, )", "System.Text.Json": "[8.0.6, )" } } diff --git a/tools/release-packages.json b/tools/release-packages.json index 6bc05dd..cd2c41d 100644 --- a/tools/release-packages.json +++ b/tools/release-packages.json @@ -1,6 +1,10 @@ { "schemaVersion": 1, "packages": [ + { + "id": "OpenGameAgent.Attachments", + "project": "src/OpenGameAgent.Attachments/OpenGameAgent.Attachments.csproj" + }, { "id": "OpenGameAgent.Kernel", "project": "src/OpenGameAgent.Kernel/OpenGameAgent.Kernel.csproj" @@ -85,6 +89,10 @@ "id": "OpenGameAgent.Persistence", "project": "src/OpenGameAgent.Persistence/OpenGameAgent.Persistence.csproj" }, + { + "id": "OpenGameAgent.Attachments.Local", + "project": "src/OpenGameAgent.Attachments.Local/OpenGameAgent.Attachments.Local.csproj" + }, { "id": "OpenGameAgent.Plugins", "project": "src/OpenGameAgent.Plugins/OpenGameAgent.Plugins.csproj" From 592fbc907edb2154c352891a5f66cdf50a046231 Mon Sep 17 00:00:00 2001 From: Eric Sun <141227631+EricSun0218@users.noreply.github.com> Date: Sat, 15 Aug 2026 19:09:06 +0800 Subject: [PATCH 2/2] build: trim unsupported server runtime assets --- tools/New-ReleaseBundle.ps1 | 2 +- tools/Release.Common.ps1 | 24 ++++++++++++++++++++++-- tools/Test-ReleaseScripts.ps1 | 6 ++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/tools/New-ReleaseBundle.ps1 b/tools/New-ReleaseBundle.ps1 index 9691205..4977c95 100644 --- a/tools/New-ReleaseBundle.ps1 +++ b/tools/New-ReleaseBundle.ps1 @@ -132,7 +132,7 @@ $releaseNotes = @( "dotnet add package OpenGameAgent --version $Version", '```', '', - 'Use the versioned Godot or Unity archive below for engine integration. The portable server archive runs with `dotnet OpenGameAgent.Server.dll` on a .NET 8 host.', + 'Use the versioned Godot or Unity archive below for engine integration. The portable server archive runs with `dotnet OpenGameAgent.Server.dll` on a Windows or Linux .NET 8 host.', '', (Get-ReleaseStabilityNotice -VersionInfo $versionInfo) ) -join [Environment]::NewLine diff --git a/tools/Release.Common.ps1 b/tools/Release.Common.ps1 index 0c39720..5b1db87 100644 --- a/tools/Release.Common.ps1 +++ b/tools/Release.Common.ps1 @@ -456,6 +456,24 @@ function Get-ReleasePackageLayers { }) } +function Test-SupportedPortableServerRuntimeAsset { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string] $AssetPath + ) + + $normalized = $AssetPath.Replace('\', '/').TrimStart('/') + if ([IO.Path]::GetExtension($normalized) -eq '.pdb') { + return $false + } + if ($normalized.StartsWith('runtimes/osx', [StringComparison]::OrdinalIgnoreCase)) { + return $false + } + + return $true +} + function Resolve-PortableServerRuntimeAssets { [CmdletBinding()] param( @@ -488,7 +506,8 @@ function Resolve-PortableServerRuntimeAssets { continue } foreach ($assetName in $section.Value.PSObject.Properties.Name) { - if ([IO.Path]::GetFileName([string]$assetName) -eq '_._') { + if ([IO.Path]::GetFileName([string]$assetName) -eq '_._' -or + -not (Test-SupportedPortableServerRuntimeAsset -AssetPath ([string]$assetName))) { continue } $null = $declaredAssets.Add([string]$assetName) @@ -546,7 +565,8 @@ function Resolve-PortableServerRuntimeAssets { $null = $resolvedSources.Add([IO.Path]::GetFullPath($source)) } $publishedRuntimeFiles = @(Get-ChildItem -LiteralPath $publishRoot -Recurse -File | Where-Object { - $_.Extension -in @('.dll', '.so', '.dylib') + $_.Extension -in @('.dll', '.so', '.dylib') -and + (Test-SupportedPortableServerRuntimeAsset -AssetPath ([IO.Path]::GetRelativePath($publishRoot, $_.FullName))) }) foreach ($publishedRuntimeFile in $publishedRuntimeFiles) { if (-not $resolvedSources.Contains([IO.Path]::GetFullPath($publishedRuntimeFile.FullName))) { diff --git a/tools/Test-ReleaseScripts.ps1 b/tools/Test-ReleaseScripts.ps1 index 632888b..27c16cf 100644 --- a/tools/Test-ReleaseScripts.ps1 +++ b/tools/Test-ReleaseScripts.ps1 @@ -64,6 +64,12 @@ foreach ($invalidVersion in @( $packages = @(Get-ReleasePackageManifest -RepositoryRoot $repositoryRoot) Assert-ReleasePackageManifestGraph -RepositoryRoot $repositoryRoot -Packages $packages +if (-not (Test-SupportedPortableServerRuntimeAsset -AssetPath 'runtimes/linux-x64/native/libExample.so') -or + -not (Test-SupportedPortableServerRuntimeAsset -AssetPath 'runtimes/win-x64/native/Example.dll') -or + (Test-SupportedPortableServerRuntimeAsset -AssetPath 'runtimes/win-x64/native/Example.pdb') -or + (Test-SupportedPortableServerRuntimeAsset -AssetPath 'runtimes/osx/native/libExample.dylib')) { + throw 'Portable server asset filtering must retain Windows/Linux runtime files and exclude symbols/macOS assets.' +} $godotDownloadPattern = "Godot_v4\.7\.1-stable_mono_win64\.zip'.*-MaximumRetryCount\s+4\s+-RetryIntervalSec\s+5" foreach ($workflowPath in @('.github\workflows\ci.yml', '.github\workflows\release.yml')) { $workflow = Get-Content -LiteralPath (Join-Path $repositoryRoot $workflowPath) -Raw