From 156ae565fcd26d58cfb553d91a0cb183edc48153 Mon Sep 17 00:00:00 2001 From: Samuel Galanakis Date: Sat, 25 Jul 2026 14:04:39 +0200 Subject: [PATCH] Add MIME hints to provider-file attachments Carry optional validated media types through core and remote provider-file attachments. Anthropic uses the hint to select image or document file blocks and rejects missing hints before transport, while OpenAI and Google continue resolving file types server-side. Release-Notes: Breaking: Provider-file attachments now carry an optional media type, and Anthropic provider file ids require that hint so Lash can choose the image or document modality. --- crates/lash-core/src/llm/transport.rs | 19 +++++-- crates/lash-provider-anthropic/src/lib.rs | 53 +++++++++++++++++++ crates/lash-provider-anthropic/src/request.rs | 24 ++++++--- crates/lash-provider-google/src/lib.rs | 22 ++++++++ .../src/tests/attachment_tests.rs | 26 +++++++++ .../src/core_conversions/llm.rs | 30 +++++++---- .../src/core_conversions_tests.rs | 27 ++++++++++ crates/lash-remote-protocol/src/llm.rs | 19 ++++++- crates/lash-sansio/src/llm/types.rs | 49 +++++++++++++++-- 9 files changed, 244 insertions(+), 25 deletions(-) diff --git a/crates/lash-core/src/llm/transport.rs b/crates/lash-core/src/llm/transport.rs index 2ce6ec94..08075d3d 100644 --- a/crates/lash-core/src/llm/transport.rs +++ b/crates/lash-core/src/llm/transport.rs @@ -85,10 +85,20 @@ pub fn unsupported_attachment_capability( accepted_by.join(", ") }; let message = match source { - AttachmentSource::ProviderFile { provider_scope, .. } => format!( - "{provider} cannot materialize attachment source `provider_file` scoped to provider `{}`; the source carries no caller MIME; providers accepting this source: {accepted}", - provider_scope.provider - ), + AttachmentSource::ProviderFile { + provider_scope, + media_type, + .. + } => match media_type { + Some(media_type) => format!( + "{provider} cannot materialize attachment MIME `{media_type}` from source `provider_file` scoped to provider `{}`; providers accepting this source: {accepted}", + provider_scope.provider + ), + None => format!( + "{provider} cannot materialize attachment source `provider_file` scoped to provider `{}`; the source carries no caller MIME; providers accepting this source: {accepted}", + provider_scope.provider + ), + }, source => { let media_type = source .media_type() @@ -176,6 +186,7 @@ mod tests { let google_file = AttachmentSource::provider_file( ProviderFileScope::new("gemini", "credential"), "file-id", + None, ); assert_eq!( known_attachment_acceptors(&google_file), diff --git a/crates/lash-provider-anthropic/src/lib.rs b/crates/lash-provider-anthropic/src/lib.rs index 79bfb8c0..e7d2e7e3 100644 --- a/crates/lash-provider-anthropic/src/lib.rs +++ b/crates/lash-provider-anthropic/src/lib.rs @@ -289,6 +289,59 @@ mod tests { assert_eq!(block["source"]["url"], "https://example.test/report.pdf"); } + #[test] + fn provider_file_uses_media_type_to_select_image_or_document_block() { + let provider = AnthropicProvider::new("key"); + + for (mime, expected_block_type, file_id) in [ + ("image/png", "image", "file-image"), + ("application/pdf", "document", "file-document"), + ] { + let mut req = request(vec![LlmMessage::new( + LlmRole::User, + vec![LlmContentBlock::Attachment { attachment_idx: 0 }], + )]); + req.attachments = vec![AttachmentSource::provider_file( + lash_core::ProviderFileScope::new("anthropic", "credential"), + file_id, + Some(lash_core::MediaType::parse(mime).unwrap()), + )]; + + let body = provider.build_request_body(&req).expect("body"); + let block = &body["messages"][0]["content"][0]; + assert_eq!(block["type"], expected_block_type); + assert_eq!(block["source"], json!({"type": "file", "file_id": file_id})); + } + } + + #[test] + fn provider_file_without_media_type_is_rejected_before_transport() { + let provider = AnthropicProvider::new("key"); + let mut req = request(vec![LlmMessage::new( + LlmRole::User, + vec![LlmContentBlock::Attachment { attachment_idx: 0 }], + )]); + req.attachments = vec![AttachmentSource::provider_file( + lash_core::ProviderFileScope::new("anthropic", "credential"), + "file-without-mime", + None, + )]; + + let err = provider + .build_request_body(&req) + .expect_err("missing MIME should be rejected before transport"); + + assert_eq!(err.kind, lash_core::ProviderFailureKind::Validation); + assert_eq!( + err.code.as_deref(), + Some("provider_file_media_type_required") + ); + assert_eq!( + err.message, + "Anthropic Messages requires the media type for provider file ids in order to choose the image/document modality; supply `media_type` on `ProviderFile`" + ); + } + #[test] fn unsupported_image_mime_is_rejected_at_request_boundary() { let provider = AnthropicProvider::new("key"); diff --git a/crates/lash-provider-anthropic/src/request.rs b/crates/lash-provider-anthropic/src/request.rs index f299ca87..66549018 100644 --- a/crates/lash-provider-anthropic/src/request.rs +++ b/crates/lash-provider-anthropic/src/request.rs @@ -16,12 +16,6 @@ impl AnthropicProvider { fn attachment_block_value(req: &LlmRequest, attachment_idx: usize) -> Option { let source = req.attachments.get(attachment_idx)?; - if let AttachmentSource::ProviderFile { id, .. } = source { - return Some(json!({ - "type": "document", - "source": {"type": "file", "file_id": id}, - })); - } let media_type = source.media_type()?; let block_type = if media_type.is_image() { "image" @@ -39,7 +33,9 @@ impl AnthropicProvider { "data": data, }) } - AttachmentSource::ProviderFile { .. } => unreachable!(), + AttachmentSource::ProviderFile { id, .. } => { + json!({"type": "file", "file_id": id}) + } }; Some(json!({"type": block_type, "source": wire_source})) } @@ -350,6 +346,20 @@ impl AnthropicProvider { pub(crate) fn build_request_body(&self, req: &LlmRequest) -> Result { for source in &req.attachments { + if matches!( + source, + AttachmentSource::ProviderFile { + provider_scope, + media_type: None, + .. + } if provider_scope.provider.eq_ignore_ascii_case("anthropic") + ) { + return Err(LlmTransportError::new( + "Anthropic Messages requires the media type for provider file ids in order to choose the image/document modality; supply `media_type` on `ProviderFile`", + ) + .with_kind(ProviderFailureKind::Validation) + .with_code("provider_file_media_type_required")); + } let supported = match source { AttachmentSource::ProviderFile { provider_scope, .. } => { provider_scope.provider.eq_ignore_ascii_case("anthropic") diff --git a/crates/lash-provider-google/src/lib.rs b/crates/lash-provider-google/src/lib.rs index bc6339e8..ae250b3a 100644 --- a/crates/lash-provider-google/src/lib.rs +++ b/crates/lash-provider-google/src/lib.rs @@ -258,6 +258,28 @@ mod tests { ); } + #[test] + fn google_provider_file_ignores_optional_media_type_hint() { + for media_type in [ + None, + Some(lash_core::MediaType::parse("image/png").unwrap()), + ] { + let attachment = AttachmentSource::provider_file( + lash_core::ProviderFileScope::new("google_oauth", "credential"), + "files/123", + media_type, + ); + let mut req = request(None); + req.attachments = vec![attachment.clone()]; + + GoogleOAuthProvider::validate_attachments(&req).expect("provider file is supported"); + assert_eq!( + GoogleOAuthProvider::inline_attachment_part(&req, &attachment), + json!({"fileData": {"fileUri": "files/123"}}) + ); + } + } + #[test] fn google_rejects_gif_attachment_at_request_boundary() { let mut req = request(None); diff --git a/crates/lash-provider-openai/src/tests/attachment_tests.rs b/crates/lash-provider-openai/src/tests/attachment_tests.rs index db39de96..71d3b81e 100644 --- a/crates/lash-provider-openai/src/tests/attachment_tests.rs +++ b/crates/lash-provider-openai/src/tests/attachment_tests.rs @@ -19,3 +19,29 @@ fn responses_pdf_url_serializes_as_input_file_url() { "https://example.test/report.pdf" ); } + +#[test] +fn responses_provider_file_ignores_optional_media_type_hint() { + let provider = OpenAiProvider::new("key"); + + for media_type in [ + None, + Some(lash_core::MediaType::parse("image/png").unwrap()), + ] { + let mut req = request(vec![LlmMessage::new( + LlmRole::User, + vec![LlmContentBlock::Attachment { attachment_idx: 0 }], + )]); + req.attachments = vec![AttachmentSource::provider_file( + lash_core::ProviderFileScope::new("openai", "credential"), + "file-123", + media_type, + )]; + + let body = provider.build_responses_request_body(&req, false).unwrap(); + assert_eq!( + body["input"][0]["content"][0], + json!({"type": "input_file", "file_id": "file-123"}) + ); + } +} diff --git a/crates/lash-remote-protocol/src/core_conversions/llm.rs b/crates/lash-remote-protocol/src/core_conversions/llm.rs index ec679669..93328a3d 100644 --- a/crates/lash-remote-protocol/src/core_conversions/llm.rs +++ b/crates/lash-remote-protocol/src/core_conversions/llm.rs @@ -651,12 +651,15 @@ impl From for RemoteAttachmentSource { media_type: media_type.to_string(), url, }, - core_llm::AttachmentSource::ProviderFile { provider_scope, id } => { - Self::ProviderFile { - provider_scope: provider_scope.into(), - id, - } - } + core_llm::AttachmentSource::ProviderFile { + provider_scope, + id, + media_type, + } => Self::ProviderFile { + provider_scope: provider_scope.into(), + id, + media_type: media_type.map(|media_type| media_type.to_string()), + }, } } } @@ -684,9 +687,18 @@ impl TryFrom for core_llm::AttachmentSource { RemoteAttachmentSource::ExternalUrl { media_type, url } => { Ok(Self::external_url(parse_media_type(&media_type)?, url)) } - RemoteAttachmentSource::ProviderFile { provider_scope, id } => { - Ok(Self::provider_file(provider_scope.into(), id)) - } + RemoteAttachmentSource::ProviderFile { + provider_scope, + id, + media_type, + } => Ok(Self::provider_file( + provider_scope.into(), + id, + media_type + .as_deref() + .map(parse_media_type) + .transpose()?, + )), } } } diff --git a/crates/lash-remote-protocol/src/core_conversions_tests.rs b/crates/lash-remote-protocol/src/core_conversions_tests.rs index bb3522ab..764ef3e2 100644 --- a/crates/lash-remote-protocol/src/core_conversions_tests.rs +++ b/crates/lash-remote-protocol/src/core_conversions_tests.rs @@ -151,6 +151,33 @@ fn turn_input_rejects_non_remote_safe_fields() { )); } +#[test] +fn provider_file_media_type_round_trips_between_core_and_remote() { + for media_type in [ + None, + Some(lash_core::MediaType::parse("image/png").unwrap()), + ] { + let core = lash_core::AttachmentSource::provider_file( + lash_core::ProviderFileScope::new("anthropic", "credential"), + "file-123", + media_type, + ); + + let remote = RemoteAttachmentSource::from(core.clone()); + remote.validate(0).expect("valid remote attachment"); + let remote_json = serde_json::to_value(&remote).expect("serialize remote attachment"); + assert_eq!( + remote_json + .get("media_type") + .and_then(|value| value.as_str()), + core.media_type().map(lash_core::MediaType::as_str) + ); + let round_trip = + lash_core::AttachmentSource::try_from(remote).expect("core attachment conversion"); + assert_eq!(round_trip, core); + } +} + #[test] fn llm_request_and_response_round_trip_owned_dtos() { let request = core_llm::LlmRequest { diff --git a/crates/lash-remote-protocol/src/llm.rs b/crates/lash-remote-protocol/src/llm.rs index 83ba2e55..f4883913 100644 --- a/crates/lash-remote-protocol/src/llm.rs +++ b/crates/lash-remote-protocol/src/llm.rs @@ -489,6 +489,8 @@ pub enum RemoteAttachmentSource { ProviderFile { provider_scope: RemoteProviderFileScope, id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + media_type: Option, }, } @@ -513,9 +515,22 @@ impl RemoteAttachmentSource { validate_media_type("RemoteAttachmentSource::ExternalUrl", media_type)?; require_non_empty("RemoteAttachmentSource::ExternalUrl", "url", url) } - Self::ProviderFile { provider_scope, id } => { + Self::ProviderFile { + provider_scope, + id, + media_type, + } => { provider_scope.validate()?; - require_non_empty("RemoteAttachmentSource::ProviderFile", "id", id) + require_non_empty("RemoteAttachmentSource::ProviderFile", "id", id)?; + if let Some(media_type) = media_type { + require_non_empty( + "RemoteAttachmentSource::ProviderFile", + "media_type", + media_type, + )?; + validate_media_type("RemoteAttachmentSource::ProviderFile", media_type)?; + } + Ok(()) } } } diff --git a/crates/lash-sansio/src/llm/types.rs b/crates/lash-sansio/src/llm/types.rs index 0fb2365a..b2e4fac0 100644 --- a/crates/lash-sansio/src/llm/types.rs +++ b/crates/lash-sansio/src/llm/types.rs @@ -354,10 +354,12 @@ pub enum AttachmentSource { ProviderFile { provider_scope: ProviderFileScope, id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + media_type: Option, }, } -// Current attachment content carrier; measured 96 B on rustc 1.97.0, +// Current attachment content carrier; measured 104 B on rustc 1.97.0, // x86_64-unknown-linux-gnu (FIG-595). const _: () = assert!(std::mem::size_of::() <= 128); @@ -377,10 +379,15 @@ impl AttachmentSource { } } - pub fn provider_file(provider_scope: ProviderFileScope, id: impl Into) -> Self { + pub fn provider_file( + provider_scope: ProviderFileScope, + id: impl Into, + media_type: Option, + ) -> Self { Self::ProviderFile { provider_scope, id: id.into(), + media_type, } } @@ -390,7 +397,7 @@ impl AttachmentSource { Some(media_type) } Self::Stored { attachment_ref } => Some(&attachment_ref.media_type), - Self::ProviderFile { .. } => None, + Self::ProviderFile { media_type, .. } => media_type.as_ref(), } } @@ -779,3 +786,39 @@ mod attempt_record_tests { assert_eq!(absent.reasoning_output_tokens, None); } } + +#[cfg(test)] +mod attachment_source_tests { + use super::*; + + #[test] + fn provider_file_media_type_is_optional_and_omitted_when_absent() { + let scope = ProviderFileScope::new("anthropic", "credential"); + let without_hint = AttachmentSource::provider_file(scope.clone(), "file-1", None); + let without_hint_json = serde_json::to_value(&without_hint).unwrap(); + assert_eq!( + without_hint_json, + serde_json::json!({ + "source": "provider_file", + "provider_scope": { + "provider": "anthropic", + "credential_scope": "credential" + }, + "id": "file-1" + }) + ); + assert_eq!( + serde_json::from_value::(without_hint_json).unwrap(), + without_hint + ); + + let with_hint = AttachmentSource::provider_file( + scope, + "file-2", + Some(MediaType::parse("image/png").unwrap()), + ); + let with_hint_json = serde_json::to_value(&with_hint).unwrap(); + assert_eq!(with_hint_json["media_type"], "image/png"); + assert_eq!(with_hint.media_type().unwrap().as_str(), "image/png"); + } +}