Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions crates/lash-core/src/llm/transport.rs
Original file line number Diff line number Diff line change
Expand Up @@ -85,10 +85,20 @@ pub fn unsupported_attachment_capability(
accepted_by.join(", ")
};
let message = match source {
AttachmentSource::ProviderFile { provider_scope, .. } => format!(
"{provider} cannot materialize attachment source `provider_file` scoped to provider `{}`; the source carries no caller MIME; providers accepting this source: {accepted}",
provider_scope.provider
),
AttachmentSource::ProviderFile {
provider_scope,
media_type,
..
} => match media_type {
Some(media_type) => format!(
"{provider} cannot materialize attachment MIME `{media_type}` from source `provider_file` scoped to provider `{}`; providers accepting this source: {accepted}",
provider_scope.provider
),
None => format!(
"{provider} cannot materialize attachment source `provider_file` scoped to provider `{}`; the source carries no caller MIME; providers accepting this source: {accepted}",
provider_scope.provider
),
},
source => {
let media_type = source
.media_type()
Expand Down Expand Up @@ -176,6 +186,7 @@ mod tests {
let google_file = AttachmentSource::provider_file(
ProviderFileScope::new("gemini", "credential"),
"file-id",
None,
);
assert_eq!(
known_attachment_acceptors(&google_file),
Expand Down
53 changes: 53 additions & 0 deletions crates/lash-provider-anthropic/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,59 @@ mod tests {
assert_eq!(block["source"]["url"], "https://example.test/report.pdf");
}

#[test]
fn provider_file_uses_media_type_to_select_image_or_document_block() {
let provider = AnthropicProvider::new("key");

for (mime, expected_block_type, file_id) in [
("image/png", "image", "file-image"),
("application/pdf", "document", "file-document"),
] {
let mut req = request(vec![LlmMessage::new(
LlmRole::User,
vec![LlmContentBlock::Attachment { attachment_idx: 0 }],
)]);
req.attachments = vec![AttachmentSource::provider_file(
lash_core::ProviderFileScope::new("anthropic", "credential"),
file_id,
Some(lash_core::MediaType::parse(mime).unwrap()),
)];

let body = provider.build_request_body(&req).expect("body");
let block = &body["messages"][0]["content"][0];
assert_eq!(block["type"], expected_block_type);
assert_eq!(block["source"], json!({"type": "file", "file_id": file_id}));
}
}

#[test]
fn provider_file_without_media_type_is_rejected_before_transport() {
let provider = AnthropicProvider::new("key");
let mut req = request(vec![LlmMessage::new(
LlmRole::User,
vec![LlmContentBlock::Attachment { attachment_idx: 0 }],
)]);
req.attachments = vec![AttachmentSource::provider_file(
lash_core::ProviderFileScope::new("anthropic", "credential"),
"file-without-mime",
None,
)];

let err = provider
.build_request_body(&req)
.expect_err("missing MIME should be rejected before transport");

assert_eq!(err.kind, lash_core::ProviderFailureKind::Validation);
assert_eq!(
err.code.as_deref(),
Some("provider_file_media_type_required")
);
assert_eq!(
err.message,
"Anthropic Messages requires the media type for provider file ids in order to choose the image/document modality; supply `media_type` on `ProviderFile`"
);
}

#[test]
fn unsupported_image_mime_is_rejected_at_request_boundary() {
let provider = AnthropicProvider::new("key");
Expand Down
24 changes: 17 additions & 7 deletions crates/lash-provider-anthropic/src/request.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,6 @@ impl AnthropicProvider {

fn attachment_block_value(req: &LlmRequest, attachment_idx: usize) -> Option<Value> {
let source = req.attachments.get(attachment_idx)?;
if let AttachmentSource::ProviderFile { id, .. } = source {
return Some(json!({
"type": "document",
"source": {"type": "file", "file_id": id},
}));
}
let media_type = source.media_type()?;
let block_type = if media_type.is_image() {
"image"
Expand All @@ -39,7 +33,9 @@ impl AnthropicProvider {
"data": data,
})
}
AttachmentSource::ProviderFile { .. } => unreachable!(),
AttachmentSource::ProviderFile { id, .. } => {
json!({"type": "file", "file_id": id})
}
};
Some(json!({"type": block_type, "source": wire_source}))
}
Expand Down Expand Up @@ -350,6 +346,20 @@ impl AnthropicProvider {

pub(crate) fn build_request_body(&self, req: &LlmRequest) -> Result<Value, LlmTransportError> {
for source in &req.attachments {
if matches!(
source,
AttachmentSource::ProviderFile {
provider_scope,
media_type: None,
..
} if provider_scope.provider.eq_ignore_ascii_case("anthropic")
) {
return Err(LlmTransportError::new(
"Anthropic Messages requires the media type for provider file ids in order to choose the image/document modality; supply `media_type` on `ProviderFile`",
)
.with_kind(ProviderFailureKind::Validation)
.with_code("provider_file_media_type_required"));
}
let supported = match source {
AttachmentSource::ProviderFile { provider_scope, .. } => {
provider_scope.provider.eq_ignore_ascii_case("anthropic")
Expand Down
22 changes: 22 additions & 0 deletions crates/lash-provider-google/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,28 @@ mod tests {
);
}

#[test]
fn google_provider_file_ignores_optional_media_type_hint() {
for media_type in [
None,
Some(lash_core::MediaType::parse("image/png").unwrap()),
] {
let attachment = AttachmentSource::provider_file(
lash_core::ProviderFileScope::new("google_oauth", "credential"),
"files/123",
media_type,
);
let mut req = request(None);
req.attachments = vec![attachment.clone()];

GoogleOAuthProvider::validate_attachments(&req).expect("provider file is supported");
assert_eq!(
GoogleOAuthProvider::inline_attachment_part(&req, &attachment),
json!({"fileData": {"fileUri": "files/123"}})
);
}
}

#[test]
fn google_rejects_gif_attachment_at_request_boundary() {
let mut req = request(None);
Expand Down
26 changes: 26 additions & 0 deletions crates/lash-provider-openai/src/tests/attachment_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,29 @@ fn responses_pdf_url_serializes_as_input_file_url() {
"https://example.test/report.pdf"
);
}

#[test]
fn responses_provider_file_ignores_optional_media_type_hint() {
let provider = OpenAiProvider::new("key");

for media_type in [
None,
Some(lash_core::MediaType::parse("image/png").unwrap()),
] {
let mut req = request(vec![LlmMessage::new(
LlmRole::User,
vec![LlmContentBlock::Attachment { attachment_idx: 0 }],
)]);
req.attachments = vec![AttachmentSource::provider_file(
lash_core::ProviderFileScope::new("openai", "credential"),
"file-123",
media_type,
)];

let body = provider.build_responses_request_body(&req, false).unwrap();
assert_eq!(
body["input"][0]["content"][0],
json!({"type": "input_file", "file_id": "file-123"})
);
}
}
30 changes: 21 additions & 9 deletions crates/lash-remote-protocol/src/core_conversions/llm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -651,12 +651,15 @@ impl From<core_llm::AttachmentSource> for RemoteAttachmentSource {
media_type: media_type.to_string(),
url,
},
core_llm::AttachmentSource::ProviderFile { provider_scope, id } => {
Self::ProviderFile {
provider_scope: provider_scope.into(),
id,
}
}
core_llm::AttachmentSource::ProviderFile {
provider_scope,
id,
media_type,
} => Self::ProviderFile {
provider_scope: provider_scope.into(),
id,
media_type: media_type.map(|media_type| media_type.to_string()),
},
}
}
}
Expand Down Expand Up @@ -684,9 +687,18 @@ impl TryFrom<RemoteAttachmentSource> for core_llm::AttachmentSource {
RemoteAttachmentSource::ExternalUrl { media_type, url } => {
Ok(Self::external_url(parse_media_type(&media_type)?, url))
}
RemoteAttachmentSource::ProviderFile { provider_scope, id } => {
Ok(Self::provider_file(provider_scope.into(), id))
}
RemoteAttachmentSource::ProviderFile {
provider_scope,
id,
media_type,
} => Ok(Self::provider_file(
provider_scope.into(),
id,
media_type
.as_deref()
.map(parse_media_type)
.transpose()?,
)),
}
}
}
Expand Down
27 changes: 27 additions & 0 deletions crates/lash-remote-protocol/src/core_conversions_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,33 @@ fn turn_input_rejects_non_remote_safe_fields() {
));
}

#[test]
fn provider_file_media_type_round_trips_between_core_and_remote() {
for media_type in [
None,
Some(lash_core::MediaType::parse("image/png").unwrap()),
] {
let core = lash_core::AttachmentSource::provider_file(
lash_core::ProviderFileScope::new("anthropic", "credential"),
"file-123",
media_type,
);

let remote = RemoteAttachmentSource::from(core.clone());
remote.validate(0).expect("valid remote attachment");
let remote_json = serde_json::to_value(&remote).expect("serialize remote attachment");
assert_eq!(
remote_json
.get("media_type")
.and_then(|value| value.as_str()),
core.media_type().map(lash_core::MediaType::as_str)
);
let round_trip =
lash_core::AttachmentSource::try_from(remote).expect("core attachment conversion");
assert_eq!(round_trip, core);
}
}

#[test]
fn llm_request_and_response_round_trip_owned_dtos() {
let request = core_llm::LlmRequest {
Expand Down
19 changes: 17 additions & 2 deletions crates/lash-remote-protocol/src/llm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -489,6 +489,8 @@ pub enum RemoteAttachmentSource {
ProviderFile {
provider_scope: RemoteProviderFileScope,
id: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
media_type: Option<String>,
},
}

Expand All @@ -513,9 +515,22 @@ impl RemoteAttachmentSource {
validate_media_type("RemoteAttachmentSource::ExternalUrl", media_type)?;
require_non_empty("RemoteAttachmentSource::ExternalUrl", "url", url)
}
Self::ProviderFile { provider_scope, id } => {
Self::ProviderFile {
provider_scope,
id,
media_type,
} => {
provider_scope.validate()?;
require_non_empty("RemoteAttachmentSource::ProviderFile", "id", id)
require_non_empty("RemoteAttachmentSource::ProviderFile", "id", id)?;
if let Some(media_type) = media_type {
require_non_empty(
"RemoteAttachmentSource::ProviderFile",
"media_type",
media_type,
)?;
validate_media_type("RemoteAttachmentSource::ProviderFile", media_type)?;
}
Ok(())
}
}
}
Expand Down
49 changes: 46 additions & 3 deletions crates/lash-sansio/src/llm/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -354,10 +354,12 @@ pub enum AttachmentSource {
ProviderFile {
provider_scope: ProviderFileScope,
id: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
media_type: Option<MediaType>,
},
}

// Current attachment content carrier; measured 96 B on rustc 1.97.0,
// Current attachment content carrier; measured 104 B on rustc 1.97.0,
// x86_64-unknown-linux-gnu (FIG-595).
const _: () = assert!(std::mem::size_of::<AttachmentSource>() <= 128);

Expand All @@ -377,10 +379,15 @@ impl AttachmentSource {
}
}

pub fn provider_file(provider_scope: ProviderFileScope, id: impl Into<String>) -> Self {
pub fn provider_file(
provider_scope: ProviderFileScope,
id: impl Into<String>,
media_type: Option<MediaType>,
) -> Self {
Self::ProviderFile {
provider_scope,
id: id.into(),
media_type,
}
}

Expand All @@ -390,7 +397,7 @@ impl AttachmentSource {
Some(media_type)
}
Self::Stored { attachment_ref } => Some(&attachment_ref.media_type),
Self::ProviderFile { .. } => None,
Self::ProviderFile { media_type, .. } => media_type.as_ref(),
}
}

Expand Down Expand Up @@ -779,3 +786,39 @@ mod attempt_record_tests {
assert_eq!(absent.reasoning_output_tokens, None);
}
}

#[cfg(test)]
mod attachment_source_tests {
use super::*;

#[test]
fn provider_file_media_type_is_optional_and_omitted_when_absent() {
let scope = ProviderFileScope::new("anthropic", "credential");
let without_hint = AttachmentSource::provider_file(scope.clone(), "file-1", None);
let without_hint_json = serde_json::to_value(&without_hint).unwrap();
assert_eq!(
without_hint_json,
serde_json::json!({
"source": "provider_file",
"provider_scope": {
"provider": "anthropic",
"credential_scope": "credential"
},
"id": "file-1"
})
);
assert_eq!(
serde_json::from_value::<AttachmentSource>(without_hint_json).unwrap(),
without_hint
);

let with_hint = AttachmentSource::provider_file(
scope,
"file-2",
Some(MediaType::parse("image/png").unwrap()),
);
let with_hint_json = serde_json::to_value(&with_hint).unwrap();
assert_eq!(with_hint_json["media_type"], "image/png");
assert_eq!(with_hint.media_type().unwrap().as_str(), "image/png");
}
}
Loading