. image - change API to have ImageSource containing the data

This commit is contained in:
Jeremy Chone
2025-01-01 11:41:37 -08:00
parent 27013c4511
commit 9cbd02b480
6 changed files with 56 additions and 74 deletions

View File

@ -2,8 +2,8 @@ use crate::adapter::adapters::support::get_api_key;
use crate::adapter::anthropic::AnthropicStreamer; use crate::adapter::anthropic::AnthropicStreamer;
use crate::adapter::{Adapter, AdapterKind, ServiceType, WebRequestData}; use crate::adapter::{Adapter, AdapterKind, ServiceType, WebRequestData};
use crate::chat::{ use crate::chat::{
ChatOptionsSet, ChatRequest, ChatResponse, ChatRole, ChatStream, ChatStreamResponse, MessageContent, MetaUsage, ChatOptionsSet, ChatRequest, ChatResponse, ChatRole, ChatStream, ChatStreamResponse, ContentPart, ImageSource,
ToolCall, ContentPart, ImageSource, MessageContent, MetaUsage, ToolCall,
}; };
use crate::resolver::{AuthData, Endpoint}; use crate::resolver::{AuthData, Endpoint};
use crate::webc::WebResponse; use crate::webc::WebResponse;
@ -239,12 +239,14 @@ impl AnthropicAdapter {
let content = match msg.content { let content = match msg.content {
MessageContent::Text(content) => json!(content), MessageContent::Text(content) => json!(content),
MessageContent::Parts(parts) => { MessageContent::Parts(parts) => {
json!(parts.iter().map(|part| match part { json!(parts
.iter()
.map(|part| match part {
ContentPart::Text(text) => json!({"type": "text", "text": text.clone()}), ContentPart::Text(text) => json!({"type": "text", "text": text.clone()}),
ContentPart::Image{content, content_type, source} => { ContentPart::Image { content_type, source } => {
match source { match source {
ImageSource::Url => todo!("Anthropic doesn't support images from URL, need to handle it gracefully"), ImageSource::Url(_) => todo!("Anthropic doesn't support images from URL, need to handle it gracefully"),
ImageSource::Base64 => json!({ ImageSource::Base64(content) => json!({
"type": "image", "type": "image",
"source": { "source": {
"type": "base64", "type": "base64",
@ -253,9 +255,10 @@ impl AnthropicAdapter {
}, },
}), }),
} }
}, }
}).collect::<Vec<Value>>()) })
}, .collect::<Vec<Value>>())
}
// Use `match` instead of `if let`. This will allow to future-proof this // Use `match` instead of `if let`. This will allow to future-proof this
// implementation in case some new message content types would appear, // implementation in case some new message content types would appear,
// this way library would not compile if not all methods are implemented // this way library would not compile if not all methods are implemented

View File

@ -3,7 +3,7 @@ use crate::adapter::gemini::GeminiStreamer;
use crate::adapter::{Adapter, AdapterKind, ServiceType, WebRequestData}; use crate::adapter::{Adapter, AdapterKind, ServiceType, WebRequestData};
use crate::chat::{ use crate::chat::{
ChatOptionsSet, ChatRequest, ChatResponse, ChatResponseFormat, ChatRole, ChatStream, ChatStreamResponse, ChatOptionsSet, ChatRequest, ChatResponse, ChatResponseFormat, ChatRole, ChatStream, ChatStreamResponse,
MessageContent, MetaUsage, ContentPart, ImageSource ContentPart, ImageSource, MessageContent, MetaUsage,
}; };
use crate::resolver::{AuthData, Endpoint}; use crate::resolver::{AuthData, Endpoint};
use crate::webc::{WebResponse, WebStream}; use crate::webc::{WebResponse, WebStream};
@ -21,7 +21,7 @@ const MODELS: &[&str] = &[
"gemini-1.5-flash-8b", "gemini-1.5-flash-8b",
"gemini-1.0-pro", "gemini-1.0-pro",
"gemini-1.5-flash-latest", "gemini-1.5-flash-latest",
"gemini-2.0-flash-exp" "gemini-2.0-flash-exp",
]; ];
// curl \ // curl \
@ -225,31 +225,34 @@ impl GeminiAdapter {
}); });
}; };
systems.push(content) systems.push(content)
}, }
ChatRole::User => { ChatRole::User => {
let content = match msg.content { let content = match msg.content {
MessageContent::Text(content) => json!([{"text": content}]), MessageContent::Text(content) => json!([{"text": content}]),
MessageContent::Parts(parts) => { MessageContent::Parts(parts) => {
json!(parts.iter().map(|part| match part { json!(parts
.iter()
.map(|part| match part {
ContentPart::Text(text) => json!({"text": text.clone()}), ContentPart::Text(text) => json!({"text": text.clone()}),
ContentPart::Image{content, content_type, source} => { ContentPart::Image { content_type, source } => {
match source { match source {
ImageSource::Url => json!({ ImageSource::Url(url) => json!({
"file_data": { "file_data": {
"mime_type": content_type, "mime_type": content_type,
"file_uri": content "file_uri": url
} }
}), }),
ImageSource::Base64 => json!({ ImageSource::Base64(content) => json!({
"inline_data": { "inline_data": {
"mime_type": content_type, "mime_type": content_type,
"data": content "data": content
} }
}), }),
} }
}, }
}).collect::<Vec<Value>>()) })
}, .collect::<Vec<Value>>())
}
// Use `match` instead of `if let`. This will allow to future-proof this // Use `match` instead of `if let`. This will allow to future-proof this
// implementation in case some new message content types would appear, // implementation in case some new message content types would appear,
// this way library would not compile if not all methods are implemented // this way library would not compile if not all methods are implemented
@ -260,7 +263,7 @@ impl GeminiAdapter {
}; };
contents.push(json!({"role": "user", "parts": content})); contents.push(json!({"role": "user", "parts": content}));
}, }
ChatRole::Assistant => { ChatRole::Assistant => {
let MessageContent::Text(content) = msg.content else { let MessageContent::Text(content) = msg.content else {
return Err(Error::MessageContentTypeNotSupported { return Err(Error::MessageContentTypeNotSupported {
@ -269,7 +272,7 @@ impl GeminiAdapter {
}); });
}; };
contents.push(json!({"role": "model", "parts": [{"text": content}]})) contents.push(json!({"role": "model", "parts": [{"text": content}]}))
}, }
ChatRole::Tool => { ChatRole::Tool => {
return Err(Error::MessageRoleNotSupported { return Err(Error::MessageRoleNotSupported {
model_iden, model_iden,

View File

@ -257,16 +257,12 @@ impl OpenAIAdapter {
.iter() .iter()
.map(|part| match part { .map(|part| match part {
ContentPart::Text(text) => json!({"type": "text", "text": text.clone()}), ContentPart::Text(text) => json!({"type": "text", "text": text.clone()}),
ContentPart::Image { ContentPart::Image { content_type, source } => {
content,
content_type,
source,
} => {
match source { match source {
ImageSource::Url => { ImageSource::Url(url) => {
json!({"type": "image_url", "image_url": {"url": content}}) json!({"type": "image_url", "image_url": {"url": url}})
} }
ImageSource::Base64 => { ImageSource::Base64(content) => {
let image_url = format!("data:{content_type};base64,{content}"); let image_url = format!("data:{content_type};base64,{content}");
json!({"type": "image_url", "image_url": {"url": image_url}}) json!({"type": "image_url", "image_url": {"url": image_url}})
} }

View File

@ -113,11 +113,7 @@ impl From<Vec<ContentPart>> for MessageContent {
#[derive(Debug, Clone, Serialize, Deserialize, From)] #[derive(Debug, Clone, Serialize, Deserialize, From)]
pub enum ContentPart { pub enum ContentPart {
Text(String), Text(String),
Image { Image { content_type: String, source: ImageSource },
content: String,
content_type: String,
source: ImageSource,
},
} }
/// Constructors /// Constructors
@ -126,19 +122,17 @@ impl ContentPart {
ContentPart::Text(text.into()) ContentPart::Text(text.into())
} }
pub fn from_image_b64(content_type: impl Into<String>, content: impl Into<String>) -> ContentPart { pub fn from_image_base64(content_type: impl Into<String>, content: impl Into<String>) -> ContentPart {
ContentPart::Image { ContentPart::Image {
content: content.into(),
content_type: content_type.into(), content_type: content_type.into(),
source: ImageSource::Base64, source: ImageSource::Base64(content.into()),
} }
} }
pub fn from_image_url(content_type: impl Into<String>, url: impl Into<String>) -> ContentPart { pub fn from_image_url(content_type: impl Into<String>, url: impl Into<String>) -> ContentPart {
ContentPart::Image { ContentPart::Image {
content: url.into(),
content_type: content_type.into(), content_type: content_type.into(),
source: ImageSource::Url, source: ImageSource::Url(url.into()),
} }
} }
} }
@ -153,10 +147,11 @@ impl<'a> From<&'a str> for ContentPart {
// endregion: --- Froms // endregion: --- Froms
#[derive(Debug, Clone, Serialize, Deserialize, From)] #[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ImageSource { pub enum ImageSource {
Url, Url(String),
Base64, // No `Local` location, this would require handling errors like "file not found" etc. Base64(String),
// No `Local` location, this would require handling errors like "file not found" etc.
// Such file can be easily provided by user as Base64, also can implement convenient // Such file can be easily provided by user as Base64, also can implement convenient
// TryFrom<File> to Base64 version. All LLMs accepts local Images only as Base64 // TryFrom<File> to Base64 version. All LLMs accepts local Images only as Base64
} }

View File

@ -352,7 +352,7 @@ pub async fn common_test_chat_image_b64_ok(model: &str) -> Result<()> {
// This is similar to sending initial system chat messages (which will be cumulative with system chat messages) // This is similar to sending initial system chat messages (which will be cumulative with system chat messages)
chat_req = chat_req.append_message(ChatMessage::user(vec![ chat_req = chat_req.append_message(ChatMessage::user(vec![
ContentPart::from_text("What is in this picture?"), ContentPart::from_text("What is in this picture?"),
ContentPart::from_image_b64("image/jpeg", get_b64_duck()?), ContentPart::from_image_base64("image/jpeg", get_b64_duck()?),
])); ]));
let chat_res = client.exec_chat(model, chat_req, None).await?; let chat_res = client.exec_chat(model, chat_req, None).await?;

View File

@ -9,21 +9,6 @@ pub fn seed_chat_req_simple() -> ChatRequest {
]) ])
} }
pub fn seed_chat_req_with_image() -> ChatRequest {
ChatRequest::new(vec![
// -- Messages (deactivate to see the differences)
ChatMessage::system("Answer in one sentence"),
ChatMessage::user(vec![
ContentPart::from("What is in this image?"),
ContentPart::Image {
content: "BASE64 ENCODED IMAGE".to_string(),
content_type:"image/png".to_string(),
source: ImageSource::Base64,
}
]),
])
}
pub fn seed_chat_req_tool_simple() -> ChatRequest { pub fn seed_chat_req_tool_simple() -> ChatRequest {
ChatRequest::new(vec![ ChatRequest::new(vec![
// -- Messages (deactivate to see the differences) // -- Messages (deactivate to see the differences)