mii/rust-genai
1
0
Fork 0
mirror of https://github.com/mii443/rust-genai.git synced 2025-08-22 16:25:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

. image - change API to have ImageSource containing the data

Browse Source
This commit is contained in:
Jeremy Chone
2025-01-01 11:41:37 -08:00
parent 27013c4511
commit 9cbd02b480
6 changed files with 56 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
src/adapter/adapters/anthropic/adapter_impl.rs
View File

@ -2,8 +2,8 @@ use crate::adapter::adapters::support::get_api_key;
use crate::adapter::anthropic::AnthropicStreamer;
use crate::adapter::{Adapter, AdapterKind, ServiceType, WebRequestData};
use crate::chat::{
ChatOptionsSet, ChatRequest, ChatResponse, ChatRole, ChatStream, ChatStreamResponse, MessageContent, MetaUsage,
ToolCall, ContentPart, ImageSource,
ChatOptionsSet, ChatRequest, ChatResponse, ChatRole, ChatStream, ChatStreamResponse, ContentPart, ImageSource,
MessageContent, MetaUsage, ToolCall,
};
use crate::resolver::{AuthData, Endpoint};
use crate::webc::WebResponse;
@ -239,12 +239,14 @@ impl AnthropicAdapter {
let content = match msg.content {
MessageContent::Text(content) => json!(content),
MessageContent::Parts(parts) => {
json!(parts.iter().map(|part| match part {
ContentPart::Text(text) => json!({"type": "text", "text": text.clone()}),
ContentPart::Image{content, content_type, source} => {
match source {
ImageSource::Url => todo!("Anthropic doesn't support images from URL, need to handle it gracefully"),
ImageSource::Base64 => json!({
json!(parts
.iter()
.map(|part| match part {
ContentPart::Text(text) => json!({"type": "text", "text": text.clone()}),
ContentPart::Image { content_type, source } => {
match source {
ImageSource::Url(_) => todo!("Anthropic doesn't support images from URL, need to handle it gracefully"),
ImageSource::Base64(content) => json!({
"type": "image",
"source": {
"type": "base64",
@ -253,9 +255,10 @@ impl AnthropicAdapter {
},
}),
}
},
}).collect::<Vec<Value>>())
},
}
})
.collect::<Vec<Value>>())
}
// Use `match` instead of `if let`. This will allow to future-proof this
// implementation in case some new message content types would appear,
// this way library would not compile if not all methods are implemented

51
src/adapter/adapters/gemini/adapter_impl.rs
View File

@ -3,7 +3,7 @@ use crate::adapter::gemini::GeminiStreamer;
use crate::adapter::{Adapter, AdapterKind, ServiceType, WebRequestData};
use crate::chat::{
ChatOptionsSet, ChatRequest, ChatResponse, ChatResponseFormat, ChatRole, ChatStream, ChatStreamResponse,
MessageContent, MetaUsage, ContentPart, ImageSource
ContentPart, ImageSource, MessageContent, MetaUsage,
};
use crate::resolver::{AuthData, Endpoint};
use crate::webc::{WebResponse, WebStream};
@ -21,7 +21,7 @@ const MODELS: &[&str] = &[
"gemini-1.5-flash-8b",
"gemini-1.0-pro",
"gemini-1.5-flash-latest",
"gemini-2.0-flash-exp"
"gemini-2.0-flash-exp",
];
// curl \
@ -225,31 +225,34 @@ impl GeminiAdapter {
});
};
systems.push(content)
},
}
ChatRole::User => {
let content = match msg.content {
MessageContent::Text(content) => json!([{"text": content}]),
MessageContent::Parts(parts) => {
json!(parts.iter().map(|part| match part {
ContentPart::Text(text) => json!({"text": text.clone()}),
ContentPart::Image{content, content_type, source} => {
match source {
ImageSource::Url => json!({
"file_data": {
"mime_type": content_type,
"file_uri": content
}
}),
ImageSource::Base64 => json!({
"inline_data": {
"mime_type": content_type,
"data": content
}
}),
json!(parts
.iter()
.map(|part| match part {
ContentPart::Text(text) => json!({"text": text.clone()}),
ContentPart::Image { content_type, source } => {
match source {
ImageSource::Url(url) => json!({
"file_data": {
"mime_type": content_type,
"file_uri": url
}
}),
ImageSource::Base64(content) => json!({
"inline_data": {
"mime_type": content_type,
"data": content
}
}),
}
}
},
}).collect::<Vec<Value>>())
},
})
.collect::<Vec<Value>>())
}
// Use `match` instead of `if let`. This will allow to future-proof this
// implementation in case some new message content types would appear,
// this way library would not compile if not all methods are implemented
@ -260,7 +263,7 @@ impl GeminiAdapter {
};
contents.push(json!({"role": "user", "parts": content}));
},
}
ChatRole::Assistant => {
let MessageContent::Text(content) = msg.content else {
return Err(Error::MessageContentTypeNotSupported {
@ -269,7 +272,7 @@ impl GeminiAdapter {
});
};
contents.push(json!({"role": "model", "parts": [{"text": content}]}))
},
}
ChatRole::Tool => {
return Err(Error::MessageRoleNotSupported {
model_iden,

12
src/adapter/adapters/openai/adapter_impl.rs
View File

@ -257,16 +257,12 @@ impl OpenAIAdapter {
.iter()
.map(|part| match part {
ContentPart::Text(text) => json!({"type": "text", "text": text.clone()}),
ContentPart::Image {
content,
content_type,
source,
} => {
ContentPart::Image { content_type, source } => {
match source {
ImageSource::Url => {
json!({"type": "image_url", "image_url": {"url": content}})
ImageSource::Url(url) => {
json!({"type": "image_url", "image_url": {"url": url}})
}
ImageSource::Base64 => {
ImageSource::Base64(content) => {
let image_url = format!("data:{content_type};base64,{content}");
json!({"type": "image_url", "image_url": {"url": image_url}})
}

25
src/chat/message_content.rs
View File

@ -113,11 +113,7 @@ impl From<Vec<ContentPart>> for MessageContent {
#[derive(Debug, Clone, Serialize, Deserialize, From)]
pub enum ContentPart {
Text(String),
Image {
content: String,
content_type: String,
source: ImageSource,
},
Image { content_type: String, source: ImageSource },
}
/// Constructors
@ -126,19 +122,17 @@ impl ContentPart {
ContentPart::Text(text.into())
}
pub fn from_image_b64(content_type: impl Into<String>, content: impl Into<String>) -> ContentPart {
pub fn from_image_base64(content_type: impl Into<String>, content: impl Into<String>) -> ContentPart {
ContentPart::Image {
content: content.into(),
content_type: content_type.into(),
source: ImageSource::Base64,
source: ImageSource::Base64(content.into()),
}
}
pub fn from_image_url(content_type: impl Into<String>, url: impl Into<String>) -> ContentPart {
ContentPart::Image {
content: url.into(),
content_type: content_type.into(),
source: ImageSource::Url,
source: ImageSource::Url(url.into()),
}
}
}
@ -153,10 +147,11 @@ impl<'a> From<&'a str> for ContentPart {
// endregion: --- Froms
#[derive(Debug, Clone, Serialize, Deserialize, From)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ImageSource {
Url,
Base64, // No `Local` location, this would require handling errors like "file not found" etc.
// Such file can be easily provided by user as Base64, also can implement convenient
// TryFrom<File> to Base64 version. All LLMs accepts local Images only as Base64
Url(String),
Base64(String),
// No `Local` location, this would require handling errors like "file not found" etc.
// Such file can be easily provided by user as Base64, also can implement convenient
// TryFrom<File> to Base64 version. All LLMs accepts local Images only as Base64
}

2
tests/support/common_tests.rs
View File

@ -352,7 +352,7 @@ pub async fn common_test_chat_image_b64_ok(model: &str) -> Result<()> {
// This is similar to sending initial system chat messages (which will be cumulative with system chat messages)
chat_req = chat_req.append_message(ChatMessage::user(vec![
ContentPart::from_text("What is in this picture?"),
ContentPart::from_image_b64("image/jpeg", get_b64_duck()?),
ContentPart::from_image_base64("image/jpeg", get_b64_duck()?),
]));
let chat_res = client.exec_chat(model, chat_req, None).await?;

15
tests/support/seeders.rs
View File

@ -9,21 +9,6 @@ pub fn seed_chat_req_simple() -> ChatRequest {
])
}
pub fn seed_chat_req_with_image() -> ChatRequest {
ChatRequest::new(vec![
// -- Messages (deactivate to see the differences)
ChatMessage::system("Answer in one sentence"),
ChatMessage::user(vec![
ContentPart::from("What is in this image?"),
ContentPart::Image {
content: "BASE64 ENCODED IMAGE".to_string(),
content_type:"image/png".to_string(),
source: ImageSource::Base64,
}
]),
])
}
pub fn seed_chat_req_tool_simple() -> ChatRequest {
ChatRequest::new(vec![
// -- Messages (deactivate to see the differences)