mirror of
https://github.com/mii443/rust-genai.git
synced 2025-09-03 16:09:26 +00:00
. image - change API to have ImageSource containing the data
This commit is contained in:
@ -2,8 +2,8 @@ use crate::adapter::adapters::support::get_api_key;
|
|||||||
use crate::adapter::anthropic::AnthropicStreamer;
|
use crate::adapter::anthropic::AnthropicStreamer;
|
||||||
use crate::adapter::{Adapter, AdapterKind, ServiceType, WebRequestData};
|
use crate::adapter::{Adapter, AdapterKind, ServiceType, WebRequestData};
|
||||||
use crate::chat::{
|
use crate::chat::{
|
||||||
ChatOptionsSet, ChatRequest, ChatResponse, ChatRole, ChatStream, ChatStreamResponse, MessageContent, MetaUsage,
|
ChatOptionsSet, ChatRequest, ChatResponse, ChatRole, ChatStream, ChatStreamResponse, ContentPart, ImageSource,
|
||||||
ToolCall, ContentPart, ImageSource,
|
MessageContent, MetaUsage, ToolCall,
|
||||||
};
|
};
|
||||||
use crate::resolver::{AuthData, Endpoint};
|
use crate::resolver::{AuthData, Endpoint};
|
||||||
use crate::webc::WebResponse;
|
use crate::webc::WebResponse;
|
||||||
@ -239,12 +239,14 @@ impl AnthropicAdapter {
|
|||||||
let content = match msg.content {
|
let content = match msg.content {
|
||||||
MessageContent::Text(content) => json!(content),
|
MessageContent::Text(content) => json!(content),
|
||||||
MessageContent::Parts(parts) => {
|
MessageContent::Parts(parts) => {
|
||||||
json!(parts.iter().map(|part| match part {
|
json!(parts
|
||||||
|
.iter()
|
||||||
|
.map(|part| match part {
|
||||||
ContentPart::Text(text) => json!({"type": "text", "text": text.clone()}),
|
ContentPart::Text(text) => json!({"type": "text", "text": text.clone()}),
|
||||||
ContentPart::Image{content, content_type, source} => {
|
ContentPart::Image { content_type, source } => {
|
||||||
match source {
|
match source {
|
||||||
ImageSource::Url => todo!("Anthropic doesn't support images from URL, need to handle it gracefully"),
|
ImageSource::Url(_) => todo!("Anthropic doesn't support images from URL, need to handle it gracefully"),
|
||||||
ImageSource::Base64 => json!({
|
ImageSource::Base64(content) => json!({
|
||||||
"type": "image",
|
"type": "image",
|
||||||
"source": {
|
"source": {
|
||||||
"type": "base64",
|
"type": "base64",
|
||||||
@ -253,9 +255,10 @@ impl AnthropicAdapter {
|
|||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
},
|
}
|
||||||
}).collect::<Vec<Value>>())
|
})
|
||||||
},
|
.collect::<Vec<Value>>())
|
||||||
|
}
|
||||||
// Use `match` instead of `if let`. This will allow to future-proof this
|
// Use `match` instead of `if let`. This will allow to future-proof this
|
||||||
// implementation in case some new message content types would appear,
|
// implementation in case some new message content types would appear,
|
||||||
// this way library would not compile if not all methods are implemented
|
// this way library would not compile if not all methods are implemented
|
||||||
|
@ -3,7 +3,7 @@ use crate::adapter::gemini::GeminiStreamer;
|
|||||||
use crate::adapter::{Adapter, AdapterKind, ServiceType, WebRequestData};
|
use crate::adapter::{Adapter, AdapterKind, ServiceType, WebRequestData};
|
||||||
use crate::chat::{
|
use crate::chat::{
|
||||||
ChatOptionsSet, ChatRequest, ChatResponse, ChatResponseFormat, ChatRole, ChatStream, ChatStreamResponse,
|
ChatOptionsSet, ChatRequest, ChatResponse, ChatResponseFormat, ChatRole, ChatStream, ChatStreamResponse,
|
||||||
MessageContent, MetaUsage, ContentPart, ImageSource
|
ContentPart, ImageSource, MessageContent, MetaUsage,
|
||||||
};
|
};
|
||||||
use crate::resolver::{AuthData, Endpoint};
|
use crate::resolver::{AuthData, Endpoint};
|
||||||
use crate::webc::{WebResponse, WebStream};
|
use crate::webc::{WebResponse, WebStream};
|
||||||
@ -21,7 +21,7 @@ const MODELS: &[&str] = &[
|
|||||||
"gemini-1.5-flash-8b",
|
"gemini-1.5-flash-8b",
|
||||||
"gemini-1.0-pro",
|
"gemini-1.0-pro",
|
||||||
"gemini-1.5-flash-latest",
|
"gemini-1.5-flash-latest",
|
||||||
"gemini-2.0-flash-exp"
|
"gemini-2.0-flash-exp",
|
||||||
];
|
];
|
||||||
|
|
||||||
// curl \
|
// curl \
|
||||||
@ -225,31 +225,34 @@ impl GeminiAdapter {
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
systems.push(content)
|
systems.push(content)
|
||||||
},
|
}
|
||||||
ChatRole::User => {
|
ChatRole::User => {
|
||||||
let content = match msg.content {
|
let content = match msg.content {
|
||||||
MessageContent::Text(content) => json!([{"text": content}]),
|
MessageContent::Text(content) => json!([{"text": content}]),
|
||||||
MessageContent::Parts(parts) => {
|
MessageContent::Parts(parts) => {
|
||||||
json!(parts.iter().map(|part| match part {
|
json!(parts
|
||||||
|
.iter()
|
||||||
|
.map(|part| match part {
|
||||||
ContentPart::Text(text) => json!({"text": text.clone()}),
|
ContentPart::Text(text) => json!({"text": text.clone()}),
|
||||||
ContentPart::Image{content, content_type, source} => {
|
ContentPart::Image { content_type, source } => {
|
||||||
match source {
|
match source {
|
||||||
ImageSource::Url => json!({
|
ImageSource::Url(url) => json!({
|
||||||
"file_data": {
|
"file_data": {
|
||||||
"mime_type": content_type,
|
"mime_type": content_type,
|
||||||
"file_uri": content
|
"file_uri": url
|
||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
ImageSource::Base64 => json!({
|
ImageSource::Base64(content) => json!({
|
||||||
"inline_data": {
|
"inline_data": {
|
||||||
"mime_type": content_type,
|
"mime_type": content_type,
|
||||||
"data": content
|
"data": content
|
||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
},
|
}
|
||||||
}).collect::<Vec<Value>>())
|
})
|
||||||
},
|
.collect::<Vec<Value>>())
|
||||||
|
}
|
||||||
// Use `match` instead of `if let`. This will allow to future-proof this
|
// Use `match` instead of `if let`. This will allow to future-proof this
|
||||||
// implementation in case some new message content types would appear,
|
// implementation in case some new message content types would appear,
|
||||||
// this way library would not compile if not all methods are implemented
|
// this way library would not compile if not all methods are implemented
|
||||||
@ -260,7 +263,7 @@ impl GeminiAdapter {
|
|||||||
};
|
};
|
||||||
|
|
||||||
contents.push(json!({"role": "user", "parts": content}));
|
contents.push(json!({"role": "user", "parts": content}));
|
||||||
},
|
}
|
||||||
ChatRole::Assistant => {
|
ChatRole::Assistant => {
|
||||||
let MessageContent::Text(content) = msg.content else {
|
let MessageContent::Text(content) = msg.content else {
|
||||||
return Err(Error::MessageContentTypeNotSupported {
|
return Err(Error::MessageContentTypeNotSupported {
|
||||||
@ -269,7 +272,7 @@ impl GeminiAdapter {
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
contents.push(json!({"role": "model", "parts": [{"text": content}]}))
|
contents.push(json!({"role": "model", "parts": [{"text": content}]}))
|
||||||
},
|
}
|
||||||
ChatRole::Tool => {
|
ChatRole::Tool => {
|
||||||
return Err(Error::MessageRoleNotSupported {
|
return Err(Error::MessageRoleNotSupported {
|
||||||
model_iden,
|
model_iden,
|
||||||
|
@ -257,16 +257,12 @@ impl OpenAIAdapter {
|
|||||||
.iter()
|
.iter()
|
||||||
.map(|part| match part {
|
.map(|part| match part {
|
||||||
ContentPart::Text(text) => json!({"type": "text", "text": text.clone()}),
|
ContentPart::Text(text) => json!({"type": "text", "text": text.clone()}),
|
||||||
ContentPart::Image {
|
ContentPart::Image { content_type, source } => {
|
||||||
content,
|
|
||||||
content_type,
|
|
||||||
source,
|
|
||||||
} => {
|
|
||||||
match source {
|
match source {
|
||||||
ImageSource::Url => {
|
ImageSource::Url(url) => {
|
||||||
json!({"type": "image_url", "image_url": {"url": content}})
|
json!({"type": "image_url", "image_url": {"url": url}})
|
||||||
}
|
}
|
||||||
ImageSource::Base64 => {
|
ImageSource::Base64(content) => {
|
||||||
let image_url = format!("data:{content_type};base64,{content}");
|
let image_url = format!("data:{content_type};base64,{content}");
|
||||||
json!({"type": "image_url", "image_url": {"url": image_url}})
|
json!({"type": "image_url", "image_url": {"url": image_url}})
|
||||||
}
|
}
|
||||||
|
@ -113,11 +113,7 @@ impl From<Vec<ContentPart>> for MessageContent {
|
|||||||
#[derive(Debug, Clone, Serialize, Deserialize, From)]
|
#[derive(Debug, Clone, Serialize, Deserialize, From)]
|
||||||
pub enum ContentPart {
|
pub enum ContentPart {
|
||||||
Text(String),
|
Text(String),
|
||||||
Image {
|
Image { content_type: String, source: ImageSource },
|
||||||
content: String,
|
|
||||||
content_type: String,
|
|
||||||
source: ImageSource,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Constructors
|
/// Constructors
|
||||||
@ -126,19 +122,17 @@ impl ContentPart {
|
|||||||
ContentPart::Text(text.into())
|
ContentPart::Text(text.into())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn from_image_b64(content_type: impl Into<String>, content: impl Into<String>) -> ContentPart {
|
pub fn from_image_base64(content_type: impl Into<String>, content: impl Into<String>) -> ContentPart {
|
||||||
ContentPart::Image {
|
ContentPart::Image {
|
||||||
content: content.into(),
|
|
||||||
content_type: content_type.into(),
|
content_type: content_type.into(),
|
||||||
source: ImageSource::Base64,
|
source: ImageSource::Base64(content.into()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn from_image_url(content_type: impl Into<String>, url: impl Into<String>) -> ContentPart {
|
pub fn from_image_url(content_type: impl Into<String>, url: impl Into<String>) -> ContentPart {
|
||||||
ContentPart::Image {
|
ContentPart::Image {
|
||||||
content: url.into(),
|
|
||||||
content_type: content_type.into(),
|
content_type: content_type.into(),
|
||||||
source: ImageSource::Url,
|
source: ImageSource::Url(url.into()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -153,10 +147,11 @@ impl<'a> From<&'a str> for ContentPart {
|
|||||||
|
|
||||||
// endregion: --- Froms
|
// endregion: --- Froms
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, From)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub enum ImageSource {
|
pub enum ImageSource {
|
||||||
Url,
|
Url(String),
|
||||||
Base64, // No `Local` location, this would require handling errors like "file not found" etc.
|
Base64(String),
|
||||||
|
// No `Local` location, this would require handling errors like "file not found" etc.
|
||||||
// Such file can be easily provided by user as Base64, also can implement convenient
|
// Such file can be easily provided by user as Base64, also can implement convenient
|
||||||
// TryFrom<File> to Base64 version. All LLMs accepts local Images only as Base64
|
// TryFrom<File> to Base64 version. All LLMs accepts local Images only as Base64
|
||||||
}
|
}
|
||||||
|
@ -352,7 +352,7 @@ pub async fn common_test_chat_image_b64_ok(model: &str) -> Result<()> {
|
|||||||
// This is similar to sending initial system chat messages (which will be cumulative with system chat messages)
|
// This is similar to sending initial system chat messages (which will be cumulative with system chat messages)
|
||||||
chat_req = chat_req.append_message(ChatMessage::user(vec![
|
chat_req = chat_req.append_message(ChatMessage::user(vec![
|
||||||
ContentPart::from_text("What is in this picture?"),
|
ContentPart::from_text("What is in this picture?"),
|
||||||
ContentPart::from_image_b64("image/jpeg", get_b64_duck()?),
|
ContentPart::from_image_base64("image/jpeg", get_b64_duck()?),
|
||||||
]));
|
]));
|
||||||
let chat_res = client.exec_chat(model, chat_req, None).await?;
|
let chat_res = client.exec_chat(model, chat_req, None).await?;
|
||||||
|
|
||||||
|
@ -9,21 +9,6 @@ pub fn seed_chat_req_simple() -> ChatRequest {
|
|||||||
])
|
])
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn seed_chat_req_with_image() -> ChatRequest {
|
|
||||||
ChatRequest::new(vec![
|
|
||||||
// -- Messages (deactivate to see the differences)
|
|
||||||
ChatMessage::system("Answer in one sentence"),
|
|
||||||
ChatMessage::user(vec![
|
|
||||||
ContentPart::from("What is in this image?"),
|
|
||||||
ContentPart::Image {
|
|
||||||
content: "BASE64 ENCODED IMAGE".to_string(),
|
|
||||||
content_type:"image/png".to_string(),
|
|
||||||
source: ImageSource::Base64,
|
|
||||||
}
|
|
||||||
]),
|
|
||||||
])
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn seed_chat_req_tool_simple() -> ChatRequest {
|
pub fn seed_chat_req_tool_simple() -> ChatRequest {
|
||||||
ChatRequest::new(vec![
|
ChatRequest::new(vec![
|
||||||
// -- Messages (deactivate to see the differences)
|
// -- Messages (deactivate to see the differences)
|
||||||
|
Reference in New Issue
Block a user