Apps/ImageApi
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Pass image as additional Insight context

Browse Source
This commit is contained in:
Cameron
2026-01-10 11:30:01 -05:00
parent 084994e0b5
commit b2cc617bc2
9 changed files with 295 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
src/ai/ollama.rs
View File

@@ -11,6 +11,7 @@ pub struct OllamaClient {
pub fallback_url: Option<String>,
pub primary_model: String,
pub fallback_model: Option<String>,
num_ctx: Option<i32>,
}
impl OllamaClient {
@@ -30,9 +31,14 @@ impl OllamaClient {
fallback_url,
primary_model,
fallback_model,
num_ctx: None,
}
}
pub fn set_num_ctx(&mut self, num_ctx: Option<i32>) {
self.num_ctx = num_ctx;
}
/// List available models on an Ollama server
pub async fn list_models(url: &str) -> Result<Vec<String>> {
let client = Client::builder()
@@ -79,12 +85,15 @@ impl OllamaClient {
model: &str,
prompt: &str,
system: Option<&str>,
images: Option<Vec<String>>,
) -> Result<String> {
let request = OllamaRequest {
model: model.to_string(),
prompt: prompt.to_string(),
stream: false,
system: system.map(|s| s.to_string()),
options: self.num_ctx.map(|ctx| OllamaOptions { num_ctx: ctx }),
images,
};
let response = self
@@ -109,12 +118,24 @@ impl OllamaClient {
}
pub async fn generate(&self, prompt: &str, system: Option<&str>) -> Result<String> {
self.generate_with_images(prompt, system, None).await
}
pub async fn generate_with_images(
&self,
prompt: &str,
system: Option<&str>,
images: Option<Vec<String>>,
) -> Result<String> {
log::debug!("=== Ollama Request ===");
log::debug!("Primary model: {}", self.primary_model);
if let Some(sys) = system {
log::debug!("System: {}", sys);
}
log::debug!("Prompt:\n{}", prompt);
if let Some(ref imgs) = images {
log::debug!("Images: {} image(s) included", imgs.len());
}
log::debug!("=====================");
// Try primary server first with primary model
@@ -124,7 +145,13 @@ impl OllamaClient {
self.primary_model
);
let primary_result = self
.try_generate(&self.primary_url, &self.primary_model, prompt, system)
.try_generate(
&self.primary_url,
&self.primary_model,
prompt,
system,
images.clone(),
)
.await;
let raw_response = match primary_result {
@@ -147,7 +174,7 @@ impl OllamaClient {
fallback_model
);
match self
.try_generate(fallback_url, fallback_model, prompt, system)
.try_generate(fallback_url, fallback_model, prompt, system, images.clone())
.await
{
Ok(response) => {
@@ -190,12 +217,30 @@ impl OllamaClient {
date: NaiveDate,
location: Option<&str>,
sms_summary: Option<&str>,
custom_system: Option<&str>,
image_base64: Option<String>,
) -> Result<String> {
let location_str = location.unwrap_or("Unknown location");
let sms_str = sms_summary.unwrap_or("No messages");
let prompt = format!(
r#"Create a short title (maximum 8 words) about this moment:
let prompt = if image_base64.is_some() {
format!(
r#"Create a short title (maximum 8 words) about this moment by analyzing the image and context:
Date: {}
Location: {}
Messages: {}
Analyze the image and use specific details from both the visual content and the context above. If limited information is available, use a simple descriptive title based on what you see.
Return ONLY the title, nothing else."#,
date.format("%B %d, %Y"),
location_str,
sms_str
)
} else {
format!(
r#"Create a short title (maximum 8 words) about this moment:
Date: {}
Location: {}
@@ -204,14 +249,18 @@ Messages: {}
Use specific details from the context above. If no specific details are available, use a simple descriptive title.
Return ONLY the title, nothing else."#,
date.format("%B %d, %Y"),
location_str,
sms_str
);
date.format("%B %d, %Y"),
location_str,
sms_str
)
};
let system = "You are my long term memory assistant. Use only the information provided. Do not invent details.";
let system = custom_system.unwrap_or("You are my long term memory assistant. Use only the information provided. Do not invent details.");
let title = self.generate(&prompt, Some(system)).await?;
let images = image_base64.map(|img| vec![img]);
let title = self
.generate_with_images(&prompt, Some(system), images)
.await?;
Ok(title.trim().trim_matches('"').to_string())
}
@@ -221,26 +270,45 @@ Return ONLY the title, nothing else."#,
date: NaiveDate,
location: Option<&str>,
sms_summary: Option<&str>,
custom_system: Option<&str>,
image_base64: Option<String>,
) -> Result<String> {
let location_str = location.unwrap_or("Unknown");
let sms_str = sms_summary.unwrap_or("No messages");
let prompt = format!(
r#"Write a 1-3 paragraph description of this moment based on the available information:
let prompt = if image_base64.is_some() {
format!(
r#"Write a 1-3 paragraph description of this moment by analyzing the image and the available context:
Date: {}
Location: {}
Messages: {}
Analyze the image and use specific details from both the visual content and the context above. Mention people's names, places, or activities if they appear in either the image or the context. Write in first person as Cameron with the tone of a journal entry. If limited information is available, keep it simple and factual based on what you see and know. If the location is unknown omit it"#,
date.format("%B %d, %Y"),
location_str,
sms_str
)
} else {
format!(
r#"Write a 1-3 paragraph description of this moment based on the available information:
Date: {}
Location: {}
Messages: {}
Use only the specific details provided above. Mention people's names, places, or activities if they appear in the context. Write in first person as Cameron with the tone of a journal entry. If limited information is available, keep it simple and factual. If the location is unknown omit it"#,
date.format("%B %d, %Y"),
location_str,
sms_str
);
date.format("%B %d, %Y"),
location_str,
sms_str
)
};
let system = "You are a memory refreshing assistant who is able to provide insights through analyzing past conversations. Use only the information provided. Do not invent details.";
let system = custom_system.unwrap_or("You are a memory refreshing assistant who is able to provide insights through analyzing past conversations. Use only the information provided. Do not invent details.");
self.generate(&prompt, Some(system)).await
let images = image_base64.map(|img| vec![img]);
self.generate_with_images(&prompt, Some(system), images)
.await
}
/// Generate an embedding vector for text using nomic-embed-text:v1.5
@@ -388,6 +456,15 @@ struct OllamaRequest {
stream: bool,
#[serde(skip_serializing_if = "Option::is_none")]
system: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
options: Option<OllamaOptions>,
#[serde(skip_serializing_if = "Option::is_none")]
images: Option<Vec<String>>,
}
#[derive(Serialize)]
struct OllamaOptions {
num_ctx: i32,
}
#[derive(Deserialize)]