Apps/ImageApi
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

OpenRouter Support, Insight Chat and User injection #56

Merged
cameron merged 24 commits from 005-llm-client-trait into master 2026-04-26 23:01:35 +00:00
Conversation 0 Commits 24 Files Changed 33 +300 -48
7 changed files with 300 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 13b9d54861 - Show all commits

70
Cargo.lock generated
View File

@@ -908,6 +908,12 @@ version = "0.8.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
[[package]]
name = "crunchy"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
[[package]]
name = "crypto-common"
version = "0.1.6"
@@ -1218,6 +1224,26 @@ version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "fax"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f05de7d48f37cd6730705cbca900770cab77a89f413d23e100ad7fad7795a0ab"
dependencies = [
"fax_derive",
]
[[package]]
name = "fax_derive"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0aca10fb742cb43f9e7bb8467c91aa9bcb8e3ffbc6a6f7389bb93ffc920577d"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "fdeflate"
version = "0.3.7"
@@ -1501,6 +1527,17 @@ dependencies = [
"tracing",
]
[[package]]
name = "half"
version = "2.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
dependencies = [
"cfg-if",
"crunchy",
"zerocopy",
]
[[package]]
name = "hashbrown"
version = "0.14.5"
@@ -1843,11 +1880,14 @@ checksum = "1c6a3ce16143778e24df6f95365f12ed105425b22abefd289dd88a64bab59605"
dependencies = [
"bytemuck",
"byteorder-lite",
"image-webp",
"moxcms",
"num-traits",
"png",
"ravif",
"rayon",
"rgb",
"tiff",
"zune-core",
"zune-jpeg",
]
@@ -1908,6 +1948,16 @@ dependencies = [
"zerocopy",
]
[[package]]
name = "image-webp"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3"
dependencies = [
"byteorder-lite",
"quick-error",
]
[[package]]
name = "imgref"
version = "1.11.0"
@@ -3712,6 +3762,20 @@ dependencies = [
"syn",
]
[[package]]
name = "tiff"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af9605de7fee8d9551863fd692cce7637f548dbd9db9180fcc07ccc6d26c336f"
dependencies = [
"fax",
"flate2",
"half",
"quick-error",
"weezl",
"zune-jpeg",
]
[[package]]
name = "time"
version = "0.3.42"
@@ -4269,6 +4333,12 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "weezl"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88"
[[package]]
name = "winapi"
version = "0.3.9"

2
Cargo.toml
View File

@@ -29,7 +29,7 @@ chrono = "0.4"
clap = { version = "4.5", features = ["derive"] }
dotenv = "0.15"
bcrypt = "0.17.1"
image = { version = "0.25.5", default-features = false, features = ["jpeg", "png", "rayon"] }
image = { version = "0.25.5", default-features = false, features = ["jpeg", "png", "rayon", "webp", "tiff", "avif"] }
infer = "0.16"
walkdir = "2.4.0"
rayon = "1.5"

33
README.md
View File

@@ -14,14 +14,43 @@ Upon first run it will generate thumbnails for all images and videos at `BASE_PA
- **RAG-based Context Retrieval** - Semantic search over daily conversation summaries
- **Automatic Daily Summaries** - LLM-generated summaries of daily conversations with embeddings
## External Dependencies
### ffmpeg (required)
`ffmpeg` must be on `PATH`. It is used for:
- **HLS video streaming** — transcoding/segmenting source videos into `.m3u8` + `.ts` playlists
- **Video thumbnails** — extracting a frame at the 3-second mark
- **Video preview clips** — short looping previews for the Video Wall
- **HEIC / HEIF thumbnails** — decoding Apple's HEIC format (your ffmpeg build must include
`libheif`; most modern builds do)
Builds used in development: the `gyan.dev` full build on Windows, and distro `ffmpeg`
packages on Linux work fine. If HEIC thumbnails silently fail, check
`ffmpeg -formats | grep heif` to confirm HEIF support.
### RAW photo thumbnails (no extra dependency)
RAW formats (ARW, NEF, CR2, CR3, DNG, RAF, ORF, RW2, PEF, SRW, TIFF) are thumbnailed
by reading the embedded JPEG preview from the TIFF IFD1 using `kamadak-exif`. No
external RAW decoder (libraw / dcraw) is required. Files without an embedded preview
fall back to ffmpeg (works for most NEF files), and anything that still can't be
decoded is marked with a `<thumb>.unsupported` sentinel in the thumbnail directory
so we don't retry it every scan. Delete those sentinels to force retries after a
tooling upgrade.
## Environment
There are a handful of required environment variables to have the API run.
They should be defined where the binary is located or above it in an `.env` file.
You must have `ffmpeg` installed for streaming video and generating video thumbnails.
- `DATABASE_URL` is a path or url to a database (currently only SQLite is tested)
- `BASE_PATH` is the root from which you want to serve images and videos
- `THUMBNAILS` is a path where generated thumbnails should be stored
- `THUMBNAILS` is a path where generated thumbnails should be stored. Thumbnails
mirror the source tree under `BASE_PATH` and keep the source's original
extension (e.g. `foo.arw` or `bar.mp4`), though the file contents are always
JPEG bytes — browsers content-sniff. Files that can't be thumbnailed by the
`image` crate, ffmpeg, or an embedded RAW preview get a zero-byte
`<thumb_path>.unsupported` sentinel in this directory so subsequent scans
skip them. Delete the `*.unsupported` files to force retries (for example
after upgrading ffmpeg or adding libheif)
- `VIDEO_PATH` is a path where HLS playlists and video parts should be stored
- `GIFS_DIRECTORY` is a path where generated video GIF thumbnails should be stored
- `BIND_URL` is the url and port to bind to (typically your own IP address)

56
src/exif.rs
View File

@@ -1,5 +1,5 @@
use std::fs::File;
use std::io::BufReader;
use std::io::{BufReader, Read, Seek, SeekFrom};
use std::path::Path;
use anyhow::{Result, anyhow};
@@ -25,6 +25,60 @@ pub struct ExifData {
pub date_taken: Option<i64>,
}
/// TIFF-based RAW formats where `JPEGInterchangeFormat` offsets are
/// absolute file offsets (the file itself is a TIFF container).
fn is_tiff_raw(path: &Path) -> bool {
matches!(
path.extension()
.and_then(|e| e.to_str())
.map(|s| s.to_lowercase())
.as_deref(),
Some(
"tiff" | "tif" | "nef" | "cr2" | "arw" | "dng" | "raf" | "orf" | "rw2" | "pef" | "srw"
)
)
}
/// Returns the bytes of the embedded JPEG thumbnail in a TIFF-based RAW or
/// TIFF file. Used to thumbnail formats whose RAW pixel data can't be decoded
/// by our normal tools (e.g. Sony ARW). Returns `None` if no preview is
/// present, the file isn't a TIFF container, or the data doesn't look like
/// a valid JPEG.
pub fn extract_embedded_jpeg_preview(path: &Path) -> Option<Vec<u8>> {
if !is_tiff_raw(path) {
return None;
}
let file = File::open(path).ok()?;
let mut bufreader = BufReader::new(file);
let exif = Reader::new().read_from_container(&mut bufreader).ok()?;
let offset = exif
.get_field(Tag::JPEGInterchangeFormat, In::THUMBNAIL)?
.value
.get_uint(0)?;
let length = exif
.get_field(Tag::JPEGInterchangeFormatLength, In::THUMBNAIL)?
.value
.get_uint(0)?;
if length == 0 {
return None;
}
let mut file = File::open(path).ok()?;
file.seek(SeekFrom::Start(offset as u64)).ok()?;
let mut buf = vec![0u8; length as usize];
file.read_exact(&mut buf).ok()?;
// JPEG SOI marker sanity check — MakerNote offsets sometimes point at
// TIFF-wrapped previews or other non-JPEG data.
if buf.len() < 2 || buf[0] != 0xFF || buf[1] != 0xD8 {
return None;
}
Some(buf)
}
pub fn supports_exif(path: &Path) -> bool {
if let Some(ext) = path.extension() {
let ext_lower = ext.to_string_lossy().to_lowercase();

15
src/file_types.rs
View File

@@ -3,9 +3,22 @@ use walkdir::DirEntry;
/// Supported image file extensions
pub const IMAGE_EXTENSIONS: &[&str] = &[
"jpg", "jpeg", "png", "webp", "tiff", "tif", "heif", "heic", "avif", "nef",
"jpg", "jpeg", "png", "webp", "tiff", "tif", "heif", "heic", "avif", "nef", "arw",
];
/// Extensions the `image` crate cannot decode — we fall back to ffmpeg to
/// extract an embedded preview or decode the frame.
pub const FFMPEG_THUMBNAIL_EXTENSIONS: &[&str] = &["heif", "heic", "nef", "arw"];
/// Returns true if thumbnail generation should go through ffmpeg instead of
/// the `image` crate (RAW formats, HEIF/HEIC).
pub fn needs_ffmpeg_thumbnail(path: &Path) -> bool {
match path.extension().and_then(|e| e.to_str()) {
Some(ext) => FFMPEG_THUMBNAIL_EXTENSIONS.contains(&ext.to_lowercase().as_str()),
None => false,
}
}
/// Supported video file extensions
pub const VIDEO_EXTENSIONS: &[&str] = &["mp4", "mov", "avi", "mkv"];

121
src/main.rs
View File

@@ -52,7 +52,8 @@ use crate::state::AppState;
use crate::tags::*;
use crate::video::actors::{
GeneratePreviewClipMessage, ProcessMessage, QueueVideosMessage, ScanDirectoryMessage,
VideoPlaylistManager, create_playlist, generate_video_thumbnail,
VideoPlaylistManager, create_playlist, generate_image_thumbnail_ffmpeg,
generate_video_thumbnail,
};
use log::{debug, error, info, trace, warn};
use opentelemetry::trace::{Span, Status, TraceContextExt, Tracer};
@@ -1060,6 +1061,47 @@ async fn delete_favorite(
}
}
/// Sentinel path written next to a would-be thumbnail when a file cannot be
/// decoded by either the `image` crate or ffmpeg. Its presence causes future
/// scans to skip the file instead of re-logging the failure.
pub fn unsupported_thumbnail_sentinel(thumb_path: &Path) -> PathBuf {
let mut s = thumb_path.as_os_str().to_owned();
s.push(".unsupported");
PathBuf::from(s)
}
fn generate_image_thumbnail(src: &Path, thumb_path: &Path) -> std::io::Result<()> {
// RAW formats (ARW/NEF/CR2/etc): try the file's embedded JPEG preview
// first. Avoids ffmpeg choking on proprietary RAW compression (Sony ARW
// in particular), and is faster than decoding RAW pixels anyway.
if let Some(preview) = exif::extract_embedded_jpeg_preview(src) {
let img = image::load_from_memory(&preview).map_err(|e| {
std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("decode embedded preview {:?}: {}", src, e),
)
})?;
let scaled = img.thumbnail(200, u32::MAX);
scaled
.save_with_format(thumb_path, image::ImageFormat::Jpeg)
.map_err(|e| std::io::Error::other(format!("save {:?}: {}", thumb_path, e)))?;
return Ok(());
}
if file_types::needs_ffmpeg_thumbnail(src) {
return generate_image_thumbnail_ffmpeg(src, thumb_path);
}
let img = image::open(src).map_err(|e| {
std::io::Error::new(std::io::ErrorKind::InvalidData, format!("{:?}: {}", src, e))
})?;
let scaled = img.thumbnail(200, u32::MAX);
scaled
.save(thumb_path)
.map_err(|e| std::io::Error::other(format!("save {:?}: {}", thumb_path, e)))?;
Ok(())
}
fn create_thumbnails(libs: &[libraries::Library]) {
let tracer = global_tracer();
let span = tracer.start("creating thumbnails");
@@ -1080,17 +1122,26 @@ fn create_thumbnails(libs: &[libraries::Library]) {
.into_par_iter()
.filter_map(|entry| entry.ok())
.filter(|entry| entry.file_type().is_file())
.filter(|entry| {
if is_video(entry) {
let relative_path = &entry.path().strip_prefix(&images).unwrap();
let thumb_path = Path::new(thumbnail_directory).join(relative_path);
std::fs::create_dir_all(
thumb_path
.parent()
.unwrap_or_else(|| panic!("Thumbnail {:?} has no parent?", thumb_path)),
)
.expect("Error creating directory");
.for_each(|entry| {
let src = entry.path();
let Ok(relative_path) = src.strip_prefix(&images) else {
return;
};
let thumb_path = Path::new(thumbnail_directory).join(relative_path);
if thumb_path.exists() || unsupported_thumbnail_sentinel(&thumb_path).exists() {
return;
}
let Some(parent) = thumb_path.parent() else {
return;
};
if let Err(e) = std::fs::create_dir_all(parent) {
error!("Failed to create thumbnail dir {:?}: {}", parent, e);
return;
}
if is_video(&entry) {
let mut video_span = tracer.start_with_context(
"generate_video_thumbnail",
&opentelemetry::Context::new()
@@ -1103,37 +1154,24 @@ fn create_thumbnails(libs: &[libraries::Library]) {
]);
debug!("Generating video thumbnail: {:?}", thumb_path);
generate_video_thumbnail(entry.path(), &thumb_path);
generate_video_thumbnail(src, &thumb_path);
video_span.end();
false
} else {
is_image(entry)
} else if is_image(&entry) {
match generate_image_thumbnail(src, &thumb_path) {
Ok(_) => info!("Saved thumbnail: {:?}", thumb_path),
Err(e) => {
let sentinel = unsupported_thumbnail_sentinel(&thumb_path);
error!(
"Unable to thumbnail {:?}: {}. Writing sentinel {:?}",
src, e, sentinel
);
if let Err(se) = std::fs::write(&sentinel, b"") {
warn!("Failed to write sentinel {:?}: {}", sentinel, se);
}
}
}
}
})
.filter(|entry| {
let path = entry.path();
let relative_path = &path.strip_prefix(&images).unwrap();
let thumb_path = Path::new(thumbnail_directory).join(relative_path);
!thumb_path.exists()
})
.map(|entry| (image::open(entry.path()), entry.path().to_path_buf()))
.filter(|(img, path)| {
if let Err(e) = img {
error!("Unable to open image: {:?}. {}", path, e);
}
img.is_ok()
})
.map(|(img, path)| (img.unwrap(), path))
.map(|(image, path)| (image.thumbnail(200, u32::MAX), path))
.map(|(image, path)| {
let relative_path = &path.strip_prefix(&images).unwrap();
let thumb_path = Path::new(thumbnail_directory).join(relative_path);
std::fs::create_dir_all(thumb_path.parent().unwrap())
.expect("There was an issue creating directory");
info!("Saving thumbnail: {:?}", thumb_path);
image.save(thumb_path).expect("Failure saving thumbnail");
})
.for_each(drop);
});
}
debug!("Finished making thumbnails");
@@ -1744,7 +1782,8 @@ fn process_new_files(
// not just photos with parseable EXIF.
for (file_path, relative_path) in &files {
let thumb_path = thumbnail_directory.join(relative_path);
let needs_thumbnail = !thumb_path.exists();
let needs_thumbnail =
!thumb_path.exists() && !unsupported_thumbnail_sentinel(&thumb_path).exists();
let needs_row = !existing_exif_paths.contains_key(relative_path);
if needs_thumbnail || needs_row {

51
src/video/actors.rs
View File

@@ -48,6 +48,14 @@ impl Handler<ProcessMessage> for StreamActor {
}
}
pub fn playlist_file_for(playlist_dir: &str, video_path: &Path) -> PathBuf {
let filename = video_path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("unknown");
PathBuf::from(format!("{}/{}.m3u8", playlist_dir, filename))
}
pub async fn create_playlist(video_path: &str, playlist_file: &str) -> Result<Child> {
if Path::new(playlist_file).exists() {
debug!("Playlist already exists: {}", playlist_file);
@@ -103,6 +111,35 @@ pub fn generate_video_thumbnail(path: &Path, destination: &Path) {
.expect("Failure to create video frame");
}
/// Use ffmpeg to extract a 200px-wide thumbnail from formats the `image` crate
/// can't decode (RAW: NEF/ARW, HEIC/HEIF). Writes JPEG bytes to `destination`
/// regardless of its extension.
pub fn generate_image_thumbnail_ffmpeg(path: &Path, destination: &Path) -> std::io::Result<()> {
let output = Command::new("ffmpeg")
.arg("-y")
.arg("-i")
.arg(path)
.arg("-vframes")
.arg("1")
.arg("-vf")
.arg("scale=200:-1")
.arg("-f")
.arg("image2")
.arg("-c:v")
.arg("mjpeg")
.arg(destination)
.output()?;
if !output.status.success() {
return Err(std::io::Error::other(format!(
"ffmpeg failed ({}): {}",
output.status,
String::from_utf8_lossy(&output.stderr).trim()
)));
}
Ok(())
}
/// Check if a video is already encoded with h264 codec
/// Returns true if the video uses h264, false otherwise or if detection fails
async fn is_h264_encoded(video_path: &str) -> bool {
@@ -246,15 +283,18 @@ impl Handler<ScanDirectoryMessage> for VideoPlaylistManager {
msg.directory
);
let playlist_output_dir = self.playlist_dir.clone();
let playlist_dir_str = playlist_output_dir.to_str().unwrap().to_string();
let video_files = WalkDir::new(&msg.directory)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| e.file_type().is_file())
.filter(is_video)
.filter(|e| !playlist_file_for(&playlist_dir_str, e.path()).exists())
.collect::<Vec<DirEntry>>();
let scan_dir_name = msg.directory.clone();
let playlist_output_dir = self.playlist_dir.clone();
let playlist_generator = self.playlist_generator.clone();
Box::pin(async move {
@@ -285,6 +325,9 @@ impl Handler<ScanDirectoryMessage> for VideoPlaylistManager {
path_as_str
);
}
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
debug!("Playlist already exists for '{:?}', skipping", path);
}
Err(e) => {
warn!("Failed to generate playlist for path '{:?}'. {:?}", path, e);
}
@@ -318,14 +361,18 @@ impl Handler<QueueVideosMessage> for VideoPlaylistManager {
);
let playlist_output_dir = self.playlist_dir.clone();
let playlist_dir_str = playlist_output_dir.to_str().unwrap().to_string();
let playlist_generator = self.playlist_generator.clone();
for video_path in msg.video_paths {
if playlist_file_for(&playlist_dir_str, &video_path).exists() {
continue;
}
let path_str = video_path.to_string_lossy().to_string();
debug!("Queueing playlist generation for: {}", path_str);
playlist_generator.do_send(GeneratePlaylistMessage {
playlist_path: playlist_output_dir.to_str().unwrap().to_string(),
playlist_path: playlist_dir_str.clone(),
video_path,
});
}