From ef39359862486bd7e59834a8973e6e6c66b68f7d Mon Sep 17 00:00:00 2001 From: Cameron Cordes Date: Thu, 7 Mar 2024 17:56:50 -0500 Subject: [PATCH 1/3] Add basic recursive tag searching support based on the search path --- src/data/mod.rs | 1 + src/files.rs | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index ed9cc5f..23d5844 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -105,6 +105,7 @@ pub struct FilesRequest { pub path: String, pub tag_ids: Option, // comma separated numbers pub tag_filter_mode: Option, + pub recursive: Option, } #[derive(Copy, Clone, Deserialize, PartialEq)] diff --git a/src/files.rs b/src/files.rs index 031420d..d7ea0dd 100644 --- a/src/files.rs +++ b/src/files.rs @@ -35,7 +35,9 @@ pub async fn list_photos( ) -> HttpResponse { let path = &req.path; + // Do we need this? if let (Some(tag_ids), Some(filter_mode)) = (&req.tag_ids, &req.tag_filter_mode) { + let search_recursively = &req.recursive.unwrap_or(false); if *filter_mode == FilterMode::All { let mut dao = tag_dao.lock().expect("Unable to get TagDao"); let tag_ids = tag_ids @@ -45,10 +47,15 @@ pub async fn list_photos( return dao .get_files_with_tag_ids(tag_ids.clone()) - .context(format!("Failed to files with tag_ids: {:?}", tag_ids)) + .context(format!("Failed to get files with tag_ids: {:?}", tag_ids)) .map(|tagged_files| { HttpResponse::Ok().json(PhotosResponse { - photos: tagged_files, + photos: tagged_files.iter().filter(|&file_path| { + let slash_count = file_path.split('/').count(); + let search_path_slash_count = path.split('/').count(); + + slash_count > search_path_slash_count + }).map(|p| p.clone()).collect::>(), dirs: vec![], }) }) @@ -77,6 +84,12 @@ pub async fn list_photos( }) .map(|f| f.to_str().unwrap().to_string()) .filter(|file_path| { + let recursive = req.recursive.unwrap_or(false); + let file_slash_count = file_path.split("/").collect::>().len(); + let filter_path_slash_count = path.split("/").collect::>().len(); + // Skip if this path is below/deeper than the search path + if !recursive && file_slash_count > filter_path_slash_count { return false; } + if let (Some(tag_ids), Ok(mut tag_dao)) = (&req.tag_ids, tag_dao.lock()) { let tag_ids = tag_ids .split(',') From b2c8ebe5582e1d6697b1e6f6479d122c03c6c30c Mon Sep 17 00:00:00 2001 From: Cameron Cordes Date: Thu, 7 Mar 2024 19:01:46 -0500 Subject: [PATCH 2/3] Break-up FilterMode.All being recursive Filtering on tags needs some reworking to handle recursive with All or Any filtering. --- src/files.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/files.rs b/src/files.rs index d7ea0dd..85a865c 100644 --- a/src/files.rs +++ b/src/files.rs @@ -1,4 +1,3 @@ - use std::fmt::Debug; use std::fs::read_dir; use std::io; @@ -38,7 +37,7 @@ pub async fn list_photos( // Do we need this? if let (Some(tag_ids), Some(filter_mode)) = (&req.tag_ids, &req.tag_filter_mode) { let search_recursively = &req.recursive.unwrap_or(false); - if *filter_mode == FilterMode::All { + if search_recursively { let mut dao = tag_dao.lock().expect("Unable to get TagDao"); let tag_ids = tag_ids .split(',') @@ -50,12 +49,7 @@ pub async fn list_photos( .context(format!("Failed to get files with tag_ids: {:?}", tag_ids)) .map(|tagged_files| { HttpResponse::Ok().json(PhotosResponse { - photos: tagged_files.iter().filter(|&file_path| { - let slash_count = file_path.split('/').count(); - let search_path_slash_count = path.split('/').count(); - - slash_count > search_path_slash_count - }).map(|p| p.clone()).collect::>(), + photos: tagged_files, dirs: vec![], }) }) From 05a56ba0bd72f12cb7682bfb46b27dc9d1d20473 Mon Sep 17 00:00:00 2001 From: Cameron Cordes Date: Sat, 9 Mar 2024 13:11:55 -0500 Subject: [PATCH 3/3] Fix Recursive searching with tags including Any and All filter modes --- README.md | 1 + src/data/mod.rs | 2 +- src/files.rs | 55 ++++++++++++++++++++++++++++++------------------- src/main.rs | 10 +++++---- src/tags.rs | 26 ++++++++++++++++++----- 5 files changed, 63 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 429e3d6..3c5b5af 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ Upon first run it will generate thumbnails for all images and videos at `BASE_PA ## Environment There are a handful of required environment variables to have the API run. They should be defined where the binary is located or above it in an `.env` file. +You must have `ffmpeg` installed for streaming video and generating video thumbnails. - `DATABASE_URL` is a path or url to a database (currently only SQLite is tested) - `BASE_PATH` is the root from which you want to serve images and videos diff --git a/src/data/mod.rs b/src/data/mod.rs index 23d5844..65049ba 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -108,7 +108,7 @@ pub struct FilesRequest { pub recursive: Option, } -#[derive(Copy, Clone, Deserialize, PartialEq)] +#[derive(Copy, Clone, Deserialize, PartialEq, Debug)] pub enum FilterMode { Any, All, diff --git a/src/files.rs b/src/files.rs index 85a865c..df2a233 100644 --- a/src/files.rs +++ b/src/files.rs @@ -14,16 +14,16 @@ use actix_web::{ web::{self, Query}, HttpResponse, }; -use log::{debug, error, info}; +use log::{debug, error, info, trace}; use crate::data::{Claims, FilesRequest, FilterMode, PhotosResponse}; -use crate::{AppState, create_thumbnails}; +use crate::{create_thumbnails, AppState}; use crate::error::IntoHttpError; use crate::tags::TagDao; +use crate::video::StreamActor; use path_absolutize::*; use serde::Deserialize; -use crate::video::StreamActor; pub async fn list_photos( _: Claims, @@ -34,10 +34,15 @@ pub async fn list_photos( ) -> HttpResponse { let path = &req.path; - // Do we need this? - if let (Some(tag_ids), Some(filter_mode)) = (&req.tag_ids, &req.tag_filter_mode) { - let search_recursively = &req.recursive.unwrap_or(false); + let search_recursively = req.recursive.unwrap_or(false); + if let Some(tag_ids) = &req.tag_ids { if search_recursively { + let filter_mode = req.tag_filter_mode.unwrap_or(FilterMode::Any); + debug!( + "Searching for tags: {}. With path: '{}' and filter mode: {:?}", + tag_ids, path, filter_mode + ); + let mut dao = tag_dao.lock().expect("Unable to get TagDao"); let tag_ids = tag_ids .split(',') @@ -45,9 +50,24 @@ pub async fn list_photos( .collect::>(); return dao - .get_files_with_tag_ids(tag_ids.clone()) + .get_files_with_any_tag_ids(tag_ids.clone()) .context(format!("Failed to get files with tag_ids: {:?}", tag_ids)) + .map(|tagged_files| match filter_mode { + FilterMode::Any => tagged_files, + FilterMode::All => tagged_files + .iter() + .filter(|&file_path| { + let file_tags = dao.get_tags_for_path(file_path).unwrap_or_default(); + tag_ids + .iter() + .all(|id| file_tags.iter().any(|tag| &tag.id == id)) + }) + .cloned() + .collect::>(), + }) .map(|tagged_files| { + trace!("Found tagged files: {:?}", tagged_files); + HttpResponse::Ok().json(PhotosResponse { photos: tagged_files, dirs: vec![], @@ -59,7 +79,7 @@ pub async fn list_photos( } if let Ok(files) = file_system.get_files_for_path(path) { - debug!("Valid path: {:?}", path); + debug!("Valid search path: {:?}", path); let photos = files .iter() @@ -78,12 +98,6 @@ pub async fn list_photos( }) .map(|f| f.to_str().unwrap().to_string()) .filter(|file_path| { - let recursive = req.recursive.unwrap_or(false); - let file_slash_count = file_path.split("/").collect::>().len(); - let filter_path_slash_count = path.split("/").collect::>().len(); - // Skip if this path is below/deeper than the search path - if !recursive && file_slash_count > filter_path_slash_count { return false; } - if let (Some(tag_ids), Ok(mut tag_dao)) = (&req.tag_ids, tag_dao.lock()) { let tag_ids = tag_ids .split(',') @@ -91,7 +105,6 @@ pub async fn list_photos( .collect::>(); let filter_mode = &req.tag_filter_mode.unwrap_or(FilterMode::Any); - let file_tags = tag_dao.get_tags_for_path(file_path).unwrap_or_default(); return match filter_mode { @@ -152,7 +165,7 @@ pub fn is_valid_full_path + Debug + AsRef>( path: &P, new_file: bool, ) -> Option { - debug!("Base: {:?}. Path: {:?}", base, path); + trace!("is_valid_full_path => Base: {:?}. Path: {:?}", base, path); let path = PathBuf::from(&path); let mut path = if path.is_relative() { @@ -398,7 +411,7 @@ mod tests { Data::new(RealFileSystem::new(String::from("/tmp"))), Data::new(Mutex::new(SqliteTagDao::default())), ) - .await; + .await; let status = response.status(); assert_eq!(status, 200); @@ -438,7 +451,7 @@ mod tests { Data::new(RealFileSystem::new(String::from("./"))), Data::new(Mutex::new(SqliteTagDao::default())), ) - .await; + .await; assert_eq!(response.status(), 400); } @@ -484,7 +497,7 @@ mod tests { Data::new(FakeFileSystem::new(files)), Data::new(Mutex::new(tag_dao)), ) - .await; + .await; assert_eq!(200, response.status()); @@ -528,7 +541,7 @@ mod tests { "path=&tag_ids={},{}&tag_filter_mode=All", tag1.id, tag3.id )) - .unwrap(); + .unwrap(); let response: HttpResponse = list_photos( claims, @@ -541,7 +554,7 @@ mod tests { Data::new(FakeFileSystem::new(files)), Data::new(Mutex::new(tag_dao)), ) - .await; + .await; assert_eq!(200, response.status()); diff --git a/src/main.rs b/src/main.rs index 6209288..4c35123 100644 --- a/src/main.rs +++ b/src/main.rs @@ -31,12 +31,14 @@ use diesel::sqlite::Sqlite; use notify::{Config, EventKind, RecommendedWatcher, RecursiveMode, Watcher}; use rayon::prelude::*; -use log::{debug, error, info, warn}; +use log::{debug, error, info, trace, warn}; use crate::auth::login; use crate::data::*; use crate::database::*; -use crate::files::{is_image_or_video, is_valid_full_path, move_file, RealFileSystem, RefreshThumbnailsMessage}; +use crate::files::{ + is_image_or_video, is_valid_full_path, move_file, RealFileSystem, RefreshThumbnailsMessage, +}; use crate::service::ServiceBuilder; use crate::state::AppState; use crate::tags::*; @@ -86,7 +88,7 @@ async fn get_image( let thumbs = &app_state.thumbnail_path; let thumb_path = Path::new(&thumbs).join(relative_path); - debug!("{:?}", thumb_path); + trace!("Thumbnail path: {:?}", thumb_path); if let Ok(file) = NamedFile::open(&thumb_path) { file.into_response(&request) } else { @@ -195,7 +197,7 @@ async fn upload_image( } app_state.stream_manager.do_send(RefreshThumbnailsMessage); - + HttpResponse::Ok().finish() } diff --git a/src/tags.rs b/src/tags.rs index eb2cb7a..e478022 100644 --- a/src/tags.rs +++ b/src/tags.rs @@ -6,7 +6,7 @@ use anyhow::Context; use chrono::Utc; use diesel::dsl::count_star; use diesel::prelude::*; -use log::{debug, info}; +use log::{debug, info, trace}; use schema::{tagged_photo, tags}; use serde::{Deserialize, Serialize}; use std::borrow::BorrowMut; @@ -198,7 +198,8 @@ pub trait TagDao { fn create_tag(&mut self, name: &str) -> anyhow::Result; fn remove_tag(&mut self, tag_name: &str, path: &str) -> anyhow::Result>; fn tag_file(&mut self, path: &str, tag_id: i32) -> anyhow::Result; - fn get_files_with_tag_ids(&mut self, tag_ids: Vec) -> anyhow::Result>; + fn get_files_with_all_tag_ids(&mut self, tag_ids: Vec) -> anyhow::Result>; + fn get_files_with_any_tag_ids(&mut self, tag_ids: Vec) -> anyhow::Result>; } pub struct SqliteTagDao { @@ -248,7 +249,7 @@ impl TagDao for SqliteTagDao { } fn get_tags_for_path(&mut self, path: &str) -> anyhow::Result> { - debug!("Getting Tags for path: {:?}", path); + trace!("Getting Tags for path: {:?}", path); tags::table .left_join(tagged_photo::table) .filter(tagged_photo::photo_name.eq(&path)) @@ -340,7 +341,7 @@ impl TagDao for SqliteTagDao { }) } - fn get_files_with_tag_ids(&mut self, tag_ids: Vec) -> anyhow::Result> { + fn get_files_with_all_tag_ids(&mut self, tag_ids: Vec) -> anyhow::Result> { use diesel::dsl::*; tagged_photo::table @@ -352,6 +353,18 @@ impl TagDao for SqliteTagDao { .get_results::(&mut self.connection) .with_context(|| format!("Unable to get Tagged photos with ids: {:?}", tag_ids)) } + + fn get_files_with_any_tag_ids(&mut self, tag_ids: Vec) -> anyhow::Result> { + use diesel::dsl::*; + + tagged_photo::table + .filter(tagged_photo::tag_id.eq_any(tag_ids.clone())) + .group_by(tagged_photo::photo_name) + .select((tagged_photo::photo_name, count(tagged_photo::tag_id))) + .select(tagged_photo::photo_name) + .get_results::(&mut self.connection) + .with_context(|| format!("Unable to get Tagged photos with ids: {:?}", tag_ids)) + } } #[cfg(test)] @@ -472,7 +485,10 @@ mod tests { } } - fn get_files_with_tag_ids(&mut self, _tag_ids: Vec) -> anyhow::Result> { + fn get_files_with_all_tag_ids( + &mut self, + _tag_ids: Vec, + ) -> anyhow::Result> { todo!() } }