Files
ImageApi/src/files.rs
Cameron 18ba5796b0 Update to rust 2021
Fix tests
2024-12-05 20:27:01 -05:00

711 lines
22 KiB
Rust

use std::fmt::Debug;
use std::fs::read_dir;
use std::io;
use std::io::ErrorKind;
use std::path::{Path, PathBuf};
use std::sync::Mutex;
use ::anyhow;
use actix::{Handler, Message};
use anyhow::{anyhow, Context};
use actix_web::web::Data;
use actix_web::{
web::{self, Query},
HttpResponse,
};
use log::{debug, error, info, trace};
use crate::data::{Claims, FilesRequest, FilterMode, PhotosResponse, SortType};
use crate::{create_thumbnails, AppState};
use crate::error::IntoHttpError;
use crate::tags::TagDao;
use crate::video::StreamActor;
use path_absolutize::*;
use rand::prelude::SliceRandom;
use rand::thread_rng;
use serde::Deserialize;
pub async fn list_photos<TagD: TagDao, FS: FileSystemAccess>(
_: Claims,
req: Query<FilesRequest>,
app_state: web::Data<AppState>,
file_system: web::Data<FS>,
tag_dao: web::Data<Mutex<TagD>>,
) -> HttpResponse {
let search_path = &req.path;
let search_recursively = req.recursive.unwrap_or(false);
if let Some(tag_ids) = &req.tag_ids {
if search_recursively {
let filter_mode = &req.tag_filter_mode.unwrap_or(FilterMode::Any);
debug!(
"Searching for tags: {}. With path: '{}' and filter mode: {:?}",
tag_ids, search_path, filter_mode
);
let mut dao = tag_dao.lock().expect("Unable to get TagDao");
let tag_ids = tag_ids
.split(',')
.filter_map(|t| t.parse().ok())
.collect::<Vec<i32>>();
let exclude_tag_ids = req
.exclude_tag_ids
.clone()
.unwrap_or(String::new())
.split(',')
.filter_map(|t| t.parse().ok())
.collect::<Vec<i32>>();
return match filter_mode {
FilterMode::Any => dao.get_files_with_any_tag_ids(tag_ids.clone(), exclude_tag_ids),
FilterMode::All => dao.get_files_with_all_tag_ids(tag_ids.clone(), exclude_tag_ids),
}
.context(format!(
"Failed to get files with tag_ids: {:?} with filter_mode: {:?}",
tag_ids, filter_mode
))
.map(|tagged_files| {
tagged_files
.into_iter()
.filter(|f| f.starts_with(&format!("{}/", search_path).to_string()))
.collect::<Vec<String>>()
})
.inspect(|files| debug!("Found {:?} files", files.len()))
.map(|tagged_files: Vec<String>| {
trace!(
"Found {:?} tagged files: {:?}",
tagged_files.len(),
tagged_files
);
HttpResponse::Ok().json(PhotosResponse {
photos: tagged_files,
dirs: vec![],
})
})
.into_http_internal_err()
.unwrap_or_else(|e| e.error_response());
}
}
if let Ok(files) = file_system.get_files_for_path(search_path) {
debug!("Valid search path: {:?}", search_path);
let mut photos = files
.iter()
.filter(|&f| {
f.metadata().map_or_else(
|e| {
error!("Failed getting file metadata: {:?}", e);
f.extension().is_some()
},
|md| md.is_file(),
)
})
.map(|path: &PathBuf| {
let relative = path.strip_prefix(&app_state.base_path).unwrap();
relative.to_path_buf()
})
.map(|f| f.to_str().unwrap().to_string())
.filter(|file_path| {
if let (Some(tag_ids), Ok(mut tag_dao)) = (&req.tag_ids, tag_dao.lock()) {
let tag_ids = tag_ids
.split(',')
.filter_map(|t| t.parse().ok())
.collect::<Vec<i32>>();
let excluded_tag_ids = &req
.exclude_tag_ids
.clone()
.unwrap_or(String::new())
.split(',')
.filter_map(|t| t.parse().ok())
.collect::<Vec<i32>>();
let filter_mode = &req.tag_filter_mode.unwrap_or(FilterMode::Any);
let file_tags = tag_dao.get_tags_for_path(file_path).unwrap_or_default();
let excluded = file_tags.iter().any(|t| excluded_tag_ids.contains(&t.id));
return !excluded
&& match filter_mode {
FilterMode::Any => file_tags.iter().any(|t| tag_ids.contains(&t.id)),
FilterMode::All => tag_ids
.iter()
.all(|id| file_tags.iter().any(|tag| &tag.id == id)),
};
}
true
})
.collect::<Vec<String>>();
if let Some(sort_type) = req.sort {
debug!("Sorting files: {:?}", sort_type);
photos = sort(photos, sort_type)
}
let dirs = files
.iter()
.filter(|&f| f.metadata().map_or(false, |md| md.is_dir()))
.map(|path: &PathBuf| {
let relative = path.strip_prefix(&app_state.base_path).unwrap();
relative.to_path_buf()
})
.map(|f| f.to_str().unwrap().to_string())
.collect::<Vec<String>>();
HttpResponse::Ok().json(PhotosResponse { photos, dirs })
} else {
error!("Bad photos request: {}", req.path);
HttpResponse::BadRequest().finish()
}
}
fn sort(mut files: Vec<String>, sort_type: SortType) -> Vec<String> {
match sort_type {
SortType::Shuffle => files.shuffle(&mut thread_rng()),
SortType::NameAsc => {
files.sort_by(|l, r| l.cmp(&r));
}
SortType::NameDesc => {
files.sort_by(|l, r| r.cmp(&l));
}
}
files
}
pub fn list_files(dir: &Path) -> io::Result<Vec<PathBuf>> {
let files = read_dir(dir)?
.filter_map(|res| res.ok())
.filter(|entry| is_image_or_video(&entry.path()) || entry.file_type().unwrap().is_dir())
.map(|entry| entry.path())
.collect::<Vec<PathBuf>>();
Ok(files)
}
pub fn is_image_or_video(path: &Path) -> bool {
let extension = path
.extension()
.and_then(|p| p.to_str())
.map_or(String::from(""), |p| p.to_lowercase());
extension == "png"
|| extension == "jpg"
|| extension == "jpeg"
|| extension == "mp4"
|| extension == "mov"
|| extension == "nef"
}
pub fn is_valid_full_path<P: AsRef<Path> + Debug + AsRef<std::ffi::OsStr>>(
base: &P,
path: &P,
new_file: bool,
) -> Option<PathBuf> {
trace!("is_valid_full_path => Base: {:?}. Path: {:?}", base, path);
let path = PathBuf::from(&path);
let mut path = if path.is_relative() {
let mut full_path = PathBuf::new();
full_path.push(base);
full_path.push(&path);
full_path
} else {
path
};
match is_path_above_base_dir(base, &mut path, new_file) {
Ok(path) => Some(path),
Err(e) => {
error!("{}", e);
None
}
}
}
fn is_path_above_base_dir<P: AsRef<Path> + Debug>(
base: P,
full_path: &mut PathBuf,
new_file: bool,
) -> anyhow::Result<PathBuf> {
full_path
.absolutize()
.with_context(|| format!("Unable to resolve absolute path: {:?}", full_path))
.map_or_else(
|e| Err(anyhow!(e)),
|p| {
if p.starts_with(base) && (new_file || p.exists()) {
Ok(p.into_owned())
} else if !p.exists() {
Err(anyhow!("Path does not exist: {:?}", p))
} else {
Err(anyhow!("Path above base directory"))
}
},
)
}
pub async fn move_file<FS: FileSystemAccess>(
_: Claims,
file_system: web::Data<FS>,
app_state: Data<AppState>,
request: web::Json<MoveFileRequest>,
) -> HttpResponse {
match is_valid_full_path(&app_state.base_path, &request.source, false)
.ok_or(ErrorKind::InvalidData)
.and_then(|source| {
is_valid_full_path(&app_state.base_path, &request.destination, true)
.ok_or(ErrorKind::InvalidData)
.and_then(|dest| {
if dest.exists() {
error!("Destination already exists, not moving file: {:?}", source);
Err(ErrorKind::AlreadyExists)
} else {
Ok(dest)
}
})
.map(|dest| (source, dest))
})
.map(|(source, dest)| file_system.move_file(source, dest))
{
Ok(_) => {
info!("Moved file: {} -> {}", request.source, request.destination,);
app_state.stream_manager.do_send(RefreshThumbnailsMessage);
HttpResponse::Ok().finish()
}
Err(e) => {
error!(
"Error moving file: {} to: {}. {}",
request.source, request.destination, e
);
if e == ErrorKind::InvalidData {
HttpResponse::BadRequest().finish()
} else {
HttpResponse::InternalServerError().finish()
}
}
}
}
#[derive(Deserialize)]
pub struct MoveFileRequest {
source: String,
destination: String,
}
pub trait FileSystemAccess {
fn get_files_for_path(&self, path: &str) -> anyhow::Result<Vec<PathBuf>>;
fn move_file<P: AsRef<Path>>(&self, from: P, destination: P) -> anyhow::Result<()>;
}
pub struct RealFileSystem {
base_path: String,
}
impl RealFileSystem {
pub(crate) fn new(base_path: String) -> RealFileSystem {
RealFileSystem { base_path }
}
}
impl FileSystemAccess for RealFileSystem {
fn get_files_for_path(&self, path: &str) -> anyhow::Result<Vec<PathBuf>> {
is_valid_full_path(&PathBuf::from(&self.base_path), &PathBuf::from(path), false)
.map(|path| {
debug!("Valid path: {:?}", path);
list_files(&path).unwrap_or_default()
})
.context("Invalid path")
}
fn move_file<P: AsRef<Path>>(&self, from: P, destination: P) -> anyhow::Result<()> {
let name = from
.as_ref()
.file_name()
.map(|n| n.to_str().unwrap_or_default().to_string())
.unwrap_or_default();
std::fs::rename(from, destination)
.with_context(|| format!("Failed to move file: {:?}", name))
}
}
pub struct RefreshThumbnailsMessage;
impl Message for RefreshThumbnailsMessage {
type Result = ();
}
impl Handler<RefreshThumbnailsMessage> for StreamActor {
type Result = ();
fn handle(&mut self, _msg: RefreshThumbnailsMessage, _ctx: &mut Self::Context) -> Self::Result {
info!("Refreshing thumbnails after upload");
create_thumbnails()
}
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use std::env;
use std::fs::File;
use super::*;
struct FakeFileSystem {
files: HashMap<String, Vec<String>>,
err: bool,
}
impl FakeFileSystem {
fn with_error() -> FakeFileSystem {
FakeFileSystem {
files: HashMap::new(),
err: true,
}
}
fn new(files: HashMap<String, Vec<String>>) -> FakeFileSystem {
FakeFileSystem { files, err: false }
}
}
impl FileSystemAccess for FakeFileSystem {
fn get_files_for_path(&self, path: &str) -> anyhow::Result<Vec<PathBuf>> {
if self.err {
Err(anyhow!("Error for test"))
} else {
if let Some(files) = self.files.get(path) {
Ok(files
.iter()
.map(|p| PathBuf::from(p))
.collect::<Vec<PathBuf>>())
} else {
Ok(Vec::new())
}
}
}
fn move_file<P: AsRef<Path>>(&self, from: P, destination: P) -> anyhow::Result<()> {
todo!()
}
}
mod api {
use super::*;
use actix::Actor;
use actix_web::{web::Query, HttpResponse};
use crate::{
data::{Claims, PhotosResponse},
testhelpers::BodyReader,
video::StreamActor,
AppState,
};
use crate::database::test::in_memory_db_connection;
use crate::tags::SqliteTagDao;
use actix_web::web::Data;
use std::{fs, sync::Arc};
fn setup() {
let _ = env_logger::builder().is_test(true).try_init();
}
#[actix_rt::test]
async fn test_list_photos() {
setup();
let claims = Claims {
sub: String::from("1"),
exp: 12345,
};
let request: Query<FilesRequest> = Query::from_query("path=").unwrap();
let mut temp_photo = env::temp_dir();
let mut tmp = temp_photo.clone();
tmp.push("test-dir");
fs::create_dir_all(tmp).unwrap();
temp_photo.push("photo.jpg");
File::create(temp_photo.clone()).unwrap();
let response: HttpResponse = list_photos(
claims,
request,
Data::new(AppState::new(
Arc::new(StreamActor {}.start()),
String::from("/tmp"),
String::from("/tmp/thumbs"),
String::from("/tmp/video"),
)),
Data::new(RealFileSystem::new(String::from("/tmp"))),
Data::new(Mutex::new(SqliteTagDao::default())),
)
.await;
let status = response.status();
assert_eq!(status, 200);
let body: PhotosResponse = serde_json::from_str(&response.read_to_str()).unwrap();
debug!("{:?}", body);
assert!(body.photos.contains(&String::from("photo.jpg")));
assert!(body.dirs.contains(&String::from("test-dir")));
assert!(body
.photos
.iter()
.filter(|filename| !filename.ends_with(".png")
&& !filename.ends_with(".jpg")
&& !filename.ends_with(".jpeg"))
.collect::<Vec<&String>>()
.is_empty());
}
#[actix_rt::test]
async fn test_list_below_base_fails_400() {
setup();
let claims = Claims {
sub: String::from("1"),
exp: 12345,
};
let request: Query<FilesRequest> = Query::from_query("path=..").unwrap();
let response = list_photos(
claims,
request,
Data::new(AppState::new(
Arc::new(StreamActor {}.start()),
String::from("/tmp"),
String::from("/tmp/thumbs"),
String::from("/tmp/video"),
)),
Data::new(RealFileSystem::new(String::from("./"))),
Data::new(Mutex::new(SqliteTagDao::default())),
)
.await;
assert_eq!(response.status(), 400);
}
#[actix_rt::test]
async fn get_files_with_tag_any_filter() {
setup();
let claims = Claims {
sub: String::from("1"),
exp: 12345,
};
let request: Query<FilesRequest> = Query::from_query("path=&tag_ids=1,3").unwrap();
let mut tag_dao = SqliteTagDao::new(in_memory_db_connection());
let tag1 = tag_dao.create_tag("tag1").unwrap();
let _tag2 = tag_dao.create_tag("tag2").unwrap();
let tag3 = tag_dao.create_tag("tag3").unwrap();
let _ = &tag_dao.tag_file("test.jpg", tag1.id).unwrap();
let _ = &tag_dao.tag_file("test.jpg", tag3.id).unwrap();
let mut files = HashMap::new();
files.insert(
String::from(""),
vec![
String::from("file1.txt"),
String::from("test.jpg"),
String::from("some-other.jpg"),
],
);
let response: HttpResponse = list_photos(
claims,
request,
Data::new(AppState::new(
Arc::new(StreamActor {}.start()),
String::from(""),
String::from("/tmp/thumbs"),
String::from("/tmp/video"),
)),
Data::new(FakeFileSystem::new(files)),
Data::new(Mutex::new(tag_dao)),
)
.await;
assert_eq!(200, response.status());
let body: PhotosResponse = serde_json::from_str(&response.read_to_str()).unwrap();
assert_eq!(1, body.photos.len());
assert!(body.photos.contains(&String::from("test.jpg")));
}
#[actix_rt::test]
async fn get_files_with_tag_all_filter() {
setup();
let claims = Claims {
sub: String::from("1"),
exp: 12345,
};
let mut tag_dao = SqliteTagDao::new(in_memory_db_connection());
let tag1 = tag_dao.create_tag("tag1").unwrap();
let _tag2 = tag_dao.create_tag("tag2").unwrap();
let tag3 = tag_dao.create_tag("tag3").unwrap();
let _ = &tag_dao.tag_file("test.jpg", tag1.id).unwrap();
let _ = &tag_dao.tag_file("test.jpg", tag3.id).unwrap();
// Should get filtered since it doesn't have tag3
tag_dao.tag_file("some-other.jpg", tag1.id).unwrap();
let mut files = HashMap::new();
files.insert(
String::from(""),
vec![
String::from("file1.txt"),
String::from("test.jpg"),
String::from("some-other.jpg"),
],
);
let request: Query<FilesRequest> = Query::from_query(&*format!(
"path=&tag_ids={},{}&tag_filter_mode=All",
tag1.id, tag3.id
))
.unwrap();
let response: HttpResponse = list_photos(
claims,
request,
Data::new(AppState::new(
Arc::new(StreamActor {}.start()),
String::from(""),
String::from("/tmp/thumbs"),
String::from("/tmp/video"),
)),
Data::new(FakeFileSystem::new(files)),
Data::new(Mutex::new(tag_dao)),
)
.await;
assert_eq!(200, response.status());
let body: PhotosResponse = serde_json::from_str(&response.read_to_str()).unwrap();
assert_eq!(1, body.photos.len());
assert!(body.photos.contains(&String::from("test.jpg")));
}
}
#[test]
fn directory_traversal_test() {
let base = env::temp_dir();
assert_eq!(
None,
is_valid_full_path(&base, &PathBuf::from("../"), false)
);
assert_eq!(None, is_valid_full_path(&base, &PathBuf::from(".."), false));
assert_eq!(
None,
is_valid_full_path(&base, &PathBuf::from("fake/../../../"), false)
);
assert_eq!(
None,
is_valid_full_path(&base, &PathBuf::from("../../../etc/passwd"), false)
);
assert_eq!(
None,
is_valid_full_path(&base, &PathBuf::from("..//etc/passwd"), false)
);
assert_eq!(
None,
is_valid_full_path(&base, &PathBuf::from("../../etc/passwd"), false)
);
}
#[test]
fn build_from_path_relative_to_base_test() {
let base = env::temp_dir();
let mut test_file = PathBuf::from(&base);
test_file.push("test.png");
File::create(test_file).unwrap();
assert!(is_valid_full_path(&base, &PathBuf::from("test.png"), false).is_some());
}
#[test]
fn build_from_relative_returns_none_if_directory_does_not_exist_test() {
let base = env::temp_dir();
let path = "relative/path/test.png";
let mut test_file = PathBuf::from(&base);
test_file.push(path);
assert_eq!(None, is_valid_full_path(&base, &test_file, false));
}
#[test]
fn build_from_absolute_path_test() {
let base = env::temp_dir();
let mut test_file = PathBuf::from(&base);
test_file.push("test.png");
File::create(&test_file).unwrap();
assert!(is_valid_full_path(&base, &test_file, false).is_some());
assert_eq!(
Some(PathBuf::from("/tmp/test.png")),
is_valid_full_path(&base, &PathBuf::from("/tmp/test.png"), false)
);
}
macro_rules! extension_test {
($name:ident, $filename:literal) => {
#[test]
fn $name() {
assert!(is_image_or_video(Path::new($filename)));
}
};
}
extension_test!(valid_png, "image.png");
extension_test!(valid_png_mixed_case, "image.pNg");
extension_test!(valid_png_upper_case, "image.PNG");
extension_test!(valid_jpeg, "image.jpeg");
extension_test!(valid_jpeg_upper_case, "image.JPEG");
extension_test!(valid_jpg, "image.jpg");
extension_test!(valid_jpg_upper_case, "image.JPG");
extension_test!(valid_mp4, "image.mp4");
extension_test!(valid_mp4_mixed_case, "image.mP4");
extension_test!(valid_mp4_upper_case, "image.MP4");
extension_test!(valid_mov, "image.mov");
extension_test!(valid_mov_mixed_case, "image.mOV");
extension_test!(valid_mov_upper_case, "image.MOV");
extension_test!(valid_nef, "image.nef");
extension_test!(valid_nef_mixed_case, "image.nEF");
extension_test!(valid_nef_upper_case, "image.NEF");
#[test]
fn hidden_file_not_valid_test() {
assert!(!is_image_or_video(Path::new(".DS_store")));
}
}