From 47d3ad722260aff6be59853f1e4c27639908a0e3 Mon Sep 17 00:00:00 2001 From: Cameron Date: Mon, 22 Dec 2025 22:54:19 -0500 Subject: [PATCH] Add polling-based file watching Remove notify and update otel creates --- Cargo.lock | 259 ++++++++++------------------------------------------ Cargo.toml | 13 ++- README.md | 11 +++ src/lib.rs | 2 +- src/main.rs | 254 ++++++++++++++++++++++++++++++++++++++++----------- 5 files changed, 267 insertions(+), 272 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9228d68..c390518 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,7 +11,7 @@ dependencies = [ "actix-macros", "actix-rt", "actix_derive", - "bitflags 2.9.3", + "bitflags", "bytes", "crossbeam-channel", "futures-core", @@ -33,7 +33,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" dependencies = [ - "bitflags 2.9.3", + "bitflags", "bytes", "futures-core", "futures-sink", @@ -69,7 +69,7 @@ dependencies = [ "actix-service", "actix-utils", "actix-web", - "bitflags 2.9.3", + "bitflags", "bytes", "derive_more 2.0.1", "futures-core", @@ -93,7 +93,7 @@ dependencies = [ "actix-service", "actix-utils", "base64", - "bitflags 2.9.3", + "bitflags", "brotli", "bytes", "bytestring", @@ -206,7 +206,7 @@ dependencies = [ "actix-utils", "futures-core", "futures-util", - "mio 1.0.4", + "mio", "socket2 0.5.10", "tokio", "tracing", @@ -524,23 +524,17 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bcrypt" -version = "0.16.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b1866ecef4f2d06a0bb77880015fdf2b89e25a1c2e5addacb87e459c86dc67e" +checksum = "abaf6da45c74385272ddf00e1ac074c7d8a6c1a1dda376902bd6a427522a8b2c" dependencies = [ "base64", "blowfish", - "getrandom 0.2.16", + "getrandom 0.3.3", "subtle", "zeroize", ] -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - [[package]] name = "bitflags" version = "2.9.3" @@ -1116,18 +1110,6 @@ dependencies = [ "simd-adler32", ] -[[package]] -name = "filetime" -version = "0.2.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc0505cd1b6fa6580283f6bdf70a73fcf4aba1184038c90902b92b3dd0df63ed" -dependencies = [ - "cfg-if", - "libc", - "libredox", - "windows-sys 0.60.2", -] - [[package]] name = "find-msvc-tools" version = "0.1.0" @@ -1165,15 +1147,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fsevent-sys" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" -dependencies = [ - "libc", -] - [[package]] name = "futures" version = "0.3.31" @@ -1657,7 +1630,6 @@ dependencies = [ "kamadak-exif", "lazy_static", "log", - "notify", "opentelemetry", "opentelemetry-appender-log", "opentelemetry-otlp", @@ -1706,26 +1678,6 @@ dependencies = [ "cfb", ] -[[package]] -name = "inotify" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff" -dependencies = [ - "bitflags 1.3.2", - "inotify-sys", - "libc", -] - -[[package]] -name = "inotify-sys" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" -dependencies = [ - "libc", -] - [[package]] name = "inout" version = "0.1.4" @@ -1752,7 +1704,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" dependencies = [ - "bitflags 2.9.3", + "bitflags", "cfg-if", "libc", ] @@ -1871,26 +1823,6 @@ dependencies = [ "mutate_once", ] -[[package]] -name = "kqueue" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a" -dependencies = [ - "kqueue-sys", - "libc", -] - -[[package]] -name = "kqueue-sys" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" -dependencies = [ - "bitflags 1.3.2", - "libc", -] - [[package]] name = "language-tags" version = "0.3.2" @@ -1919,17 +1851,6 @@ dependencies = [ "cc", ] -[[package]] -name = "libredox" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3" -dependencies = [ - "bitflags 2.9.3", - "libc", - "redox_syscall", -] - [[package]] name = "libsqlite3-sys" version = "0.35.0" @@ -2063,18 +1984,6 @@ dependencies = [ "simd-adler32", ] -[[package]] -name = "mio" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" -dependencies = [ - "libc", - "log", - "wasi 0.11.1+wasi-snapshot-preview1", - "windows-sys 0.48.0", -] - [[package]] name = "mio" version = "1.0.4" @@ -2125,25 +2034,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" -[[package]] -name = "notify" -version = "6.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" -dependencies = [ - "bitflags 2.9.3", - "crossbeam-channel", - "filetime", - "fsevent-sys", - "inotify", - "kqueue", - "libc", - "log", - "mio 0.8.11", - "walkdir", - "windows-sys 0.48.0", -] - [[package]] name = "num-bigint" version = "0.4.6" @@ -2223,9 +2113,9 @@ checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" [[package]] name = "opentelemetry" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf416e4cb72756655126f7dd7bb0af49c674f4c1b9903e80c009e0c37e552e6" +checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" dependencies = [ "futures-core", "futures-sink", @@ -2237,9 +2127,9 @@ dependencies = [ [[package]] name = "opentelemetry-appender-log" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e688026e48f4603494f619583e0aa0b0edd9c0b9430e1c46804df2ff32bc8798" +checksum = "9e50c59a96bd6a723a4329c5db31eb04fa4488c5f141ae7b9d4fd587439e6ee1" dependencies = [ "log", "opentelemetry", @@ -2247,9 +2137,9 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f6639e842a97dbea8886e3439710ae463120091e2e064518ba8e716e6ac36d" +checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" dependencies = [ "async-trait", "bytes", @@ -2260,9 +2150,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbee664a43e07615731afc539ca60c6d9f1a9425e25ca09c57bc36c87c55852b" +checksum = "7a2366db2dca4d2ad033cad11e6ee42844fd727007af5ad04a1730f4cb8163bf" dependencies = [ "http 1.3.1", "opentelemetry", @@ -2279,21 +2169,22 @@ dependencies = [ [[package]] name = "opentelemetry-proto" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e046fd7660710fe5a05e8748e70d9058dc15c94ba914e7c4faa7c728f0e8ddc" +checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" dependencies = [ "opentelemetry", "opentelemetry_sdk", "prost", "tonic", + "tonic-prost", ] [[package]] name = "opentelemetry-stdout" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447191061af41c3943e082ea359ab8b64ff27d6d34d30d327df309ddef1eef6f" +checksum = "bc8887887e169414f637b18751487cce4e095be787d23fad13c454e2fb1b3811" dependencies = [ "chrono", "opentelemetry", @@ -2302,9 +2193,9 @@ dependencies = [ [[package]] name = "opentelemetry_sdk" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11f644aa9e5e31d11896e024305d7e3c98a88884d9f8919dbf37a9991bc47a4b" +checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" dependencies = [ "futures-channel", "futures-executor", @@ -2312,7 +2203,6 @@ dependencies = [ "opentelemetry", "percent-encoding", "rand 0.9.2", - "serde_json", "thiserror 2.0.16", "tokio", "tokio-stream", @@ -2431,7 +2321,7 @@ version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97baced388464909d42d89643fe4361939af9b7ce7a31ee32a168f832a70f2a0" dependencies = [ - "bitflags 2.9.3", + "bitflags", "crc32fast", "fdeflate", "flate2", @@ -2522,9 +2412,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.5" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" dependencies = [ "bytes", "prost-derive", @@ -2532,9 +2422,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.5" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" dependencies = [ "anyhow", "itertools 0.14.0", @@ -2714,7 +2604,7 @@ version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" dependencies = [ - "bitflags 2.9.3", + "bitflags", ] [[package]] @@ -2827,7 +2717,7 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" dependencies = [ - "bitflags 2.9.3", + "bitflags", "errno", "libc", "linux-raw-sys", @@ -3206,7 +3096,7 @@ dependencies = [ "bytes", "io-uring", "libc", - "mio 1.0.4", + "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", @@ -3314,9 +3204,9 @@ checksum = "fcc842091f2def52017664b53082ecbbeb5c7731092bad69d2c63050401dfd64" [[package]] name = "tonic" -version = "0.13.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" dependencies = [ "async-trait", "base64", @@ -3329,7 +3219,7 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost", + "sync_wrapper", "tokio", "tokio-stream", "tower", @@ -3338,6 +3228,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "tonic-prost" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" +dependencies = [ + "bytes", + "prost", + "tonic", +] + [[package]] name = "tower" version = "0.5.2" @@ -3363,7 +3264,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ - "bitflags 2.9.3", + "bitflags", "bytes", "futures-util", "http 1.3.1", @@ -3713,15 +3614,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - [[package]] name = "windows-sys" version = "0.52.0" @@ -3749,21 +3641,6 @@ dependencies = [ "windows-targets 0.53.3", ] -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - [[package]] name = "windows-targets" version = "0.52.6" @@ -3797,12 +3674,6 @@ dependencies = [ "windows_x86_64_msvc 0.53.0", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -3815,12 +3686,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -3833,12 +3698,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -3863,12 +3722,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -3881,12 +3734,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -3899,12 +3746,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -3917,12 +3758,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" diff --git a/Cargo.toml b/Cargo.toml index c5f2406..55305b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,12 +27,11 @@ chrono = "0.4" clap = { version = "4.5", features = ["derive"] } dialoguer = "0.11" dotenv = "0.15" -bcrypt = "0.16.0" +bcrypt = "0.17.1" image = { version = "0.25.5", default-features = false, features = ["jpeg", "png", "rayon"] } infer = "0.16" walkdir = "2.4.0" rayon = "1.5" -notify = "6.1.1" path-absolutize = "3.1" log = "0.4" env_logger = "0.11.5" @@ -41,11 +40,11 @@ prometheus = "0.13" lazy_static = "1.5" anyhow = "1.0" rand = "0.8.5" -opentelemetry = { version = "0.30.0", features = ["default", "metrics", "tracing"] } -opentelemetry_sdk = { version = "0.30.0", features = ["default", "rt-tokio-current-thread", "metrics"] } -opentelemetry-otlp = { version = "0.30.0", features = ["default", "metrics", "tracing", "grpc-tonic"] } -opentelemetry-stdout = "0.30.0" -opentelemetry-appender-log = "0.30.0" +opentelemetry = { version = "0.31.0", features = ["default", "metrics", "tracing"] } +opentelemetry_sdk = { version = "0.31.0", features = ["default", "rt-tokio-current-thread", "metrics"] } +opentelemetry-otlp = { version = "0.31.0", features = ["default", "metrics", "tracing", "grpc-tonic"] } +opentelemetry-stdout = "0.31.0" +opentelemetry-appender-log = "0.31.0" tempfile = "3.20.0" regex = "1.11.1" exif = { package = "kamadak-exif", version = "0.6.1" } \ No newline at end of file diff --git a/README.md b/README.md index e03657f..e340dc1 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,14 @@ This is an Actix-web server for serving images and videos from a filesystem. Upon first run it will generate thumbnails for all images and videos at `BASE_PATH`. +## Features +- Automatic thumbnail generation for images and videos +- EXIF data extraction and storage for photos +- File watching with NFS support (polling-based) +- Video streaming with HLS +- Tag-based organization +- Memories API for browsing photos by date + ## Environment There are a handful of required environment variables to have the API run. They should be defined where the binary is located or above it in an `.env` file. @@ -15,3 +23,6 @@ You must have `ffmpeg` installed for streaming video and generating video thumbn - `SECRET_KEY` is the *hopefully* random string to sign Tokens with - `RUST_LOG` is one of `off, error, warn, info, debug, trace`, from least to most noisy [error is default] - `EXCLUDED_DIRS` is a comma separated list of directories to exclude from the Memories API +- `WATCH_QUICK_INTERVAL_SECONDS` (optional) is the interval in seconds for quick file scans [default: 60] +- `WATCH_FULL_INTERVAL_SECONDS` (optional) is the interval in seconds for full file scans [default: 3600] + diff --git a/src/lib.rs b/src/lib.rs index 7729d98..bed9574 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,9 +14,9 @@ pub mod otel; pub mod service; pub mod state; pub mod tags; -pub mod video; #[cfg(test)] pub mod testhelpers; +pub mod video; // Re-export commonly used types pub use data::{Claims, ThumbnailRequest}; diff --git a/src/main.rs b/src/main.rs index 948c2dd..19eb487 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,8 +9,8 @@ use futures::stream::StreamExt; use lazy_static::lazy_static; use prometheus::{self, IntGauge}; use std::error::Error; -use std::sync::Mutex; -use std::sync::mpsc::channel; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, SystemTime}; use std::{collections::HashMap, io::prelude::*}; use std::{env, fs::File}; use std::{ @@ -26,10 +26,8 @@ use actix_web::{ App, HttpRequest, HttpResponse, HttpServer, Responder, delete, get, middleware, post, put, web::{self, BufMut, BytesMut}, }; -use anyhow::Context; use chrono::Utc; use diesel::sqlite::Sqlite; -use notify::{Config, EventKind, RecommendedWatcher, RecursiveMode, Watcher}; use rayon::prelude::*; use crate::auth::login; @@ -824,62 +822,214 @@ fn run_migrations( fn watch_files() { std::thread::spawn(|| { - let (wtx, wrx) = channel(); - let mut watcher = RecommendedWatcher::new(wtx, Config::default()).unwrap(); let base_str = dotenv::var("BASE_PATH").unwrap(); - let base_path = Path::new(&base_str); + let base_path = PathBuf::from(&base_str); - watcher - .watch(base_path, RecursiveMode::Recursive) - .context(format!("Unable to watch BASE_PATH: '{}'", base_str)) - .unwrap(); + // Get polling intervals from environment variables + // Quick scan: Check recently modified files (default: 60 seconds) + let quick_interval_secs = dotenv::var("WATCH_QUICK_INTERVAL_SECONDS") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(60); + + // Full scan: Check all files regardless of modification time (default: 3600 seconds = 1 hour) + let full_interval_secs = dotenv::var("WATCH_FULL_INTERVAL_SECONDS") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(3600); + + info!("Starting optimized file watcher"); + info!(" Quick scan interval: {} seconds", quick_interval_secs); + info!(" Full scan interval: {} seconds", full_interval_secs); + info!(" Watching directory: {}", base_str); + + // Create EXIF DAO for tracking processed files + let exif_dao = Arc::new(Mutex::new( + Box::new(SqliteExifDao::new()) as Box + )); + + let mut last_quick_scan = SystemTime::now(); + let mut last_full_scan = SystemTime::now(); + let mut scan_count = 0u64; loop { - let ev = wrx.recv(); - if let Ok(Ok(event)) = ev { - match event.kind { - EventKind::Create(create_kind) => { - info!( - "Creating thumbnails {:?} create event kind: {:?}", - event.paths, create_kind - ); - create_thumbnails(); - } - EventKind::Modify(kind) => { - debug!("All modified paths: {:?}", event.paths); - debug!("Modify kind: {:?}", kind); + std::thread::sleep(Duration::from_secs(quick_interval_secs)); - if let Some(orig) = event.paths.first() { - let image_base_path = PathBuf::from(env::var("BASE_PATH").unwrap()); - let image_relative = orig.strip_prefix(&image_base_path).unwrap(); - if let Ok(old_thumbnail) = - env::var("THUMBNAILS").map(PathBuf::from).map(|mut base| { - base.push(image_relative); - base - }) - { - if let Err(e) = std::fs::remove_file(&old_thumbnail) { - error!( - "Error removing thumbnail: {}\n{}", - old_thumbnail.display(), - e - ); - } else { - info!("Deleted moved thumbnail: {}", old_thumbnail.display()); + let now = SystemTime::now(); + let since_last_full = now + .duration_since(last_full_scan) + .unwrap_or(Duration::from_secs(0)); - create_thumbnails(); - } - } - } - } + let is_full_scan = since_last_full.as_secs() >= full_interval_secs; - EventKind::Remove(_) => { - update_media_counts(&PathBuf::from(env::var("BASE_PATH").unwrap())) - } + if is_full_scan { + info!("Running full scan (scan #{})", scan_count); + process_new_files(&base_path, Arc::clone(&exif_dao), None); + last_full_scan = now; + } else { + debug!( + "Running quick scan (checking files modified in last {} seconds)", + quick_interval_secs + 10 + ); + // Check files modified since last quick scan, plus 10 second buffer + let check_since = last_quick_scan + .checked_sub(Duration::from_secs(10)) + .unwrap_or(last_quick_scan); + process_new_files(&base_path, Arc::clone(&exif_dao), Some(check_since)); + } - _ => {} - } - }; + last_quick_scan = now; + scan_count += 1; + + // Update media counts + update_media_counts(&base_path); } }); } + +fn process_new_files( + base_path: &Path, + exif_dao: Arc>>, + modified_since: Option, +) { + let thumbs = dotenv::var("THUMBNAILS").expect("THUMBNAILS not defined"); + let thumbnail_directory = Path::new(&thumbs); + + // Collect all image and video files, optionally filtered by modification time + let files: Vec<(PathBuf, String)> = WalkDir::new(base_path) + .into_iter() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.file_type().is_file()) + .filter(|entry| { + // Filter by modification time if specified + if let Some(since) = modified_since { + if let Ok(metadata) = entry.metadata() { + if let Ok(modified) = metadata.modified() { + return modified >= since; + } + } + // If we can't get metadata, include the file to be safe + return true; + } + true + }) + .filter(|entry| is_image(entry) || is_video(entry)) + .filter_map(|entry| { + let file_path = entry.path().to_path_buf(); + let relative_path = file_path + .strip_prefix(base_path) + .ok()? + .to_str()? + .to_string(); + Some((file_path, relative_path)) + }) + .collect(); + + if files.is_empty() { + debug!("No files to process"); + return; + } + + debug!("Found {} files to check", files.len()); + + // Batch query: Get all EXIF data for these files in one query + let file_paths: Vec = files.iter().map(|(_, rel_path)| rel_path.clone()).collect(); + + let existing_exif_paths: HashMap = { + let mut dao = exif_dao.lock().expect("Unable to lock ExifDao"); + match dao.get_exif_batch(&file_paths) { + Ok(exif_records) => exif_records + .into_iter() + .map(|record| (record.file_path, true)) + .collect(), + Err(e) => { + error!("Error batch querying EXIF data: {:?}", e); + HashMap::new() + } + } + }; + + let mut new_files_found = false; + let mut files_needing_exif = Vec::new(); + + // Check each file for missing thumbnail or EXIF data + for (file_path, relative_path) in files { + // Check if thumbnail exists + let thumb_path = thumbnail_directory.join(&relative_path); + let needs_thumbnail = !thumb_path.exists(); + + // Check if EXIF data exists (for supported files) + let needs_exif = if exif::supports_exif(&file_path) { + !existing_exif_paths.contains_key(&relative_path) + } else { + false + }; + + if needs_thumbnail || needs_exif { + new_files_found = true; + + if needs_thumbnail { + info!("New file detected (missing thumbnail): {}", relative_path); + } + + if needs_exif { + files_needing_exif.push((file_path, relative_path)); + } + } + } + + // Process EXIF data for files that need it + if !files_needing_exif.is_empty() { + info!( + "Processing EXIF data for {} files", + files_needing_exif.len() + ); + + for (file_path, relative_path) in files_needing_exif { + match exif::extract_exif_from_path(&file_path) { + Ok(exif_data) => { + let timestamp = Utc::now().timestamp(); + let insert_exif = InsertImageExif { + file_path: relative_path.clone(), + camera_make: exif_data.camera_make, + camera_model: exif_data.camera_model, + lens_model: exif_data.lens_model, + width: exif_data.width, + height: exif_data.height, + orientation: exif_data.orientation, + gps_latitude: exif_data.gps_latitude, + gps_longitude: exif_data.gps_longitude, + gps_altitude: exif_data.gps_altitude, + focal_length: exif_data.focal_length, + aperture: exif_data.aperture, + shutter_speed: exif_data.shutter_speed, + iso: exif_data.iso, + date_taken: exif_data.date_taken, + created_time: timestamp, + last_modified: timestamp, + }; + + let mut dao = exif_dao.lock().expect("Unable to lock ExifDao"); + if let Err(e) = dao.store_exif(insert_exif) { + error!("Failed to store EXIF data for {}: {:?}", relative_path, e); + } else { + debug!("EXIF data stored for {}", relative_path); + } + } + Err(e) => { + debug!( + "No EXIF data or error extracting from {}: {:?}", + file_path.display(), + e + ); + } + } + } + } + + // Generate thumbnails for all files that need them + if new_files_found { + info!("Processing thumbnails for new files..."); + create_thumbnails(); + } +}