Apps/ImageApi
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Add polling-based file watching

Browse Source 
Remove notify and update otel creates
This commit is contained in:
Cameron
2025-12-22 22:54:19 -05:00
parent df94010d21
commit 47d3ad7222
5 changed files with 267 additions and 272 deletions
Download Patch File Download Diff File Expand all files Collapse all files

259
Cargo.lock generated
View File

@@ -11,7 +11,7 @@ dependencies = [
"actix-macros",
"actix-rt",
"actix_derive",
"bitflags 2.9.3",
"bitflags",
"bytes",
"crossbeam-channel",
"futures-core",
@@ -33,7 +33,7 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a"
dependencies = [
"bitflags 2.9.3",
"bitflags",
"bytes",
"futures-core",
"futures-sink",
@@ -69,7 +69,7 @@ dependencies = [
"actix-service",
"actix-utils",
"actix-web",
"bitflags 2.9.3",
"bitflags",
"bytes",
"derive_more 2.0.1",
"futures-core",
@@ -93,7 +93,7 @@ dependencies = [
"actix-service",
"actix-utils",
"base64",
"bitflags 2.9.3",
"bitflags",
"brotli",
"bytes",
"bytestring",
@@ -206,7 +206,7 @@ dependencies = [
"actix-utils",
"futures-core",
"futures-util",
"mio 1.0.4",
"mio",
"socket2 0.5.10",
"tokio",
"tracing",
@@ -524,23 +524,17 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "bcrypt"
version = "0.16.0"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b1866ecef4f2d06a0bb77880015fdf2b89e25a1c2e5addacb87e459c86dc67e"
checksum = "abaf6da45c74385272ddf00e1ac074c7d8a6c1a1dda376902bd6a427522a8b2c"
dependencies = [
"base64",
"blowfish",
"getrandom 0.2.16",
"getrandom 0.3.3",
"subtle",
"zeroize",
]
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.9.3"
@@ -1116,18 +1110,6 @@ dependencies = [
"simd-adler32",
]
[[package]]
name = "filetime"
version = "0.2.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc0505cd1b6fa6580283f6bdf70a73fcf4aba1184038c90902b92b3dd0df63ed"
dependencies = [
"cfg-if",
"libc",
"libredox",
"windows-sys 0.60.2",
]
[[package]]
name = "find-msvc-tools"
version = "0.1.0"
@@ -1165,15 +1147,6 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "fsevent-sys"
version = "4.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2"
dependencies = [
"libc",
]
[[package]]
name = "futures"
version = "0.3.31"
@@ -1657,7 +1630,6 @@ dependencies = [
"kamadak-exif",
"lazy_static",
"log",
"notify",
"opentelemetry",
"opentelemetry-appender-log",
"opentelemetry-otlp",
@@ -1706,26 +1678,6 @@ dependencies = [
"cfb",
]
[[package]]
name = "inotify"
version = "0.9.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff"
dependencies = [
"bitflags 1.3.2",
"inotify-sys",
"libc",
]
[[package]]
name = "inotify-sys"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb"
dependencies = [
"libc",
]
[[package]]
name = "inout"
version = "0.1.4"
@@ -1752,7 +1704,7 @@ version = "0.7.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b"
dependencies = [
"bitflags 2.9.3",
"bitflags",
"cfg-if",
"libc",
]
@@ -1871,26 +1823,6 @@ dependencies = [
"mutate_once",
]
[[package]]
name = "kqueue"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a"
dependencies = [
"kqueue-sys",
"libc",
]
[[package]]
name = "kqueue-sys"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b"
dependencies = [
"bitflags 1.3.2",
"libc",
]
[[package]]
name = "language-tags"
version = "0.3.2"
@@ -1919,17 +1851,6 @@ dependencies = [
"cc",
]
[[package]]
name = "libredox"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3"
dependencies = [
"bitflags 2.9.3",
"libc",
"redox_syscall",
]
[[package]]
name = "libsqlite3-sys"
version = "0.35.0"
@@ -2063,18 +1984,6 @@ dependencies = [
"simd-adler32",
]
[[package]]
name = "mio"
version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
dependencies = [
"libc",
"log",
"wasi 0.11.1+wasi-snapshot-preview1",
"windows-sys 0.48.0",
]
[[package]]
name = "mio"
version = "1.0.4"
@@ -2125,25 +2034,6 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8"
[[package]]
name = "notify"
version = "6.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d"
dependencies = [
"bitflags 2.9.3",
"crossbeam-channel",
"filetime",
"fsevent-sys",
"inotify",
"kqueue",
"libc",
"log",
"mio 0.8.11",
"walkdir",
"windows-sys 0.48.0",
]
[[package]]
name = "num-bigint"
version = "0.4.6"
@@ -2223,9 +2113,9 @@ checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad"
[[package]]
name = "opentelemetry"
version = "0.30.0"
version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aaf416e4cb72756655126f7dd7bb0af49c674f4c1b9903e80c009e0c37e552e6"
checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0"
dependencies = [
"futures-core",
"futures-sink",
@@ -2237,9 +2127,9 @@ dependencies = [
[[package]]
name = "opentelemetry-appender-log"
version = "0.30.0"
version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e688026e48f4603494f619583e0aa0b0edd9c0b9430e1c46804df2ff32bc8798"
checksum = "9e50c59a96bd6a723a4329c5db31eb04fa4488c5f141ae7b9d4fd587439e6ee1"
dependencies = [
"log",
"opentelemetry",
@@ -2247,9 +2137,9 @@ dependencies = [
[[package]]
name = "opentelemetry-http"
version = "0.30.0"
version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50f6639e842a97dbea8886e3439710ae463120091e2e064518ba8e716e6ac36d"
checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d"
dependencies = [
"async-trait",
"bytes",
@@ -2260,9 +2150,9 @@ dependencies = [
[[package]]
name = "opentelemetry-otlp"
version = "0.30.0"
version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbee664a43e07615731afc539ca60c6d9f1a9425e25ca09c57bc36c87c55852b"
checksum = "7a2366db2dca4d2ad033cad11e6ee42844fd727007af5ad04a1730f4cb8163bf"
dependencies = [
"http 1.3.1",
"opentelemetry",
@@ -2279,21 +2169,22 @@ dependencies = [
[[package]]
name = "opentelemetry-proto"
version = "0.30.0"
version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e046fd7660710fe5a05e8748e70d9058dc15c94ba914e7c4faa7c728f0e8ddc"
checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f"
dependencies = [
"opentelemetry",
"opentelemetry_sdk",
"prost",
"tonic",
"tonic-prost",
]
[[package]]
name = "opentelemetry-stdout"
version = "0.30.0"
version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "447191061af41c3943e082ea359ab8b64ff27d6d34d30d327df309ddef1eef6f"
checksum = "bc8887887e169414f637b18751487cce4e095be787d23fad13c454e2fb1b3811"
dependencies = [
"chrono",
"opentelemetry",
@@ -2302,9 +2193,9 @@ dependencies = [
[[package]]
name = "opentelemetry_sdk"
version = "0.30.0"
version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11f644aa9e5e31d11896e024305d7e3c98a88884d9f8919dbf37a9991bc47a4b"
checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd"
dependencies = [
"futures-channel",
"futures-executor",
@@ -2312,7 +2203,6 @@ dependencies = [
"opentelemetry",
"percent-encoding",
"rand 0.9.2",
"serde_json",
"thiserror 2.0.16",
"tokio",
"tokio-stream",
@@ -2431,7 +2321,7 @@ version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97baced388464909d42d89643fe4361939af9b7ce7a31ee32a168f832a70f2a0"
dependencies = [
"bitflags 2.9.3",
"bitflags",
"crc32fast",
"fdeflate",
"flate2",
@@ -2522,9 +2412,9 @@ dependencies = [
[[package]]
name = "prost"
version = "0.13.5"
version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5"
checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d"
dependencies = [
"bytes",
"prost-derive",
@@ -2532,9 +2422,9 @@ dependencies = [
[[package]]
name = "prost-derive"
version = "0.13.5"
version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -2714,7 +2604,7 @@ version = "0.5.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77"
dependencies = [
"bitflags 2.9.3",
"bitflags",
]
[[package]]
@@ -2827,7 +2717,7 @@ version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8"
dependencies = [
"bitflags 2.9.3",
"bitflags",
"errno",
"libc",
"linux-raw-sys",
@@ -3206,7 +3096,7 @@ dependencies = [
"bytes",
"io-uring",
"libc",
"mio 1.0.4",
"mio",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
@@ -3314,9 +3204,9 @@ checksum = "fcc842091f2def52017664b53082ecbbeb5c7731092bad69d2c63050401dfd64"
[[package]]
name = "tonic"
version = "0.13.1"
version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9"
checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203"
dependencies = [
"async-trait",
"base64",
@@ -3329,7 +3219,7 @@ dependencies = [
"hyper-util",
"percent-encoding",
"pin-project",
"prost",
"sync_wrapper",
"tokio",
"tokio-stream",
"tower",
@@ -3338,6 +3228,17 @@ dependencies = [
"tracing",
]
[[package]]
name = "tonic-prost"
version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67"
dependencies = [
"bytes",
"prost",
"tonic",
]
[[package]]
name = "tower"
version = "0.5.2"
@@ -3363,7 +3264,7 @@ version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2"
dependencies = [
"bitflags 2.9.3",
"bitflags",
"bytes",
"futures-util",
"http 1.3.1",
@@ -3713,15 +3614,6 @@ dependencies = [
"windows-link",
]
[[package]]
name = "windows-sys"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
dependencies = [
"windows-targets 0.48.5",
]
[[package]]
name = "windows-sys"
version = "0.52.0"
@@ -3749,21 +3641,6 @@ dependencies = [
"windows-targets 0.53.3",
]
[[package]]
name = "windows-targets"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
dependencies = [
"windows_aarch64_gnullvm 0.48.5",
"windows_aarch64_msvc 0.48.5",
"windows_i686_gnu 0.48.5",
"windows_i686_msvc 0.48.5",
"windows_x86_64_gnu 0.48.5",
"windows_x86_64_gnullvm 0.48.5",
"windows_x86_64_msvc 0.48.5",
]
[[package]]
name = "windows-targets"
version = "0.52.6"
@@ -3797,12 +3674,6 @@ dependencies = [
"windows_x86_64_msvc 0.53.0",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.52.6"
@@ -3815,12 +3686,6 @@ version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764"
[[package]]
name = "windows_aarch64_msvc"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
@@ -3833,12 +3698,6 @@ version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c"
[[package]]
name = "windows_i686_gnu"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
@@ -3863,12 +3722,6 @@ version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11"
[[package]]
name = "windows_i686_msvc"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
@@ -3881,12 +3734,6 @@ version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d"
[[package]]
name = "windows_x86_64_gnu"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
@@ -3899,12 +3746,6 @@ version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
@@ -3917,12 +3758,6 @@ version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57"
[[package]]
name = "windows_x86_64_msvc"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"

13
Cargo.toml
View File

@@ -27,12 +27,11 @@ chrono = "0.4"
clap = { version = "4.5", features = ["derive"] }
dialoguer = "0.11"
dotenv = "0.15"
bcrypt = "0.16.0"
bcrypt = "0.17.1"
image = { version = "0.25.5", default-features = false, features = ["jpeg", "png", "rayon"] }
infer = "0.16"
walkdir = "2.4.0"
rayon = "1.5"
notify = "6.1.1"
path-absolutize = "3.1"
log = "0.4"
env_logger = "0.11.5"
@@ -41,11 +40,11 @@ prometheus = "0.13"
lazy_static = "1.5"
anyhow = "1.0"
rand = "0.8.5"
opentelemetry = { version = "0.30.0", features = ["default", "metrics", "tracing"] }
opentelemetry_sdk = { version = "0.30.0", features = ["default", "rt-tokio-current-thread", "metrics"] }
opentelemetry-otlp = { version = "0.30.0", features = ["default", "metrics", "tracing", "grpc-tonic"] }
opentelemetry-stdout = "0.30.0"
opentelemetry-appender-log = "0.30.0"
opentelemetry = { version = "0.31.0", features = ["default", "metrics", "tracing"] }
opentelemetry_sdk = { version = "0.31.0", features = ["default", "rt-tokio-current-thread", "metrics"] }
opentelemetry-otlp = { version = "0.31.0", features = ["default", "metrics", "tracing", "grpc-tonic"] }
opentelemetry-stdout = "0.31.0"
opentelemetry-appender-log = "0.31.0"
tempfile = "3.20.0"
regex = "1.11.1"
exif = { package = "kamadak-exif", version = "0.6.1" }

11
README.md
View File

@@ -2,6 +2,14 @@
This is an Actix-web server for serving images and videos from a filesystem.
Upon first run it will generate thumbnails for all images and videos at `BASE_PATH`.
## Features
- Automatic thumbnail generation for images and videos
- EXIF data extraction and storage for photos
- File watching with NFS support (polling-based)
- Video streaming with HLS
- Tag-based organization
- Memories API for browsing photos by date
## Environment
There are a handful of required environment variables to have the API run.
They should be defined where the binary is located or above it in an `.env` file.
@@ -15,3 +23,6 @@ You must have `ffmpeg` installed for streaming video and generating video thumbn
- `SECRET_KEY` is the *hopefully* random string to sign Tokens with
- `RUST_LOG` is one of `off, error, warn, info, debug, trace`, from least to most noisy [error is default]
- `EXCLUDED_DIRS` is a comma separated list of directories to exclude from the Memories API
- `WATCH_QUICK_INTERVAL_SECONDS` (optional) is the interval in seconds for quick file scans [default: 60]
- `WATCH_FULL_INTERVAL_SECONDS` (optional) is the interval in seconds for full file scans [default: 3600]

2
src/lib.rs
View File

@@ -14,9 +14,9 @@ pub mod otel;
pub mod service;
pub mod state;
pub mod tags;
pub mod video;
#[cfg(test)]
pub mod testhelpers;
pub mod video;
// Re-export commonly used types
pub use data::{Claims, ThumbnailRequest};

252
src/main.rs
View File

@@ -9,8 +9,8 @@ use futures::stream::StreamExt;
use lazy_static::lazy_static;
use prometheus::{self, IntGauge};
use std::error::Error;
use std::sync::Mutex;
use std::sync::mpsc::channel;
use std::sync::{Arc, Mutex};
use std::time::{Duration, SystemTime};
use std::{collections::HashMap, io::prelude::*};
use std::{env, fs::File};
use std::{
@@ -26,10 +26,8 @@ use actix_web::{
App, HttpRequest, HttpResponse, HttpServer, Responder, delete, get, middleware, post, put,
web::{self, BufMut, BytesMut},
};
use anyhow::Context;
use chrono::Utc;
use diesel::sqlite::Sqlite;
use notify::{Config, EventKind, RecommendedWatcher, RecursiveMode, Watcher};
use rayon::prelude::*;
use crate::auth::login;
@@ -824,62 +822,214 @@ fn run_migrations(
fn watch_files() {
std::thread::spawn(|| {
let (wtx, wrx) = channel();
let mut watcher = RecommendedWatcher::new(wtx, Config::default()).unwrap();
let base_str = dotenv::var("BASE_PATH").unwrap();
let base_path = Path::new(&base_str);
let base_path = PathBuf::from(&base_str);
watcher
.watch(base_path, RecursiveMode::Recursive)
.context(format!("Unable to watch BASE_PATH: '{}'", base_str))
.unwrap();
// Get polling intervals from environment variables
// Quick scan: Check recently modified files (default: 60 seconds)
let quick_interval_secs = dotenv::var("WATCH_QUICK_INTERVAL_SECONDS")
.ok()
.and_then(|s| s.parse::<u64>().ok())
.unwrap_or(60);
// Full scan: Check all files regardless of modification time (default: 3600 seconds = 1 hour)
let full_interval_secs = dotenv::var("WATCH_FULL_INTERVAL_SECONDS")
.ok()
.and_then(|s| s.parse::<u64>().ok())
.unwrap_or(3600);
info!("Starting optimized file watcher");
info!(" Quick scan interval: {} seconds", quick_interval_secs);
info!(" Full scan interval: {} seconds", full_interval_secs);
info!(" Watching directory: {}", base_str);
// Create EXIF DAO for tracking processed files
let exif_dao = Arc::new(Mutex::new(
Box::new(SqliteExifDao::new()) as Box<dyn ExifDao>
));
let mut last_quick_scan = SystemTime::now();
let mut last_full_scan = SystemTime::now();
let mut scan_count = 0u64;
loop {
let ev = wrx.recv();
if let Ok(Ok(event)) = ev {
match event.kind {
EventKind::Create(create_kind) => {
info!(
"Creating thumbnails {:?} create event kind: {:?}",
event.paths, create_kind
);
create_thumbnails();
}
EventKind::Modify(kind) => {
debug!("All modified paths: {:?}", event.paths);
debug!("Modify kind: {:?}", kind);
std::thread::sleep(Duration::from_secs(quick_interval_secs));
if let Some(orig) = event.paths.first() {
let image_base_path = PathBuf::from(env::var("BASE_PATH").unwrap());
let image_relative = orig.strip_prefix(&image_base_path).unwrap();
if let Ok(old_thumbnail) =
env::var("THUMBNAILS").map(PathBuf::from).map(|mut base| {
base.push(image_relative);
base
})
{
if let Err(e) = std::fs::remove_file(&old_thumbnail) {
error!(
"Error removing thumbnail: {}\n{}",
old_thumbnail.display(),
e
);
let now = SystemTime::now();
let since_last_full = now
.duration_since(last_full_scan)
.unwrap_or(Duration::from_secs(0));
let is_full_scan = since_last_full.as_secs() >= full_interval_secs;
if is_full_scan {
info!("Running full scan (scan #{})", scan_count);
process_new_files(&base_path, Arc::clone(&exif_dao), None);
last_full_scan = now;
} else {
info!("Deleted moved thumbnail: {}", old_thumbnail.display());
create_thumbnails();
}
}
}
debug!(
"Running quick scan (checking files modified in last {} seconds)",
quick_interval_secs + 10
);
// Check files modified since last quick scan, plus 10 second buffer
let check_since = last_quick_scan
.checked_sub(Duration::from_secs(10))
.unwrap_or(last_quick_scan);
process_new_files(&base_path, Arc::clone(&exif_dao), Some(check_since));
}
EventKind::Remove(_) => {
update_media_counts(&PathBuf::from(env::var("BASE_PATH").unwrap()))
}
last_quick_scan = now;
scan_count += 1;
_ => {}
}
};
// Update media counts
update_media_counts(&base_path);
}
});
}
fn process_new_files(
base_path: &Path,
exif_dao: Arc<Mutex<Box<dyn ExifDao>>>,
modified_since: Option<SystemTime>,
) {
let thumbs = dotenv::var("THUMBNAILS").expect("THUMBNAILS not defined");
let thumbnail_directory = Path::new(&thumbs);
// Collect all image and video files, optionally filtered by modification time
let files: Vec<(PathBuf, String)> = WalkDir::new(base_path)
.into_iter()
.filter_map(|entry| entry.ok())
.filter(|entry| entry.file_type().is_file())
.filter(|entry| {
// Filter by modification time if specified
if let Some(since) = modified_since {
if let Ok(metadata) = entry.metadata() {
if let Ok(modified) = metadata.modified() {
return modified >= since;
}
}
// If we can't get metadata, include the file to be safe
return true;
}
true
})
.filter(|entry| is_image(entry) || is_video(entry))
.filter_map(|entry| {
let file_path = entry.path().to_path_buf();
let relative_path = file_path
.strip_prefix(base_path)
.ok()?
.to_str()?
.to_string();
Some((file_path, relative_path))
})
.collect();
if files.is_empty() {
debug!("No files to process");
return;
}
debug!("Found {} files to check", files.len());
// Batch query: Get all EXIF data for these files in one query
let file_paths: Vec<String> = files.iter().map(|(_, rel_path)| rel_path.clone()).collect();
let existing_exif_paths: HashMap<String, bool> = {
let mut dao = exif_dao.lock().expect("Unable to lock ExifDao");
match dao.get_exif_batch(&file_paths) {
Ok(exif_records) => exif_records
.into_iter()
.map(|record| (record.file_path, true))
.collect(),
Err(e) => {
error!("Error batch querying EXIF data: {:?}", e);
HashMap::new()
}
}
};
let mut new_files_found = false;
let mut files_needing_exif = Vec::new();
// Check each file for missing thumbnail or EXIF data
for (file_path, relative_path) in files {
// Check if thumbnail exists
let thumb_path = thumbnail_directory.join(&relative_path);
let needs_thumbnail = !thumb_path.exists();
// Check if EXIF data exists (for supported files)
let needs_exif = if exif::supports_exif(&file_path) {
!existing_exif_paths.contains_key(&relative_path)
} else {
false
};
if needs_thumbnail || needs_exif {
new_files_found = true;
if needs_thumbnail {
info!("New file detected (missing thumbnail): {}", relative_path);
}
if needs_exif {
files_needing_exif.push((file_path, relative_path));
}
}
}
// Process EXIF data for files that need it
if !files_needing_exif.is_empty() {
info!(
"Processing EXIF data for {} files",
files_needing_exif.len()
);
for (file_path, relative_path) in files_needing_exif {
match exif::extract_exif_from_path(&file_path) {
Ok(exif_data) => {
let timestamp = Utc::now().timestamp();
let insert_exif = InsertImageExif {
file_path: relative_path.clone(),
camera_make: exif_data.camera_make,
camera_model: exif_data.camera_model,
lens_model: exif_data.lens_model,
width: exif_data.width,
height: exif_data.height,
orientation: exif_data.orientation,
gps_latitude: exif_data.gps_latitude,
gps_longitude: exif_data.gps_longitude,
gps_altitude: exif_data.gps_altitude,
focal_length: exif_data.focal_length,
aperture: exif_data.aperture,
shutter_speed: exif_data.shutter_speed,
iso: exif_data.iso,
date_taken: exif_data.date_taken,
created_time: timestamp,
last_modified: timestamp,
};
let mut dao = exif_dao.lock().expect("Unable to lock ExifDao");
if let Err(e) = dao.store_exif(insert_exif) {
error!("Failed to store EXIF data for {}: {:?}", relative_path, e);
} else {
debug!("EXIF data stored for {}", relative_path);
}
}
Err(e) => {
debug!(
"No EXIF data or error extracting from {}: {:?}",
file_path.display(),
e
);
}
}
}
}
// Generate thumbnails for all files that need them
if new_files_found {
info!("Processing thumbnails for new files...");
create_thumbnails();
}
}