Generate thumbnails before API starts up

Clients can now request a thumbnail by specifying a size on the image request.
2020-07-15 12:06:50 -04:00
parent ca761f605b
commit 7d369aab5c
5 changed files with 305 additions and 8 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -473,6 +473,12 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"

+[[package]]
+name = "bytemuck"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37fa13df2292ecb479ec23aa06f4507928bef07839be9ef15281411076629431"
+
 [[package]]
 name = "byteorder"
 version = "1.3.4"
@@ -529,6 +535,12 @@ dependencies = [
 "bitflags",
 ]

+[[package]]
+name = "color_quant"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0dbbb57365263e881e805dc77d94697c9118fd94d8da011240555aa7b23445bd"
+
 [[package]]
 name = "copyless"
 version = "0.1.5"
@@ -544,6 +556,54 @@ dependencies = [
 "cfg-if",
 ]

+[[package]]
+name = "crossbeam-deque"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+ "maybe-uninit",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace"
+dependencies = [
+ "autocfg",
+ "cfg-if",
+ "crossbeam-utils",
+ "lazy_static",
+ "maybe-uninit",
+ "memoffset",
+ "scopeguard",
+]
+
+[[package]]
+name = "crossbeam-queue"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+ "maybe-uninit",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
+dependencies = [
+ "autocfg",
+ "cfg-if",
+ "lazy_static",
+]
+
 [[package]]
 name = "crypto-mac"
 version = "0.7.0"
@@ -554,6 +614,16 @@ dependencies = [
 "subtle",
 ]

+[[package]]
+name = "deflate"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73770f8e1fe7d64df17ca66ad28994a0a623ea497fa69486e14984e715c5d174"
+dependencies = [
+ "adler32",
+ "byteorder",
+]
+
 [[package]]
 name = "derive_more"
 version = "0.99.9"
@@ -844,6 +914,16 @@ dependencies = [
 "wasi",
 ]

+[[package]]
+name = "gif"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "471d90201b3b223f3451cd4ad53e34295f16a1df17b1edf3736d47761c3981af"
+dependencies = [
+ "color_quant",
+ "lzw",
+]
+
 [[package]]
 name = "gimli"
 version = "0.21.0"
@@ -936,6 +1016,24 @@ dependencies = [
 "unicode-normalization",
 ]

+[[package]]
+name = "image"
+version = "0.23.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2397fc43bd5648b7117aabb3c5e62d0e62c194826ec77b0b4d0c41e62744635"
+dependencies = [
+ "bytemuck",
+ "byteorder",
+ "gif",
+ "jpeg-decoder",
+ "num-iter",
+ "num-rational",
+ "num-traits",
+ "png",
+ "scoped_threadpool",
+ "tiff",
+]
+
 [[package]]
 name = "image-api"
 version = "0.1.0"
@@ -949,10 +1047,13 @@ dependencies = [
 "dotenv",
 "futures",
 "hmac",
+ "image",
 "jsonwebtoken",
+ "rayon",
 "serde",
 "serde_json",
 "sha2",
+ "walkdir",
 ]

 [[package]]
@@ -991,6 +1092,16 @@ version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6"

+[[package]]
+name = "jpeg-decoder"
+version = "0.1.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc797adac5f083b8ff0ca6f6294a999393d76e197c36488e2ef732c4715f6fa3"
+dependencies = [
+ "byteorder",
+ "rayon",
+]
+
 [[package]]
 name = "js-sys"
 version = "0.3.41"
@@ -1096,6 +1207,12 @@ dependencies = [
 "linked-hash-map",
 ]

+[[package]]
+name = "lzw"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d947cbb889ed21c2a84be6ffbaebf5b4e0f4340638cba0444907e38b56be084"
+
 [[package]]
 name = "match_cfg"
 version = "0.1.0"
@@ -1108,12 +1225,27 @@ version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"

+[[package]]
+name = "maybe-uninit"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
+
 [[package]]
 name = "memchr"
 version = "2.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400"

+[[package]]
+name = "memoffset"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c198b026e1bbf08a937e94c6c60f9ec4a2267f5b0d2eec9c1b21b061ce2be55f"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.16"
@@ -1232,6 +1364,28 @@ dependencies = [
 "num-traits",
 ]

+[[package]]
+name = "num-iter"
+version = "0.1.41"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a6e6b7c748f995c4c29c5f5ae0248536e04a5739927c74ec0fa564805094b9f"
+dependencies = [
+ "autocfg",
+ "num-integer",
+ "num-traits",
+]
+
+[[package]]
+name = "num-rational"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5b4d7360f362cfb50dde8143501e6940b22f644be75a4cc90b2d81968908138"
+dependencies = [
+ "autocfg",
+ "num-integer",
+ "num-traits",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.12"
@@ -1348,6 +1502,18 @@ version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677"

+[[package]]
+name = "png"
+version = "0.16.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c150bf7479fafe3dd8740dbe48cc33b2a3efb7b0fe3483aced8bbc39f6d0238d"
+dependencies = [
+ "bitflags",
+ "crc32fast",
+ "deflate",
+ "miniz_oxide 0.3.7",
+]
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.8"
@@ -1431,6 +1597,31 @@ dependencies = [
 "rand_core",
 ]

+[[package]]
+name = "rayon"
+version = "1.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62f02856753d04e03e26929f820d0a0a337ebe71f849801eea335d464b349080"
+dependencies = [
+ "autocfg",
+ "crossbeam-deque",
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e92e15d89083484e11353891f1af602cc661426deb9564c298b270c726973280"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-queue",
+ "crossbeam-utils",
+ "lazy_static",
+ "num_cpus",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.1.56"
@@ -1501,12 +1692,27 @@ version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"

+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
 [[package]]
 name = "scoped-tls"
 version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "332ffa32bf586782a3efaeb58f127980944bbc8c4d6913a86107ac2a5ab24b28"

+[[package]]
+name = "scoped_threadpool"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8"
+
 [[package]]
 name = "scopeguard"
 version = "1.1.0"
@@ -1687,6 +1893,17 @@ dependencies = [
 "num_cpus",
 ]

+[[package]]
+name = "tiff"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f3b8a87c4da944c3f27e5943289171ac71a6150a79ff6bacfff06d159dfff2f"
+dependencies = [
+ "byteorder",
+ "lzw",
+ "miniz_oxide 0.3.7",
+]
+
 [[package]]
 name = "time"
 version = "0.1.43"
@@ -1901,6 +2118,17 @@ version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed"

+[[package]]
+name = "walkdir"
+version = "2.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d"
+dependencies = [
+ "same-file",
+ "winapi 0.3.9",
+ "winapi-util",
+]
+
 [[package]]
 name = "wasi"
 version = "0.9.0+wasi-snapshot-preview1"
@@ -2005,6 +2233,15 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"

+[[package]]
+name = "winapi-util"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+dependencies = [
+ "winapi 0.3.9",
+]
+
 [[package]]
 name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -20,3 +20,6 @@ sha2 = "0.8.2"
 chrono = "0.4.11"
 dotenv = "0.15"
 bcrypt = "0.8.1"
+image = "0.23.7"
+walkdir = "2"
+rayon = "1.3"
--- a/src/data/mod.rs
+++ b/src/data/mod.rs
@@ -60,6 +60,7 @@ impl FromRequest for Claims {
 #[derive(Deserialize)]
 pub struct ThumbnailRequest {
    pub path: String,
+    pub size: Option<String>,
 }

 #[derive(Deserialize)]
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,5 +1,6 @@
 #[macro_use]
 extern crate diesel;
+extern crate rayon;

 use actix_files::NamedFile;
 use actix_web::web::{HttpRequest, HttpResponse, Json};
@@ -7,8 +8,9 @@ use actix_web::{get, post, web, App, HttpServer, Responder};
 use chrono::{Duration, Utc};
 use data::{LoginRequest, ThumbnailRequest};
 use jsonwebtoken::{encode, EncodingKey, Header};
+use rayon::prelude::*;
 use serde::Serialize;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};

 use crate::data::{Claims, CreateAccountRequest, Token};
 use crate::database::{create_user, get_user, user_exists};
@@ -105,16 +107,28 @@ fn is_valid_path(path: &str) -> Option<PathBuf> {
 }

 #[get("/image")]
-async fn image(
+async fn get_image(
    _claims: Claims,
    request: HttpRequest,
    req: web::Query<ThumbnailRequest>,
 ) -> impl Responder {
    if let Some(path) = is_valid_path(&req.path) {
-        if let Ok(file) = NamedFile::open(path) {
-            file.into_response(&request).unwrap()
+        if let Some(_) = req.size {
+            let thumbs = dotenv::var("THUMBNAILS").unwrap();
+            let thumb_path = Path::new(&thumbs).join(path.file_name().unwrap());
+
+            println!("{:?}", thumb_path);
+            if let Ok(file) = NamedFile::open(&thumb_path) {
+                file.into_response(&request).unwrap()
+            } else {
+                HttpResponse::NotFound().finish()
+            }
        } else {
-            HttpResponse::NotFound().finish()
+            if let Ok(file) = NamedFile::open(path) {
+                file.into_response(&request).unwrap()
+            } else {
+                HttpResponse::NotFound().finish()
+            }
        }
    } else {
        HttpResponse::BadRequest().finish()
@@ -164,14 +178,57 @@ async fn get_video_part(request: HttpRequest, path: web::Path<ThumbnailRequest>)
    }
 }

+async fn create_thumbnails() {
+    let thumbs = &dotenv::var("THUMBNAILS").expect("THUMBNAILS not defined");
+    let thumbnail_directory: &Path = Path::new(thumbs);
+
+    let t: Vec<_> = thumbnail_directory
+        .read_dir()
+        .expect("Error reading thumbnail directory")
+        .collect();
+    if t.len() > 0 {
+        println!("Skipping thumbs");
+        return;
+    }
+
+    let images = PathBuf::from(dotenv::var("BASE_PATH").unwrap());
+
+    walkdir::WalkDir::new(images)
+        .into_iter()
+        .collect::<Vec<Result<_, _>>>()
+        .into_par_iter()
+        .filter_map(|entry| entry.ok())
+        .filter(|entry| {
+            println!("{:?}", entry.path());
+            match entry.path().extension() {
+                Some(ext) if ext == "jpg" || ext == "jpeg" || ext == "png" => true,
+                _ => false,
+            }
+        })
+        .map(|entry| (image::open(entry.path()), entry.path().to_path_buf()))
+        .filter(|(img, _)| img.is_ok())
+        .map(|(img, path)| (img.unwrap(), path))
+        .map(|(image, path)| (image.thumbnail(200, 200), path))
+        .map(|(image, path)| {
+            let thumb = Path::new(thumbnail_directory).join(path.file_name().unwrap());
+            println!("{:?}", thumb);
+            image.save(thumb).expect("Failure saving thumbnail");
+        })
+        .for_each(drop);
+
+    println!("Finished");
+}
+
 #[actix_rt::main]
 async fn main() -> std::io::Result<()> {
+    create_thumbnails().await;
+
    HttpServer::new(|| {
        App::new()
            .service(register)
            .service(login)
            .service(list_photos)
-            .service(image)
+            .service(get_image)
            .service(generate_video)
            .service(stream_video)
            .service(get_video_part)
--- a/src/video.rs
+++ b/src/video.rs
@@ -1,5 +1,5 @@
-use std::process::Command;
 use std::path::Path;
+use std::process::Command;

 // ffmpeg -i test.mp4 -c:v h264 -flags +cgop -g 30 -hls_time 3 out.m3u8
 // ffmpeg -i "filename.mp4" -preset veryfast -c:v libx264 -f hls -hls_list_size 100 -hls_time 2 -crf 24 -vf scale=1080:-2,setsar=1:1 attempt/vid_out.m3u8
@@ -32,4 +32,3 @@ pub fn create_playlist(video_path: &str, playlist_file: &str) {
    println!("{:?}", result);
    println!("Status: {}", String::from_utf8(result.stdout).unwrap())
 }
-