From 6abc99d9b671eecc7dafb42e099a4bb11dd361da Mon Sep 17 00:00:00 2001 From: Cameron Cordes Date: Mon, 5 Apr 2021 20:14:34 -0400 Subject: [PATCH 1/3] Add PrometheusMetrics --- Cargo.lock | 29 +++++++++++++++++++++++++++++ Cargo.toml | 1 + src/main.rs | 7 ++++++- 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 3c1e301..ded651d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -326,6 +326,20 @@ dependencies = [ "syn", ] +[[package]] +name = "actix-web-prom" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec994b7dd52e686b9ae8f63bc485e6027fb44b936ed09bdf648495b9966c36ce" +dependencies = [ + "actix-http", + "actix-service", + "actix-web", + "futures", + "pin-project 1.0.6", + "prometheus", +] + [[package]] name = "actix_derive" version = "0.5.0" @@ -1136,6 +1150,7 @@ dependencies = [ "actix-multipart", "actix-rt", "actix-web", + "actix-web-prom", "bcrypt", "chrono", "diesel", @@ -1728,6 +1743,20 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "prometheus" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8425533e7122f0c3cc7a37e6244b16ad3a2cc32ae7ac6276e2a75da0d9c200d" +dependencies = [ + "cfg-if 1.0.0", + "fnv", + "lazy_static", + "parking_lot", + "regex", + "thiserror", +] + [[package]] name = "quick-error" version = "1.2.3" diff --git a/Cargo.toml b/Cargo.toml index a8f3ed8..f1f92ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,3 +32,4 @@ notify = "4.0" path-absolutize = "3.0.6" log="0.4" env_logger="0.8" +actix-web-prom = "0.5.1" diff --git a/src/main.rs b/src/main.rs index 7718412..b549c60 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,12 +3,13 @@ extern crate diesel; extern crate rayon; use crate::auth::login; +use actix_web_prom::PrometheusMetrics; use database::{DbError, DbErrorKind, FavoriteDao, SqliteFavoriteDao, SqliteUserDao, UserDao}; use futures::stream::StreamExt; -use std::io::prelude::*; use std::path::{Path, PathBuf}; use std::sync::mpsc::channel; use std::sync::Arc; +use std::{collections::HashMap, io::prelude::*}; use std::{env, fs::File}; use actix::prelude::*; @@ -404,6 +405,9 @@ fn main() -> std::io::Result<()> { stream_manager: Arc::new(act), }); + let labels = HashMap::new(); + let prometheus = PrometheusMetrics::new("", Some("/metrics"), Some(labels)); + HttpServer::new(move || { let user_dao = SqliteUserDao::new(); let favorites_dao = SqliteFavoriteDao::new(); @@ -422,6 +426,7 @@ fn main() -> std::io::Result<()> { .app_data(app_data.clone()) .data::>(Box::new(user_dao)) .data::>(Box::new(favorites_dao)) + .wrap(prometheus.clone()) }) .bind(dotenv::var("BIND_URL").unwrap())? .bind("localhost:8088")? From a79179c5c308ea3468d95df5cc4b286c174f29bb Mon Sep 17 00:00:00 2001 From: Cameron Cordes Date: Fri, 30 Apr 2021 23:53:10 -0400 Subject: [PATCH 2/3] Add Image and Video total gauges --- Cargo.lock | 9 ++++++++ Cargo.toml | 2 ++ src/main.rs | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 75 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ded651d..0de7d06 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1160,9 +1160,11 @@ dependencies = [ "hmac", "image", "jsonwebtoken", + "lazy_static", "log", "notify", "path-absolutize", + "prometheus", "rayon", "serde", "serde_json", @@ -1753,10 +1755,17 @@ dependencies = [ "fnv", "lazy_static", "parking_lot", + "protobuf", "regex", "thiserror", ] +[[package]] +name = "protobuf" +version = "2.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45604fc7a88158e7d514d8e22e14ac746081e7a70d7690074dd0029ee37458d6" + [[package]] name = "quick-error" version = "1.2.3" diff --git a/Cargo.toml b/Cargo.toml index f1f92ec..df48e1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,3 +33,5 @@ path-absolutize = "3.0.6" log="0.4" env_logger="0.8" actix-web-prom = "0.5.1" +prometheus = "0.11" +lazy_static = "1.1" diff --git a/src/main.rs b/src/main.rs index b549c60..d52700c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,11 +6,14 @@ use crate::auth::login; use actix_web_prom::PrometheusMetrics; use database::{DbError, DbErrorKind, FavoriteDao, SqliteFavoriteDao, SqliteUserDao, UserDao}; use futures::stream::StreamExt; +use lazy_static::lazy_static; +use prometheus::{self, Gauge}; use std::path::{Path, PathBuf}; use std::sync::mpsc::channel; use std::sync::Arc; use std::{collections::HashMap, io::prelude::*}; use std::{env, fs::File}; +use walkdir::{DirEntry, WalkDir}; use actix::prelude::*; use actix_files::NamedFile; @@ -39,6 +42,19 @@ mod database; mod files; mod video; +lazy_static! { + static ref IMAGE_GAUGE: Gauge = Gauge::new( + "imageserver_image_total", + "Count of the images on the server" + ) + .unwrap(); + static ref VIDEO_GAUGE: Gauge = Gauge::new( + "imageserver_video_total", + "Count of the videos on the server" + ) + .unwrap(); +} + #[get("/photos")] async fn list_photos(_claims: Claims, req: Query) -> impl Responder { info!("{}", req.path); @@ -317,7 +333,7 @@ fn create_thumbnails() { generate_video_thumbnail(entry.path(), &thumb_path); false } else { - ext == "jpg" || ext == "jpeg" || ext == "png" || ext == "nef" + is_image(entry) } } else { error!("Unable to get extension for file: {:?}", entry.path()); @@ -349,7 +365,42 @@ fn create_thumbnails() { }) .for_each(drop); - debug!("Finished"); + debug!("Finished making thumbnails"); + + update_media_counts(&images); +} + +fn update_media_counts(media_dir: &PathBuf) { + let mut image_count = 0; + let mut video_count = 0; + for ref entry in WalkDir::new(media_dir).into_iter().filter_map(|e| e.ok()) { + if is_image(entry) { + image_count += 1; + } else if is_video(entry) { + video_count += 1; + } + } + + IMAGE_GAUGE.set(image_count as f64); + VIDEO_GAUGE.set(video_count as f64); +} + +fn is_image(entry: &DirEntry) -> bool { + entry + .path() + .extension() + .and_then(|ext| ext.to_str()) + .map(|ext| ext == "jpg" || ext == "jpeg" || ext == "png" || ext == "nef") + .unwrap_or(false) +} + +fn is_video(entry: &DirEntry) -> bool { + entry + .path() + .extension() + .and_then(|ext| ext.to_str()) + .map(|ext| ext == "mp4" || ext == "mov") + .unwrap_or(false) } fn main() -> std::io::Result<()> { @@ -392,6 +443,9 @@ fn main() -> std::io::Result<()> { } } } + DebouncedEvent::Remove(_) => { + update_media_counts(&PathBuf::from(env::var("BASE_PATH").unwrap())) + } _ => continue, }; } @@ -407,6 +461,14 @@ fn main() -> std::io::Result<()> { let labels = HashMap::new(); let prometheus = PrometheusMetrics::new("", Some("/metrics"), Some(labels)); + prometheus + .registry + .register(Box::new(IMAGE_GAUGE.clone())) + .unwrap(); + prometheus + .registry + .register(Box::new(VIDEO_GAUGE.clone())) + .unwrap(); HttpServer::new(move || { let user_dao = SqliteUserDao::new(); From c5a76759865e4535d9b9a9da323daf140977686b Mon Sep 17 00:00:00 2001 From: Cameron Cordes Date: Sat, 1 May 2021 00:01:17 -0400 Subject: [PATCH 3/3] Use IntGauge for media counts --- src/main.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main.rs b/src/main.rs index d52700c..16ee93b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,7 +7,7 @@ use actix_web_prom::PrometheusMetrics; use database::{DbError, DbErrorKind, FavoriteDao, SqliteFavoriteDao, SqliteUserDao, UserDao}; use futures::stream::StreamExt; use lazy_static::lazy_static; -use prometheus::{self, Gauge}; +use prometheus::{self, IntGauge}; use std::path::{Path, PathBuf}; use std::sync::mpsc::channel; use std::sync::Arc; @@ -43,12 +43,12 @@ mod files; mod video; lazy_static! { - static ref IMAGE_GAUGE: Gauge = Gauge::new( + static ref IMAGE_GAUGE: IntGauge = IntGauge::new( "imageserver_image_total", "Count of the images on the server" ) .unwrap(); - static ref VIDEO_GAUGE: Gauge = Gauge::new( + static ref VIDEO_GAUGE: IntGauge = IntGauge::new( "imageserver_video_total", "Count of the videos on the server" ) @@ -381,8 +381,8 @@ fn update_media_counts(media_dir: &PathBuf) { } } - IMAGE_GAUGE.set(image_count as f64); - VIDEO_GAUGE.set(video_count as f64); + IMAGE_GAUGE.set(image_count); + VIDEO_GAUGE.set(video_count); } fn is_image(entry: &DirEntry) -> bool {