feat(bins): multi-library populate_knowledge + progress UX
populate_knowledge now loads real libraries from the DB instead of fabricating a single library_id=1 row from BASE_PATH. Adds --library <id|name> to restrict the walk and validates --path against the selected library roots. The full library set is still passed to InsightGenerator so resolve_full_path can probe every root when an insight resolves to a different library than the one being walked. Adds indicatif progress bars across the long-running utility binaries via a shared src/bin_progress.rs helper (determinate bar + open-ended spinner with consistent styling). Per-batch info! noise is replaced by the bar's throughput/ETA; warnings and errors route through pb.println so they scroll above the bar instead of fighting with it. populate_knowledge spinner during scan, determinate bar over all libs backfill_hashes spinner with running hashed/missing/errors counts import_calendar determinate bar; embedding/store failures inline import_location_* determinate bar advancing by chunk size import_search_* determinate bar; pb cloned into the spawn task cleanup_files P1 determinate bar over DB paths cleanup_files P2 determinate bar; pb.suspend() around y/n/a/s prompt Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -2,10 +2,10 @@ use anyhow::{Context, Result};
|
||||
use chrono::Utc;
|
||||
use clap::Parser;
|
||||
use image_api::ai::ollama::OllamaClient;
|
||||
use image_api::bin_progress;
|
||||
use image_api::database::calendar_dao::{InsertCalendarEvent, SqliteCalendarEventDao};
|
||||
use image_api::parsers::ical_parser::parse_ics_file;
|
||||
use log::{error, info};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
// Import the trait to use its methods
|
||||
use image_api::database::CalendarEventDao;
|
||||
@@ -64,9 +64,11 @@ async fn main() -> Result<()> {
|
||||
None
|
||||
};
|
||||
|
||||
let inserted_count = Arc::new(Mutex::new(0));
|
||||
let skipped_count = Arc::new(Mutex::new(0));
|
||||
let error_count = Arc::new(Mutex::new(0));
|
||||
let mut inserted_count = 0usize;
|
||||
let mut skipped_count = 0usize;
|
||||
let mut error_count = 0usize;
|
||||
|
||||
let pb = bin_progress::determinate(events.len() as u64, "importing");
|
||||
|
||||
// Process events in batches
|
||||
// Can't use rayon with async, so process sequentially
|
||||
@@ -82,7 +84,8 @@ async fn main() -> Result<()> {
|
||||
)
|
||||
&& exists
|
||||
{
|
||||
*skipped_count.lock().unwrap() += 1;
|
||||
skipped_count += 1;
|
||||
pb.inc(1);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -101,10 +104,7 @@ async fn main() -> Result<()> {
|
||||
}) {
|
||||
Ok(emb) => Some(emb),
|
||||
Err(e) => {
|
||||
error!(
|
||||
"Failed to generate embedding for event '{}': {}",
|
||||
event.summary, e
|
||||
);
|
||||
pb.println(format!("embedding failed for '{}': {}", event.summary, e));
|
||||
None
|
||||
}
|
||||
}
|
||||
@@ -133,28 +133,26 @@ async fn main() -> Result<()> {
|
||||
};
|
||||
|
||||
match dao_instance.store_event(&context, insert_event) {
|
||||
Ok(_) => {
|
||||
*inserted_count.lock().unwrap() += 1;
|
||||
if *inserted_count.lock().unwrap() % 100 == 0 {
|
||||
info!("Imported {} events...", *inserted_count.lock().unwrap());
|
||||
}
|
||||
}
|
||||
Ok(_) => inserted_count += 1,
|
||||
Err(e) => {
|
||||
error!("Failed to store event '{}': {:?}", event.summary, e);
|
||||
*error_count.lock().unwrap() += 1;
|
||||
pb.println(format!("store failed for '{}': {:?}", event.summary, e));
|
||||
error_count += 1;
|
||||
}
|
||||
}
|
||||
pb.set_message(format!(
|
||||
"inserted={} skipped={} errors={}",
|
||||
inserted_count, skipped_count, error_count
|
||||
));
|
||||
pb.inc(1);
|
||||
}
|
||||
|
||||
let final_inserted = *inserted_count.lock().unwrap();
|
||||
let final_skipped = *skipped_count.lock().unwrap();
|
||||
let final_errors = *error_count.lock().unwrap();
|
||||
pb.finish_and_clear();
|
||||
|
||||
info!("\n=== Import Summary ===");
|
||||
info!("=== Import Summary ===");
|
||||
info!("Total events found: {}", events.len());
|
||||
info!("Successfully inserted: {}", final_inserted);
|
||||
info!("Skipped (already exist): {}", final_skipped);
|
||||
info!("Errors: {}", final_errors);
|
||||
info!("Successfully inserted: {}", inserted_count);
|
||||
info!("Skipped (already exist): {}", skipped_count);
|
||||
info!("Errors: {}", error_count);
|
||||
|
||||
if args.generate_embeddings {
|
||||
info!("Embeddings were generated for semantic search");
|
||||
@@ -162,5 +160,12 @@ async fn main() -> Result<()> {
|
||||
info!("No embeddings generated (use --generate-embeddings to enable semantic search)");
|
||||
}
|
||||
|
||||
if error_count > 0 {
|
||||
error!(
|
||||
"Completed with {} errors — review log output above",
|
||||
error_count
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user