use axum::{ extract::{Path, Query, State}, http::StatusCode, response::Json, routing::get, Router, }; use serde::{Deserialize, Serialize}; use super::types::AppState; use crate::core::db::schema; #[derive(Debug, Serialize, Deserialize)] struct ScannedFileInfo { file_name: String, relative_path: String, file_path: String, file_size: u64, modified_time: String, is_registered: bool, file_uuid: Option, status: Option, registration_time: Option, job_id: Option, } #[derive(Debug, Serialize, Deserialize)] struct ScanFilesResponse { files: Vec, total: usize, filtered_total: usize, page: usize, page_size: usize, total_pages: usize, registered_count: usize, unregistered_count: usize, total_chunks: i64, searchable_chunks: i64, pending_videos: i64, } #[derive(Debug, Deserialize)] struct ScanFilesQuery { limit: Option, page: Option, page_size: Option, pattern: Option, sort_by: Option, sort_order: Option, } fn scan_directory_recursive( dir: &std::path::Path, root: &std::path::Path, allowed_extensions: &[&str], registered_paths: &std::collections::HashMap< String, (String, String, Option, Option), >, files: &mut Vec, ) { if let Ok(entries) = std::fs::read_dir(dir) { for entry in entries.flatten() { let path = entry.path(); if path.is_dir() { if let Some(name) = path.file_name().and_then(|n| n.to_str()) { if name.starts_with('.') { return; } } scan_directory_recursive(&path, root, allowed_extensions, registered_paths, files); } else if path.is_file() { if let Some(ext) = path.extension().and_then(|e| e.to_str()) { if allowed_extensions.contains(&ext.to_lowercase().as_str()) { if let Ok(meta) = entry.metadata() { let abs_path = path.to_string_lossy().to_string(); let rel_path = path .strip_prefix(root) .map(|p| p.to_string_lossy().to_string()) .unwrap_or_else(|_| abs_path.clone()); let file_name = path .file_name() .map(|n| n.to_string_lossy().to_string()) .unwrap_or_default(); let modified_time = meta .modified() .ok() .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) .map(|d| { chrono::DateTime::from_timestamp(d.as_secs() as i64, 0) .map(|dt| dt.to_rfc3339()) .unwrap_or_default() }) .unwrap_or_default(); match registered_paths.get(&abs_path) { Some((uuid, status, reg_time, jid)) if status != "unregistered" => { files.push(ScannedFileInfo { file_name, relative_path: rel_path, file_path: abs_path, file_size: meta.len(), modified_time, is_registered: true, file_uuid: Some(uuid.clone()), status: Some(status.clone()), registration_time: reg_time.clone(), job_id: *jid, }); } _ => { files.push(ScannedFileInfo { file_name, relative_path: rel_path, file_path: abs_path, file_size: meta.len(), modified_time, is_registered: false, file_uuid: None, status: Some("unregistered".to_string()), registration_time: None, job_id: None, }); } } } } } } } } } async fn scan_files( State(state): State, Query(params): Query, ) -> Result, StatusCode> { let demo_dir_str = std::env::var("MOMENTRY_SFTP_ROOT") .unwrap_or_else(|_| "/Users/accusys/momentry/var/sftpgo/data/demo".to_string()); let demo_dir = std::path::Path::new(&demo_dir_str); let allowed_extensions = vec![ "mp4", "mov", "mkv", "avi", "webm", "jpg", "jpeg", "png", "gif", "webp", ]; let table = schema::table_name("videos"); let mj_table = schema::table_name("monitor_jobs"); let registered_db: Vec<(String, String, String, String, Option, Option)> = sqlx::query_as(&format!( "SELECT v.file_path, v.file_name, v.file_uuid, v.status, v.registration_time::text, \ latest_job.id as job_id \ FROM {} v \ LEFT JOIN LATERAL ( \ SELECT id FROM {} WHERE uuid = v.file_uuid ORDER BY id DESC LIMIT 1 \ ) latest_job ON true \ ORDER BY v.id", table, mj_table )) .fetch_all(state.db.pool()) .await .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; let registered_paths: std::collections::HashMap< String, (String, String, Option, Option), > = registered_db .into_iter() .map(|(path, _name, uuid, status, reg_time, jid)| (path, (uuid, status, reg_time, jid))) .collect(); let mut result_files = Vec::new(); if demo_dir.exists() { scan_directory_recursive( demo_dir, demo_dir, &allowed_extensions, ®istered_paths, &mut result_files, ); } let desc = params.sort_order.as_deref().unwrap_or("asc") == "desc"; match params.sort_by.as_deref().unwrap_or("name") { "size" => { if desc { result_files.sort_by(|a, b| b.file_size.cmp(&a.file_size)); } else { result_files.sort_by(|a, b| a.file_size.cmp(&b.file_size)); } } "modified" | "time" => { if desc { result_files.sort_by(|a, b| b.modified_time.cmp(&a.modified_time)); } else { result_files.sort_by(|a, b| a.modified_time.cmp(&b.modified_time)); } } "status" => { if desc { result_files .sort_by(|a, b| b.status.cmp(&a.status).then(b.file_name.cmp(&a.file_name))); } else { result_files .sort_by(|a, b| a.status.cmp(&b.status).then(a.file_name.cmp(&b.file_name))); } } _ => { if desc { result_files.sort_by(|a, b| { a.is_registered .cmp(&b.is_registered) .then(b.file_name.cmp(&a.file_name)) }); } else { result_files.sort_by(|a, b| { b.is_registered .cmp(&a.is_registered) .then(a.file_name.cmp(&b.file_name)) }); } } } let total_all = result_files.len(); let registered_count = result_files.iter().filter(|f| f.is_registered).count(); let unregistered_count = result_files.iter().filter(|f| !f.is_registered).count(); let filtered: Vec = if let Some(ref pat) = params.pattern { let re = match regex::Regex::new(&format!("(?i){}", pat)) { Ok(r) => r, Err(_) => return Err(StatusCode::BAD_REQUEST), }; result_files .into_iter() .filter(|f| re.is_match(&f.file_name)) .collect() } else { result_files }; let filtered_total = filtered.len(); let page = params.page.unwrap_or(1).max(1); let page_size = params .page_size .or(params.limit) .unwrap_or(filtered_total.max(1)); let total_pages = if page_size > 0 { (filtered_total + page_size - 1) / page_size } else { 1 }; let start = (page - 1) * page_size; let files: Vec = filtered.into_iter().skip(start).take(page_size).collect(); let table_videos = schema::table_name("videos"); let table_chunks = schema::table_name("chunk"); let total_chunks: i64 = sqlx::query_scalar(&format!("SELECT COUNT(*) FROM {}", table_chunks)) .fetch_one(state.db.pool()) .await .unwrap_or(0); let searchable_chunks: i64 = sqlx::query_scalar(&format!( "SELECT COUNT(*) FROM {} WHERE vector_id IS NOT NULL", table_chunks )) .fetch_one(state.db.pool()) .await .unwrap_or(0); let pending_videos: i64 = sqlx::query_scalar(&format!( "SELECT COUNT(*) FROM {} WHERE status = 'pending'", table_videos )) .fetch_one(state.db.pool()) .await .unwrap_or(0); Ok(Json(ScanFilesResponse { files, total: total_all, filtered_total, page, page_size, total_pages, registered_count, unregistered_count, total_chunks, searchable_chunks, pending_videos, })) } #[derive(Debug, Serialize)] struct SftpgoStatusResponse { username: String, home_dir: String, files_count: i64, registered_videos: Vec, last_login: Option, } #[derive(Debug, Serialize)] struct RegisteredVideo { uuid: String, file_name: String, status: String, } async fn get_sftpgo_status( State(state): State, ) -> Result, StatusCode> { let demo_dir = "/Users/accusys/momentry/var/sftpgo/data/demo"; let files_count: i64 = std::fs::read_dir(demo_dir) .map(|entries| entries.count() as i64) .unwrap_or(0); let table_videos = schema::table_name("videos"); let registered_videos: Vec<(String, String, String)> = sqlx::query_as(&format!( "SELECT file_uuid, file_name, status FROM {} WHERE file_path LIKE '%demo%' ORDER BY id", table_videos )) .fetch_all(state.db.pool()) .await .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; let registered_videos = registered_videos .into_iter() .map(|(uuid, file_name, status)| RegisteredVideo { uuid, file_name, status, }) .collect(); Ok(Json(SftpgoStatusResponse { username: "demo".to_string(), home_dir: demo_dir.to_string(), files_count, registered_videos, last_login: None, })) } #[derive(Debug, Serialize)] struct IngestionStep { name: String, status: String, detail: Option, } #[derive(Debug, Serialize)] struct IdentityRef { uuid: String, name: String, } #[derive(Debug, Serialize)] struct IngestionStatusResponse { file_uuid: String, steps: Vec, related_identities: Vec, strangers: i64, } async fn get_ingestion_status( State(state): State, Path(file_uuid): Path, ) -> Result, StatusCode> { let pool = state.db.pool(); let chunk = schema::table_name("chunk"); let fd = schema::table_name("face_detections"); let identities = schema::table_name("identities"); let scene_meta_path = format!( "{}/{}.scene_meta.json", crate::core::config::OUTPUT_DIR.as_str(), file_uuid ); let scene_meta_ok = std::path::Path::new(&scene_meta_path).exists(); macro_rules! count_sql { ($sql:expr) => { sqlx::query_scalar::<_, i64>($sql) .fetch_one(pool) .await .unwrap_or(0) }; } let sentence_count = count_sql!(&format!( "SELECT COUNT(*) FROM {chunk} WHERE file_uuid = '{file_uuid}' AND chunk_type = 'sentence'" )); let sentence_embedded = count_sql!(&format!("SELECT COUNT(*) FROM {chunk} WHERE file_uuid = '{file_uuid}' AND chunk_type = 'sentence' AND embedding IS NOT NULL")); let scene_count = count_sql!(&format!( "SELECT COUNT(*) FROM {chunk} WHERE file_uuid = '{file_uuid}' AND chunk_type = 'cut'" )); let face_total = count_sql!(&format!( "SELECT COUNT(*) FROM {fd} WHERE file_uuid = '{file_uuid}'" )); let trace_count = count_sql!(&format!("SELECT COUNT(DISTINCT trace_id) FROM {fd} WHERE file_uuid = '{file_uuid}' AND trace_id IS NOT NULL")); let trace_chunks = count_sql!(&format!( "SELECT COUNT(*) FROM {chunk} WHERE file_uuid = '{file_uuid}' AND chunk_type = 'trace'" )); let identity_count = count_sql!(&format!("SELECT COUNT(DISTINCT identity_id) FROM {fd} WHERE file_uuid = '{file_uuid}' AND identity_id IS NOT NULL")); let tkg_nodes = count_sql!(&format!( "SELECT COUNT(*) FROM {} WHERE file_uuid = '{file_uuid}'", schema::table_name("tkg_nodes") )); let tkg_edges = count_sql!(&format!( "SELECT COUNT(*) FROM {} WHERE file_uuid = '{file_uuid}'", schema::table_name("tkg_edges") )); let scene_5w1h = count_sql!(&format!("SELECT COUNT(*) FROM {chunk} WHERE file_uuid = '{file_uuid}' AND chunk_type = 'cut' AND summary_text IS NOT NULL AND summary_text != ''")); let related_identities: Vec = match sqlx::query_as::<_, (String, String)>(&format!( "SELECT DISTINCT i.uuid::text, i.name FROM {identities} i \ JOIN {fd} fd ON fd.identity_id = i.id \ WHERE fd.file_uuid = '{file_uuid}' AND fd.identity_id IS NOT NULL \ ORDER BY i.name" )) .fetch_all(pool) .await { Ok(rows) => rows .into_iter() .map(|(uuid, name)| IdentityRef { uuid: uuid.replace('-', ""), name, }) .collect(), Err(e) => { tracing::error!("related_identities query failed: {}", e); vec![] } }; let strangers = count_sql!(&format!( "SELECT COUNT(DISTINCT trace_id) FROM {fd} \ WHERE file_uuid = '{file_uuid}' AND trace_id IS NOT NULL AND identity_id IS NULL" )); macro_rules! step { ($name:expr, $done:expr, $detail:expr) => { IngestionStep { name: $name.into(), status: if $done { "done" } else { "pending" }.into(), detail: $detail, } }; } let steps = vec![ step!( "rule1_sentence", sentence_count > 0, Some(format!("{sentence_count} sentence chunks")) ), step!( "auto_vectorize", sentence_embedded > 0, Some(format!("{sentence_embedded} embedded")) ), step!( "rule3_scene", scene_count > 0, Some(format!("{scene_count} scene chunks")) ), step!( "face_trace", trace_count > 0, Some(format!("{trace_count} traces / {face_total} detections")) ), step!( "trace_chunks", trace_chunks > 0, Some(format!("{trace_chunks} trace chunks")) ), step!( "tkg", tkg_nodes > 0 || tkg_edges > 0, Some(format!("{tkg_nodes} nodes, {tkg_edges} edges")) ), step!( "identity_match", identity_count > 0, Some(format!("{identity_count} identities matched")) ), step!("scene_metadata", scene_meta_ok, None), step!( "5w1h", scene_5w1h > 0, Some(format!("{scene_5w1h} scenes with 5W1H")) ), ]; Ok(Json(IngestionStatusResponse { file_uuid, steps, related_identities, strangers, })) } pub fn scan_routes() -> Router { Router::new() .route("/api/v1/files/scan", get(scan_files)) .route("/api/v1/stats/sftpgo", get(get_sftpgo_status)) .route( "/api/v1/stats/ingestion-status/:file_uuid", get(get_ingestion_status), ) }