refactor: modularize server.rs into separate route modules
- Extract scan.rs, files.rs, types.rs, processing.rs, visual_chunk_search.rs - Move AppState and AppConfig to types.rs - Each module exposes pub fn xxx_routes() -> Router<AppState> - server.rs reduced from 5005 to 118 lines (orchestrator only) - All stubs filled with real implementations from git history - Verify: cargo check, clippy, tests all pass
This commit is contained in:
M5Max128
517
src/api/scan.rs
Normal file
517
src/api/scan.rs
Normal file
@@ -0,0 +1,517 @@
|
||||
use axum::{
|
||||
extract::{Path, Query, State},
|
||||
http::StatusCode,
|
||||
response::Json,
|
||||
routing::get,
|
||||
Router,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::types::AppState;
|
||||
use crate::core::db::schema;
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct ScannedFileInfo {
|
||||
file_name: String,
|
||||
relative_path: String,
|
||||
file_path: String,
|
||||
file_size: u64,
|
||||
modified_time: String,
|
||||
is_registered: bool,
|
||||
file_uuid: Option<String>,
|
||||
status: Option<String>,
|
||||
registration_time: Option<String>,
|
||||
job_id: Option<i32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct ScanFilesResponse {
|
||||
files: Vec<ScannedFileInfo>,
|
||||
total: usize,
|
||||
filtered_total: usize,
|
||||
page: usize,
|
||||
page_size: usize,
|
||||
total_pages: usize,
|
||||
registered_count: usize,
|
||||
unregistered_count: usize,
|
||||
total_chunks: i64,
|
||||
searchable_chunks: i64,
|
||||
pending_videos: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ScanFilesQuery {
|
||||
limit: Option<usize>,
|
||||
page: Option<usize>,
|
||||
page_size: Option<usize>,
|
||||
pattern: Option<String>,
|
||||
sort_by: Option<String>,
|
||||
sort_order: Option<String>,
|
||||
}
|
||||
|
||||
fn scan_directory_recursive(
|
||||
dir: &std::path::Path,
|
||||
root: &std::path::Path,
|
||||
allowed_extensions: &[&str],
|
||||
registered_paths: &std::collections::HashMap<
|
||||
String,
|
||||
(String, String, Option<String>, Option<i32>),
|
||||
>,
|
||||
files: &mut Vec<ScannedFileInfo>,
|
||||
) {
|
||||
if let Ok(entries) = std::fs::read_dir(dir) {
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if path.is_dir() {
|
||||
if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
|
||||
if name.starts_with('.') {
|
||||
return;
|
||||
}
|
||||
}
|
||||
scan_directory_recursive(&path, root, allowed_extensions, registered_paths, files);
|
||||
} else if path.is_file() {
|
||||
if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
|
||||
if allowed_extensions.contains(&ext.to_lowercase().as_str()) {
|
||||
if let Ok(meta) = entry.metadata() {
|
||||
let abs_path = path.to_string_lossy().to_string();
|
||||
let rel_path = path
|
||||
.strip_prefix(root)
|
||||
.map(|p| p.to_string_lossy().to_string())
|
||||
.unwrap_or_else(|_| abs_path.clone());
|
||||
|
||||
let file_name = path
|
||||
.file_name()
|
||||
.map(|n| n.to_string_lossy().to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
let modified_time = meta
|
||||
.modified()
|
||||
.ok()
|
||||
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
|
||||
.map(|d| {
|
||||
chrono::DateTime::from_timestamp(d.as_secs() as i64, 0)
|
||||
.map(|dt| dt.to_rfc3339())
|
||||
.unwrap_or_default()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
match registered_paths.get(&abs_path) {
|
||||
Some((uuid, status, reg_time, jid)) if status != "unregistered" => {
|
||||
files.push(ScannedFileInfo {
|
||||
file_name,
|
||||
relative_path: rel_path,
|
||||
file_path: abs_path,
|
||||
file_size: meta.len(),
|
||||
modified_time,
|
||||
is_registered: true,
|
||||
file_uuid: Some(uuid.clone()),
|
||||
status: Some(status.clone()),
|
||||
registration_time: reg_time.clone(),
|
||||
job_id: *jid,
|
||||
});
|
||||
}
|
||||
_ => {
|
||||
files.push(ScannedFileInfo {
|
||||
file_name,
|
||||
relative_path: rel_path,
|
||||
file_path: abs_path,
|
||||
file_size: meta.len(),
|
||||
modified_time,
|
||||
is_registered: false,
|
||||
file_uuid: None,
|
||||
status: Some("unregistered".to_string()),
|
||||
registration_time: None,
|
||||
job_id: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn scan_files(
|
||||
State(state): State<AppState>,
|
||||
Query(params): Query<ScanFilesQuery>,
|
||||
) -> Result<Json<ScanFilesResponse>, StatusCode> {
|
||||
let demo_dir_str = std::env::var("MOMENTRY_SFTP_ROOT")
|
||||
.unwrap_or_else(|_| "/Users/accusys/momentry/var/sftpgo/data/demo".to_string());
|
||||
let demo_dir = std::path::Path::new(&demo_dir_str);
|
||||
|
||||
let allowed_extensions = vec![
|
||||
"mp4", "mov", "mkv", "avi", "webm", "jpg", "jpeg", "png", "gif", "webp",
|
||||
];
|
||||
|
||||
let table = schema::table_name("videos");
|
||||
let mj_table = schema::table_name("monitor_jobs");
|
||||
let registered_db: Vec<(String, String, String, String, Option<String>, Option<i32>)> =
|
||||
sqlx::query_as(&format!(
|
||||
"SELECT v.file_path, v.file_name, v.file_uuid, v.status, v.registration_time::text, \
|
||||
latest_job.id as job_id \
|
||||
FROM {} v \
|
||||
LEFT JOIN LATERAL ( \
|
||||
SELECT id FROM {} WHERE uuid = v.file_uuid ORDER BY id DESC LIMIT 1 \
|
||||
) latest_job ON true \
|
||||
ORDER BY v.id",
|
||||
table, mj_table
|
||||
))
|
||||
.fetch_all(state.db.pool())
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
|
||||
let registered_paths: std::collections::HashMap<
|
||||
String,
|
||||
(String, String, Option<String>, Option<i32>),
|
||||
> = registered_db
|
||||
.into_iter()
|
||||
.map(|(path, _name, uuid, status, reg_time, jid)| (path, (uuid, status, reg_time, jid)))
|
||||
.collect();
|
||||
|
||||
let mut result_files = Vec::new();
|
||||
|
||||
if demo_dir.exists() {
|
||||
scan_directory_recursive(
|
||||
demo_dir,
|
||||
demo_dir,
|
||||
&allowed_extensions,
|
||||
®istered_paths,
|
||||
&mut result_files,
|
||||
);
|
||||
}
|
||||
|
||||
let desc = params.sort_order.as_deref().unwrap_or("asc") == "desc";
|
||||
match params.sort_by.as_deref().unwrap_or("name") {
|
||||
"size" => {
|
||||
if desc {
|
||||
result_files.sort_by(|a, b| b.file_size.cmp(&a.file_size));
|
||||
} else {
|
||||
result_files.sort_by(|a, b| a.file_size.cmp(&b.file_size));
|
||||
}
|
||||
}
|
||||
"modified" | "time" => {
|
||||
if desc {
|
||||
result_files.sort_by(|a, b| b.modified_time.cmp(&a.modified_time));
|
||||
} else {
|
||||
result_files.sort_by(|a, b| a.modified_time.cmp(&b.modified_time));
|
||||
}
|
||||
}
|
||||
"status" => {
|
||||
if desc {
|
||||
result_files
|
||||
.sort_by(|a, b| b.status.cmp(&a.status).then(b.file_name.cmp(&a.file_name)));
|
||||
} else {
|
||||
result_files
|
||||
.sort_by(|a, b| a.status.cmp(&b.status).then(a.file_name.cmp(&b.file_name)));
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
if desc {
|
||||
result_files.sort_by(|a, b| {
|
||||
a.is_registered
|
||||
.cmp(&b.is_registered)
|
||||
.then(b.file_name.cmp(&a.file_name))
|
||||
});
|
||||
} else {
|
||||
result_files.sort_by(|a, b| {
|
||||
b.is_registered
|
||||
.cmp(&a.is_registered)
|
||||
.then(a.file_name.cmp(&b.file_name))
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let total_all = result_files.len();
|
||||
let registered_count = result_files.iter().filter(|f| f.is_registered).count();
|
||||
let unregistered_count = result_files.iter().filter(|f| !f.is_registered).count();
|
||||
|
||||
let filtered: Vec<ScannedFileInfo> = if let Some(ref pat) = params.pattern {
|
||||
let re = match regex::Regex::new(&format!("(?i){}", pat)) {
|
||||
Ok(r) => r,
|
||||
Err(_) => return Err(StatusCode::BAD_REQUEST),
|
||||
};
|
||||
result_files
|
||||
.into_iter()
|
||||
.filter(|f| re.is_match(&f.file_name))
|
||||
.collect()
|
||||
} else {
|
||||
result_files
|
||||
};
|
||||
|
||||
let filtered_total = filtered.len();
|
||||
|
||||
let page = params.page.unwrap_or(1).max(1);
|
||||
let page_size = params
|
||||
.page_size
|
||||
.or(params.limit)
|
||||
.unwrap_or(filtered_total.max(1));
|
||||
let total_pages = if page_size > 0 {
|
||||
(filtered_total + page_size - 1) / page_size
|
||||
} else {
|
||||
1
|
||||
};
|
||||
let start = (page - 1) * page_size;
|
||||
let files: Vec<ScannedFileInfo> = filtered.into_iter().skip(start).take(page_size).collect();
|
||||
|
||||
let table_videos = schema::table_name("videos");
|
||||
let table_chunks = schema::table_name("chunk");
|
||||
let total_chunks: i64 = sqlx::query_scalar(&format!("SELECT COUNT(*) FROM {}", table_chunks))
|
||||
.fetch_one(state.db.pool())
|
||||
.await
|
||||
.unwrap_or(0);
|
||||
let searchable_chunks: i64 = sqlx::query_scalar(&format!(
|
||||
"SELECT COUNT(*) FROM {} WHERE vector_id IS NOT NULL",
|
||||
table_chunks
|
||||
))
|
||||
.fetch_one(state.db.pool())
|
||||
.await
|
||||
.unwrap_or(0);
|
||||
let pending_videos: i64 = sqlx::query_scalar(&format!(
|
||||
"SELECT COUNT(*) FROM {} WHERE status = 'pending'",
|
||||
table_videos
|
||||
))
|
||||
.fetch_one(state.db.pool())
|
||||
.await
|
||||
.unwrap_or(0);
|
||||
|
||||
Ok(Json(ScanFilesResponse {
|
||||
files,
|
||||
total: total_all,
|
||||
filtered_total,
|
||||
page,
|
||||
page_size,
|
||||
total_pages,
|
||||
registered_count,
|
||||
unregistered_count,
|
||||
total_chunks,
|
||||
searchable_chunks,
|
||||
pending_videos,
|
||||
}))
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct SftpgoStatusResponse {
|
||||
username: String,
|
||||
home_dir: String,
|
||||
files_count: i64,
|
||||
registered_videos: Vec<RegisteredVideo>,
|
||||
last_login: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct RegisteredVideo {
|
||||
uuid: String,
|
||||
file_name: String,
|
||||
status: String,
|
||||
}
|
||||
|
||||
async fn get_sftpgo_status(
|
||||
State(state): State<AppState>,
|
||||
) -> Result<Json<SftpgoStatusResponse>, StatusCode> {
|
||||
let demo_dir = "/Users/accusys/momentry/var/sftpgo/data/demo";
|
||||
|
||||
let files_count: i64 = std::fs::read_dir(demo_dir)
|
||||
.map(|entries| entries.count() as i64)
|
||||
.unwrap_or(0);
|
||||
|
||||
let table_videos = schema::table_name("videos");
|
||||
|
||||
let registered_videos: Vec<(String, String, String)> = sqlx::query_as(&format!(
|
||||
"SELECT file_uuid, file_name, status FROM {} WHERE file_path LIKE '%demo%' ORDER BY id",
|
||||
table_videos
|
||||
))
|
||||
.fetch_all(state.db.pool())
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
|
||||
let registered_videos = registered_videos
|
||||
.into_iter()
|
||||
.map(|(uuid, file_name, status)| RegisteredVideo {
|
||||
uuid,
|
||||
file_name,
|
||||
status,
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(Json(SftpgoStatusResponse {
|
||||
username: "demo".to_string(),
|
||||
home_dir: demo_dir.to_string(),
|
||||
files_count,
|
||||
registered_videos,
|
||||
last_login: None,
|
||||
}))
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct IngestionStep {
|
||||
name: String,
|
||||
status: String,
|
||||
detail: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct IdentityRef {
|
||||
uuid: String,
|
||||
name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct IngestionStatusResponse {
|
||||
file_uuid: String,
|
||||
steps: Vec<IngestionStep>,
|
||||
related_identities: Vec<IdentityRef>,
|
||||
strangers: i64,
|
||||
}
|
||||
|
||||
async fn get_ingestion_status(
|
||||
State(state): State<AppState>,
|
||||
Path(file_uuid): Path<String>,
|
||||
) -> Result<Json<IngestionStatusResponse>, StatusCode> {
|
||||
let pool = state.db.pool();
|
||||
let chunk = schema::table_name("chunk");
|
||||
let fd = schema::table_name("face_detections");
|
||||
let identities = schema::table_name("identities");
|
||||
|
||||
let scene_meta_path = format!(
|
||||
"{}/{}.scene_meta.json",
|
||||
crate::core::config::OUTPUT_DIR.as_str(),
|
||||
file_uuid
|
||||
);
|
||||
let scene_meta_ok = std::path::Path::new(&scene_meta_path).exists();
|
||||
|
||||
macro_rules! count_sql {
|
||||
($sql:expr) => {
|
||||
sqlx::query_scalar::<_, i64>($sql)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap_or(0)
|
||||
};
|
||||
}
|
||||
|
||||
let sentence_count = count_sql!(&format!(
|
||||
"SELECT COUNT(*) FROM {chunk} WHERE file_uuid = '{file_uuid}' AND chunk_type = 'sentence'"
|
||||
));
|
||||
let sentence_embedded = count_sql!(&format!("SELECT COUNT(*) FROM {chunk} WHERE file_uuid = '{file_uuid}' AND chunk_type = 'sentence' AND embedding IS NOT NULL"));
|
||||
let scene_count = count_sql!(&format!(
|
||||
"SELECT COUNT(*) FROM {chunk} WHERE file_uuid = '{file_uuid}' AND chunk_type = 'cut'"
|
||||
));
|
||||
let face_total = count_sql!(&format!(
|
||||
"SELECT COUNT(*) FROM {fd} WHERE file_uuid = '{file_uuid}'"
|
||||
));
|
||||
let trace_count = count_sql!(&format!("SELECT COUNT(DISTINCT trace_id) FROM {fd} WHERE file_uuid = '{file_uuid}' AND trace_id IS NOT NULL"));
|
||||
let trace_chunks = count_sql!(&format!(
|
||||
"SELECT COUNT(*) FROM {chunk} WHERE file_uuid = '{file_uuid}' AND chunk_type = 'trace'"
|
||||
));
|
||||
let identity_count = count_sql!(&format!("SELECT COUNT(DISTINCT identity_id) FROM {fd} WHERE file_uuid = '{file_uuid}' AND identity_id IS NOT NULL"));
|
||||
let tkg_nodes = count_sql!(&format!(
|
||||
"SELECT COUNT(*) FROM {} WHERE file_uuid = '{file_uuid}'",
|
||||
schema::table_name("tkg_nodes")
|
||||
));
|
||||
let tkg_edges = count_sql!(&format!(
|
||||
"SELECT COUNT(*) FROM {} WHERE file_uuid = '{file_uuid}'",
|
||||
schema::table_name("tkg_edges")
|
||||
));
|
||||
let scene_5w1h = count_sql!(&format!("SELECT COUNT(*) FROM {chunk} WHERE file_uuid = '{file_uuid}' AND chunk_type = 'cut' AND summary_text IS NOT NULL AND summary_text != ''"));
|
||||
|
||||
let related_identities: Vec<IdentityRef> =
|
||||
match sqlx::query_as::<_, (String, String)>(&format!(
|
||||
"SELECT DISTINCT i.uuid::text, i.name FROM {identities} i \
|
||||
JOIN {fd} fd ON fd.identity_id = i.id \
|
||||
WHERE fd.file_uuid = '{file_uuid}' AND fd.identity_id IS NOT NULL \
|
||||
ORDER BY i.name"
|
||||
))
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
{
|
||||
Ok(rows) => rows
|
||||
.into_iter()
|
||||
.map(|(uuid, name)| IdentityRef {
|
||||
uuid: uuid.replace('-', ""),
|
||||
name,
|
||||
})
|
||||
.collect(),
|
||||
Err(e) => {
|
||||
tracing::error!("related_identities query failed: {}", e);
|
||||
vec![]
|
||||
}
|
||||
};
|
||||
|
||||
let strangers = count_sql!(&format!(
|
||||
"SELECT COUNT(DISTINCT trace_id) FROM {fd} \
|
||||
WHERE file_uuid = '{file_uuid}' AND trace_id IS NOT NULL AND identity_id IS NULL"
|
||||
));
|
||||
|
||||
macro_rules! step {
|
||||
($name:expr, $done:expr, $detail:expr) => {
|
||||
IngestionStep {
|
||||
name: $name.into(),
|
||||
status: if $done { "done" } else { "pending" }.into(),
|
||||
detail: $detail,
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
let steps = vec![
|
||||
step!(
|
||||
"rule1_sentence",
|
||||
sentence_count > 0,
|
||||
Some(format!("{sentence_count} sentence chunks"))
|
||||
),
|
||||
step!(
|
||||
"auto_vectorize",
|
||||
sentence_embedded > 0,
|
||||
Some(format!("{sentence_embedded} embedded"))
|
||||
),
|
||||
step!(
|
||||
"rule3_scene",
|
||||
scene_count > 0,
|
||||
Some(format!("{scene_count} scene chunks"))
|
||||
),
|
||||
step!(
|
||||
"face_trace",
|
||||
trace_count > 0,
|
||||
Some(format!("{trace_count} traces / {face_total} detections"))
|
||||
),
|
||||
step!(
|
||||
"trace_chunks",
|
||||
trace_chunks > 0,
|
||||
Some(format!("{trace_chunks} trace chunks"))
|
||||
),
|
||||
step!(
|
||||
"tkg",
|
||||
tkg_nodes > 0 || tkg_edges > 0,
|
||||
Some(format!("{tkg_nodes} nodes, {tkg_edges} edges"))
|
||||
),
|
||||
step!(
|
||||
"identity_match",
|
||||
identity_count > 0,
|
||||
Some(format!("{identity_count} identities matched"))
|
||||
),
|
||||
step!("scene_metadata", scene_meta_ok, None),
|
||||
step!(
|
||||
"5w1h",
|
||||
scene_5w1h > 0,
|
||||
Some(format!("{scene_5w1h} scenes with 5W1H"))
|
||||
),
|
||||
];
|
||||
|
||||
Ok(Json(IngestionStatusResponse {
|
||||
file_uuid,
|
||||
steps,
|
||||
related_identities,
|
||||
strangers,
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn scan_routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/api/v1/files/scan", get(scan_files))
|
||||
.route("/api/v1/stats/sftpgo", get(get_sftpgo_status))
|
||||
.route(
|
||||
"/api/v1/stats/ingestion-status/:file_uuid",
|
||||
get(get_ingestion_status),
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user