- job_worker.rs: add upsert_processor_result when output file exists - job_worker.rs: add load JSON and store to pre_chunks when output exists - rule3_ingest.rs: fix SQL bind order (scene_number was occupying chunk_type slot) - files.rs: fix unregister WHERE clause (uuid -> file_uuid) + add pre_chunks delete - asrx_self/main_fixed.py: fix KeyError (s['start'] -> s['start_time']) - wrapper_worker_playground.sh: add Worker launchd script - com.momentry.playground.plist: add Playground launchd config
1121 lines
38 KiB
Rust
1121 lines
38 KiB
Rust
use axum::{
|
|
extract::{Path, Query, State},
|
|
http::StatusCode,
|
|
response::Json,
|
|
routing::{get, post},
|
|
Router,
|
|
};
|
|
use serde::{Deserialize, Serialize};
|
|
use sqlx::Row;
|
|
use std::collections::HashMap;
|
|
|
|
use super::types::AppState;
|
|
use crate::core::config;
|
|
use crate::core::db::schema;
|
|
use crate::core::db::{Database, PostgresDb};
|
|
use crate::core::storage::content_hash;
|
|
use crate::FileManager;
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
struct RegisterFileRequest {
|
|
file_path: String,
|
|
user_id: Option<i64>,
|
|
content_hash: Option<String>,
|
|
pattern: Option<String>,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize, Serialize)]
|
|
struct FileLookupMatch {
|
|
file_uuid: String,
|
|
file_name: String,
|
|
file_type: Option<String>,
|
|
status: String,
|
|
content_hash: Option<String>,
|
|
file_size: Option<i64>,
|
|
duration: Option<f64>,
|
|
width: Option<i32>,
|
|
height: Option<i32>,
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
struct FileLookupResponse {
|
|
file_name: String,
|
|
exists: bool,
|
|
matches: Vec<FileLookupMatch>,
|
|
next_name: String,
|
|
}
|
|
|
|
async fn lookup_file_by_name(
|
|
State(state): State<AppState>,
|
|
Query(params): Query<HashMap<String, String>>,
|
|
) -> Result<Json<FileLookupResponse>, StatusCode> {
|
|
let file_name = params.get("file_name").ok_or(StatusCode::BAD_REQUEST)?;
|
|
let base = file_name.to_string();
|
|
let dot_pos = base.rfind('.');
|
|
let (stem, _ext) = match dot_pos {
|
|
Some(p) => (base[..p].to_string(), base[p..].to_string()),
|
|
None => (base.clone(), String::new()),
|
|
};
|
|
let pattern = format!("{}%%", &stem);
|
|
|
|
let table = schema::table_name("videos");
|
|
let query_sql = format!("SELECT file_uuid, file_name, file_type, status, content_hash, duration, width, height FROM {} WHERE file_name = $1 OR file_name LIKE $2 ORDER BY file_name", table);
|
|
let rows = sqlx::query(&query_sql)
|
|
.bind(&base)
|
|
.bind(&pattern)
|
|
.fetch_all(state.db.pool())
|
|
.await
|
|
.map_err(|e| {
|
|
tracing::error!("lookup query error: {}", e);
|
|
StatusCode::INTERNAL_SERVER_ERROR
|
|
})?;
|
|
|
|
let exists = rows.iter().any(|r| r.get::<String, _>("file_name") == base);
|
|
let matches: Vec<FileLookupMatch> = rows
|
|
.iter()
|
|
.map(|r| FileLookupMatch {
|
|
file_uuid: r.get("file_uuid"),
|
|
file_name: r.get("file_name"),
|
|
file_type: r.get("file_type"),
|
|
status: r.get("status"),
|
|
content_hash: r.get("content_hash"),
|
|
file_size: None,
|
|
duration: r.get("duration"),
|
|
width: r.get("width"),
|
|
height: r.get("height"),
|
|
})
|
|
.collect();
|
|
|
|
let next_name = if exists {
|
|
let mut attempt = 1usize;
|
|
loop {
|
|
let candidate = if let Some(p) = dot_pos {
|
|
format!("{} ({}){}", &base[..p], attempt, &base[p..])
|
|
} else {
|
|
format!("{} ({})", base, attempt)
|
|
};
|
|
let conflict: Option<String> = sqlx::query_scalar(&format!(
|
|
"SELECT file_uuid FROM {} WHERE file_name = $1",
|
|
table
|
|
))
|
|
.bind(&candidate)
|
|
.fetch_optional(state.db.pool())
|
|
.await
|
|
.unwrap_or(None);
|
|
if conflict.is_none() {
|
|
break candidate;
|
|
}
|
|
attempt += 1;
|
|
}
|
|
} else {
|
|
base.clone()
|
|
};
|
|
|
|
Ok(Json(FileLookupResponse {
|
|
file_name: base,
|
|
exists,
|
|
matches,
|
|
next_name,
|
|
}))
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
struct RegisterFileResponse {
|
|
success: bool,
|
|
file_uuid: String,
|
|
file_name: String,
|
|
file_path: String,
|
|
file_type: Option<String>,
|
|
duration: f64,
|
|
width: u32,
|
|
height: u32,
|
|
fps: f64,
|
|
total_frames: u64,
|
|
registration_time: Option<String>,
|
|
already_exists: bool,
|
|
message: String,
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
struct ProbeResponse {
|
|
file_uuid: String,
|
|
file_name: String,
|
|
file_size: Option<i64>,
|
|
duration: f64,
|
|
width: u32,
|
|
height: u32,
|
|
fps: f64,
|
|
total_frames: u64,
|
|
file_type: Option<String>,
|
|
probe_result: Option<serde_json::Value>,
|
|
}
|
|
|
|
// --- Fields-only structs for response ---
|
|
|
|
fn sha256_file(path: &std::path::Path) -> Option<String> {
|
|
content_hash::compute_sha256(path).ok()
|
|
}
|
|
|
|
async fn resolve_filename(db: &PostgresDb, file_name: &str, content_hash: &str) -> String {
|
|
let table = schema::table_name("videos");
|
|
let base = file_name.to_string();
|
|
let dot_pos = base.rfind('.');
|
|
let (stem, ext) = match dot_pos {
|
|
Some(p) => (base[..p].to_string(), base[p..].to_string()),
|
|
None => (base.clone(), String::new()),
|
|
};
|
|
let mut candidate = base.clone();
|
|
let mut attempt = 0usize;
|
|
loop {
|
|
let conflict: Option<String> = sqlx::query_scalar(
|
|
&format!("SELECT file_uuid FROM {} WHERE file_name = $1 AND (content_hash IS DISTINCT FROM $2 OR content_hash IS NULL)", table)
|
|
)
|
|
.bind(&candidate)
|
|
.bind(content_hash)
|
|
.fetch_optional(db.pool())
|
|
.await
|
|
.unwrap_or(None);
|
|
if conflict.is_none() {
|
|
return candidate;
|
|
}
|
|
attempt += 1;
|
|
candidate = format!("{} ({}){}", stem, attempt, ext);
|
|
}
|
|
}
|
|
|
|
async fn register_single_file(
|
|
state: &AppState,
|
|
file_path: &str,
|
|
_user_id: Option<i64>,
|
|
provided_hash: Option<String>,
|
|
) -> RegisterFileResponse {
|
|
tracing::info!("[REGISTER] Starting registration for: {}", file_path);
|
|
|
|
let path = std::path::Path::new(file_path);
|
|
if !path.exists() {
|
|
return RegisterFileResponse {
|
|
success: false,
|
|
file_uuid: String::new(),
|
|
file_name: String::new(),
|
|
file_path: file_path.to_string(),
|
|
file_type: None,
|
|
duration: 0.0,
|
|
width: 0,
|
|
height: 0,
|
|
fps: 0.0,
|
|
total_frames: 0,
|
|
registration_time: None,
|
|
already_exists: false,
|
|
message: format!("File not found: {}", file_path),
|
|
};
|
|
}
|
|
|
|
let canonical_path = path
|
|
.canonicalize()
|
|
.map(|p| p.to_string_lossy().to_string())
|
|
.unwrap_or_else(|_| file_path.to_string());
|
|
|
|
let file_name = path
|
|
.file_name()
|
|
.map(|n| n.to_string_lossy().to_string())
|
|
.unwrap_or_default();
|
|
|
|
let db = match PostgresDb::init().await {
|
|
Ok(db) => db,
|
|
Err(e) => {
|
|
tracing::error!("[REGISTER] DB init failed: {}", e);
|
|
return RegisterFileResponse {
|
|
success: false,
|
|
file_uuid: String::new(),
|
|
file_name,
|
|
file_path: canonical_path,
|
|
file_type: None,
|
|
duration: 0.0,
|
|
width: 0,
|
|
height: 0,
|
|
fps: 0.0,
|
|
total_frames: 0,
|
|
registration_time: None,
|
|
already_exists: false,
|
|
message: format!("DB init failed: {}", e),
|
|
};
|
|
}
|
|
};
|
|
|
|
let output_dir = std::env::var("MOMENTRY_OUTPUT_DIR")
|
|
.unwrap_or_else(|_| "/Users/accusys/momentry/output_dev".to_string());
|
|
|
|
let birthday = std::fs::metadata(&path)
|
|
.ok()
|
|
.and_then(|m| m.modified().ok())
|
|
.map(|t| {
|
|
let secs = t
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.unwrap_or_default()
|
|
.as_secs();
|
|
chrono::DateTime::from_timestamp(secs as i64, 0)
|
|
.map(|dt| dt.to_rfc3339())
|
|
.unwrap_or_else(|| chrono::Utc::now().to_rfc3339())
|
|
})
|
|
.unwrap_or_else(|| chrono::Utc::now().to_rfc3339());
|
|
|
|
let mac_address = crate::core::storage::uuid::get_mac_address();
|
|
let pre_file_uuid = crate::core::storage::uuid::compute_birth_uuid(
|
|
&mac_address,
|
|
&birthday,
|
|
&canonical_path,
|
|
&file_name,
|
|
);
|
|
let pre_path = std::path::Path::new(&output_dir).join(format!("{}.pre.json", pre_file_uuid));
|
|
let pre_data: Option<serde_json::Value> = std::fs::read_to_string(&pre_path)
|
|
.ok()
|
|
.and_then(|s| serde_json::from_str(&s).ok());
|
|
|
|
let (content_hash_val, birthday, _pre_file_uuid) = if let Some(ref pre) = pre_data {
|
|
let h = pre
|
|
.get("content_hash")
|
|
.and_then(|v| v.as_str())
|
|
.unwrap_or("")
|
|
.to_string();
|
|
let b = pre
|
|
.get("birthday")
|
|
.and_then(|v| v.as_str())
|
|
.unwrap_or(&birthday)
|
|
.to_string();
|
|
let u = pre
|
|
.get("file_uuid")
|
|
.and_then(|v| v.as_str())
|
|
.unwrap_or(&pre_file_uuid)
|
|
.to_string();
|
|
(h, b, u)
|
|
} else {
|
|
let h = provided_hash
|
|
.filter(|h| !h.is_empty())
|
|
.unwrap_or_else(|| sha256_file(&path).unwrap_or_default());
|
|
(h, birthday, pre_file_uuid)
|
|
};
|
|
|
|
let file_uuid = crate::core::storage::uuid::compute_birth_uuid(
|
|
&mac_address,
|
|
&birthday,
|
|
&canonical_path,
|
|
&file_name,
|
|
);
|
|
tracing::info!(
|
|
"[REGISTER] UUID inputs: mac={} birthday={} path={} name={} pre_found={} → {}",
|
|
mac_address,
|
|
birthday,
|
|
canonical_path,
|
|
file_name,
|
|
pre_data.is_some(),
|
|
file_uuid
|
|
);
|
|
|
|
let videos_table = schema::table_name("videos");
|
|
if !content_hash_val.is_empty() {
|
|
if let Ok(Some(existing_uuid)) = sqlx::query_scalar::<_, String>(&format!(
|
|
"SELECT file_uuid FROM {} WHERE content_hash = $1 LIMIT 1",
|
|
videos_table
|
|
))
|
|
.bind(&content_hash_val)
|
|
.fetch_optional(db.pool())
|
|
.await
|
|
{
|
|
tracing::info!(
|
|
"[REGISTER] Content hash collision → already registered: {}",
|
|
existing_uuid
|
|
);
|
|
let existing_info: Option<(String, String, f64, i32, i32, f64, i64, Option<String>)> = sqlx::query_as(
|
|
&format!("SELECT file_name, file_path, duration, width, height, fps, total_frames, registration_time::text FROM {} WHERE file_uuid = $1", videos_table)
|
|
).bind(&existing_uuid).fetch_optional(db.pool()).await.unwrap_or(None);
|
|
if let Some((ename, epath, dur, w, h, f, tf, rt)) = existing_info {
|
|
return RegisterFileResponse {
|
|
success: true,
|
|
file_uuid: existing_uuid,
|
|
file_name: ename,
|
|
file_path: epath.clone(),
|
|
file_type: None,
|
|
duration: dur,
|
|
width: w as u32,
|
|
height: h as u32,
|
|
fps: f,
|
|
total_frames: tf as u64,
|
|
registration_time: rt,
|
|
already_exists: true,
|
|
message: format!("Content already registered: {}", epath),
|
|
};
|
|
}
|
|
return RegisterFileResponse {
|
|
success: true,
|
|
file_uuid: existing_uuid,
|
|
file_name: file_name.clone(),
|
|
file_path: canonical_path.clone(),
|
|
file_type: None,
|
|
duration: 0.0,
|
|
width: 0,
|
|
height: 0,
|
|
fps: 0.0,
|
|
total_frames: 0,
|
|
registration_time: None,
|
|
already_exists: true,
|
|
message: "Content already registered (identical file)".to_string(),
|
|
};
|
|
}
|
|
}
|
|
|
|
let final_name = resolve_filename(&db, &file_name, &content_hash_val).await;
|
|
|
|
let mac_address = crate::core::storage::uuid::get_mac_address();
|
|
let file_uuid = crate::core::storage::uuid::compute_birth_uuid(
|
|
&mac_address,
|
|
&birthday,
|
|
&canonical_path,
|
|
&final_name,
|
|
);
|
|
|
|
let temp_probe_json: serde_json::Value = if let Some(ref pre) = pre_data {
|
|
pre.get("probe_json").cloned().unwrap_or_default()
|
|
} else {
|
|
let scripts_dir = std::env::var("MOMENTRY_SCRIPTS_DIR")
|
|
.unwrap_or_else(|_| "/Users/accusys/momentry_core_0.1/scripts".to_string());
|
|
let python_path = std::env::var("MOMENTRY_PYTHON_PATH")
|
|
.unwrap_or_else(|_| "/opt/homebrew/bin/python3.11".to_string());
|
|
crate::core::probe::unified::unified_probe(&path, &scripts_dir, &python_path).await
|
|
};
|
|
let probe_json = Some(temp_probe_json.clone());
|
|
|
|
let has_video = temp_probe_json
|
|
.get("streams")
|
|
.and_then(|s| s.as_array())
|
|
.map_or(false, |streams| {
|
|
streams
|
|
.iter()
|
|
.any(|st| st.get("codec_type").and_then(|c| c.as_str()) == Some("video"))
|
|
});
|
|
let has_audio = temp_probe_json
|
|
.get("streams")
|
|
.and_then(|s| s.as_array())
|
|
.map_or(false, |streams| {
|
|
streams
|
|
.iter()
|
|
.any(|st| st.get("codec_type").and_then(|c| c.as_str()) == Some("audio"))
|
|
});
|
|
|
|
let final_file_type = if has_video {
|
|
Some("video".to_string())
|
|
} else if has_audio {
|
|
Some("audio".to_string())
|
|
} else {
|
|
Some(
|
|
temp_probe_json
|
|
.get("format")
|
|
.and_then(|f| f.get("file_type"))
|
|
.and_then(|v| v.as_str())
|
|
.unwrap_or("unknown")
|
|
.to_string(),
|
|
)
|
|
};
|
|
|
|
let duration = temp_probe_json
|
|
.get("format")
|
|
.and_then(|f| {
|
|
let src = if has_video { f.get("duration") } else { None };
|
|
src.and_then(|v| v.as_str())
|
|
.and_then(|s| s.parse::<f64>().ok())
|
|
})
|
|
.unwrap_or(0.0);
|
|
let mut width = 0u32;
|
|
let mut height = 0u32;
|
|
let mut fps = 0.0;
|
|
let mut total_frames = 0u64;
|
|
if let Some(streams) = temp_probe_json.get("streams").and_then(|s| s.as_array()) {
|
|
if let Some(s) = streams
|
|
.iter()
|
|
.find(|st| st.get("codec_type").and_then(|c| c.as_str()) == Some("video"))
|
|
{
|
|
width = s.get("width").and_then(|v| v.as_i64()).unwrap_or(0) as u32;
|
|
height = s.get("height").and_then(|v| v.as_i64()).unwrap_or(0) as u32;
|
|
if let Some(fps_str) = s.get("r_frame_rate").and_then(|v| v.as_str()) {
|
|
if let Some((num, den)) = fps_str.split_once('/') {
|
|
if let (Ok(n), Ok(d)) = (num.parse::<f64>(), den.parse::<f64>()) {
|
|
if d > 0.0 {
|
|
fps = n / d;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
total_frames = s
|
|
.get("nb_frames")
|
|
.and_then(|v| v.as_str())
|
|
.and_then(|s| s.parse().ok())
|
|
.unwrap_or_else(|| (duration * fps) as u64);
|
|
}
|
|
}
|
|
|
|
let status = "registered";
|
|
let _ = sqlx::query(&format!(
|
|
"INSERT INTO {} (file_uuid, file_path, file_name, file_type, duration, width, height, fps, probe_json, status, content_hash, registration_time) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, NOW()) ON CONFLICT (file_uuid) DO UPDATE SET file_path = EXCLUDED.file_path, file_name = EXCLUDED.file_name, status = EXCLUDED.status, content_hash = EXCLUDED.content_hash",
|
|
videos_table
|
|
))
|
|
.bind(&file_uuid).bind(&canonical_path).bind(&final_name).bind(&final_file_type)
|
|
.bind(duration).bind(width as i32).bind(height as i32).bind(fps)
|
|
.bind(&probe_json).bind(status).bind(&content_hash_val)
|
|
.execute(db.pool()).await;
|
|
|
|
let mut cut_done = false;
|
|
let mut scene_done = false;
|
|
if has_video && total_frames > 0 && fps > 0.0 {
|
|
let output_dir = std::env::var("MOMENTRY_OUTPUT_DIR")
|
|
.unwrap_or_else(|_| "/Users/accusys/momentry/output_dev".to_string());
|
|
let scripts_dir = std::env::var("MOMENTRY_SCRIPTS_DIR")
|
|
.unwrap_or_else(|_| "/Users/accusys/momentry_core_0.1/scripts".to_string());
|
|
let python_path = std::env::var("MOMENTRY_PYTHON_PATH")
|
|
.unwrap_or_else(|_| "/opt/homebrew/bin/python3.11".to_string());
|
|
|
|
let cut_path = std::path::Path::new(&output_dir).join(format!("{}.cut.json", file_uuid));
|
|
if !cut_path.exists() {
|
|
let cut_script = std::path::Path::new(&scripts_dir).join("cut_processor.py");
|
|
if cut_script.exists() {
|
|
let cut_output = std::process::Command::new(&python_path)
|
|
.arg(&cut_script)
|
|
.arg(&canonical_path)
|
|
.arg(&cut_path)
|
|
.output();
|
|
if let Ok(output) = cut_output {
|
|
if output.status.success() {
|
|
cut_done = true;
|
|
tracing::info!("[REGISTER] CUT completed for {}", file_uuid);
|
|
} else {
|
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
|
tracing::error!("[REGISTER] CUT failed for {}: {}", file_uuid, stderr);
|
|
}
|
|
} else {
|
|
tracing::error!("[REGISTER] CUT execution error for {}", file_uuid);
|
|
}
|
|
}
|
|
} else {
|
|
cut_done = true;
|
|
if let Ok(content) = std::fs::read_to_string(&cut_path) {
|
|
if let Ok(cut_data) = serde_json::from_str::<serde_json::Value>(&content) {
|
|
let scenes = cut_data
|
|
.get("scenes")
|
|
.and_then(|s| s.as_array())
|
|
.map(|a| a.len() as i32)
|
|
.unwrap_or(0);
|
|
tracing::info!(
|
|
"[REGISTER] CUT already exists: {} scenes for {}",
|
|
scenes,
|
|
file_uuid
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
let scene_path =
|
|
std::path::Path::new(&output_dir).join(format!("{}.scene.json", file_uuid));
|
|
if !scene_path.exists() {
|
|
let scene_script = std::path::Path::new(&scripts_dir).join("scene_classifier.py");
|
|
if scene_script.exists() {
|
|
let scene_output = std::process::Command::new(&python_path)
|
|
.arg(&scene_script)
|
|
.arg(&canonical_path)
|
|
.arg(&scene_path)
|
|
.arg("--sample-interval")
|
|
.arg("2")
|
|
.output();
|
|
if let Ok(output) = scene_output {
|
|
if output.status.success() {
|
|
scene_done = true;
|
|
tracing::info!(
|
|
"[REGISTER] Scene classification completed for {}",
|
|
file_uuid
|
|
);
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
scene_done = true;
|
|
}
|
|
}
|
|
|
|
let audio_tracks: Vec<serde_json::Value> = temp_probe_json
|
|
.get("streams")
|
|
.and_then(|s| s.as_array())
|
|
.map_or(vec![], |streams| {
|
|
streams
|
|
.iter()
|
|
.filter(|st| st.get("codec_type").and_then(|c| c.as_str()) == Some("audio"))
|
|
.map(|st| {
|
|
serde_json::json!({
|
|
"index": st.get("index").and_then(|v| v.as_i64()),
|
|
"codec": st.get("codec_name").and_then(|v| v.as_str()),
|
|
"channels": st.get("channels").and_then(|v| v.as_i64()),
|
|
"sample_rate": st.get("sample_rate").and_then(|v| v.as_str()),
|
|
"language": st.get("tags").and_then(|t| t.get("language")),
|
|
})
|
|
})
|
|
.collect()
|
|
});
|
|
let audio_tracks_json = serde_json::to_value(&audio_tracks).ok();
|
|
let cut_path = std::path::Path::new(
|
|
&std::env::var("MOMENTRY_OUTPUT_DIR")
|
|
.unwrap_or_else(|_| "/Users/accusys/momentry/output_dev".to_string()),
|
|
)
|
|
.join(format!("{}.cut.json", file_uuid));
|
|
let mut cut_count = 0i32;
|
|
let mut cut_max_duration = 0.0f64;
|
|
if let Ok(content) = std::fs::read_to_string(&cut_path) {
|
|
if let Ok(cut_data) = serde_json::from_str::<serde_json::Value>(&content) {
|
|
if let Some(scenes) = cut_data.get("scenes").and_then(|s| s.as_array()) {
|
|
cut_count = scenes.len() as i32;
|
|
cut_max_duration = scenes
|
|
.iter()
|
|
.filter_map(|s| {
|
|
let start = s.get("start_time").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
|
let end = s.get("end_time").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
|
if end > start {
|
|
Some(end - start)
|
|
} else {
|
|
None
|
|
}
|
|
})
|
|
.fold(0.0_f64, f64::max);
|
|
}
|
|
}
|
|
}
|
|
let _ = sqlx::query(
|
|
&format!("UPDATE {} SET cut_done = $1, scene_done = $2, audio_tracks = $3, cut_count = $4, cut_max_duration = $5 WHERE file_uuid = $6", videos_table)
|
|
)
|
|
.bind(cut_done).bind(scene_done).bind(&audio_tracks_json).bind(cut_count).bind(cut_max_duration).bind(&file_uuid)
|
|
.execute(db.pool()).await;
|
|
|
|
if let Some(json_val) = probe_json {
|
|
let probe_path = std::path::Path::new(
|
|
&std::env::var("MOMENTRY_OUTPUT_DIR")
|
|
.unwrap_or_else(|_| "/Users/accusys/momentry/output_dev".to_string()),
|
|
)
|
|
.join(format!("{}.probe.json", file_uuid));
|
|
let json_str = serde_json::to_string_pretty(&json_val).unwrap_or_default();
|
|
let _ = std::fs::write(&probe_path, json_str);
|
|
}
|
|
|
|
if final_file_type.as_deref() == Some("video") {
|
|
let auto_file_uuid = file_uuid.clone();
|
|
let auto_db = db.clone();
|
|
tokio::spawn(async move {
|
|
let identities_dir =
|
|
std::path::Path::new(&*crate::core::config::OUTPUT_DIR).join("identities");
|
|
let index_path = identities_dir.join("_index.json");
|
|
let cache_path = format!(
|
|
"{}/{}.tmdb.json",
|
|
*crate::core::config::OUTPUT_DIR,
|
|
auto_file_uuid
|
|
);
|
|
let cache_file = std::path::Path::new(&cache_path);
|
|
|
|
if index_path.exists() && cache_file.exists() {
|
|
tracing::info!(
|
|
"[AUTO-TMDB] Offline cache found for {}, running probe",
|
|
auto_file_uuid
|
|
);
|
|
if let Err(e) =
|
|
crate::core::tmdb::probe::probe_from_cache(&auto_db, &auto_file_uuid).await
|
|
{
|
|
tracing::warn!("[AUTO-TMDB] Probe failed for {}: {}", auto_file_uuid, e);
|
|
} else {
|
|
tracing::info!("[AUTO-TMDB] Probe completed for {}", auto_file_uuid);
|
|
}
|
|
} else {
|
|
tracing::info!(
|
|
"[AUTO-TMDB] No offline cache for {}, skipping",
|
|
auto_file_uuid
|
|
);
|
|
}
|
|
});
|
|
}
|
|
|
|
RegisterFileResponse {
|
|
success: true,
|
|
file_uuid,
|
|
file_name,
|
|
file_path: canonical_path,
|
|
file_type: final_file_type,
|
|
duration,
|
|
width,
|
|
height,
|
|
fps,
|
|
total_frames,
|
|
registration_time: None,
|
|
already_exists: false,
|
|
message: "File registered successfully".to_string(),
|
|
}
|
|
}
|
|
|
|
async fn register_file(
|
|
State(state): State<AppState>,
|
|
Json(req): Json<RegisterFileRequest>,
|
|
) -> Result<Json<RegisterFileResponse>, StatusCode> {
|
|
let file_path = req.file_path.clone();
|
|
let pattern = req.pattern;
|
|
|
|
if let Some(ref pat) = pattern {
|
|
let dir = std::path::Path::new(&file_path);
|
|
if !dir.is_dir() {
|
|
return Ok(Json(RegisterFileResponse {
|
|
success: false,
|
|
file_uuid: String::new(),
|
|
file_name: String::new(),
|
|
file_path: file_path.clone(),
|
|
file_type: None,
|
|
duration: 0.0,
|
|
width: 0,
|
|
height: 0,
|
|
fps: 0.0,
|
|
total_frames: 0,
|
|
registration_time: None,
|
|
already_exists: false,
|
|
message: format!(
|
|
"Pattern requires a directory, but path is not a dir: {}",
|
|
file_path
|
|
),
|
|
}));
|
|
}
|
|
let re = regex::Regex::new(pat).map_err(|e| {
|
|
tracing::error!("[REGISTER] Invalid regex pattern: {}", e);
|
|
StatusCode::BAD_REQUEST
|
|
})?;
|
|
|
|
let mut registered = 0u32;
|
|
let mut failed = 0u32;
|
|
let mut skipped = 0u32;
|
|
|
|
if let Ok(entries) = std::fs::read_dir(dir) {
|
|
for entry in entries.flatten() {
|
|
let entry_path = entry.path();
|
|
if !entry_path.is_file() {
|
|
continue;
|
|
}
|
|
let fname = entry_path
|
|
.file_name()
|
|
.and_then(|n| n.to_str())
|
|
.unwrap_or("");
|
|
if !re.is_match(fname) {
|
|
continue;
|
|
}
|
|
|
|
let result = register_single_file(
|
|
&state,
|
|
&entry_path.to_string_lossy().to_string(),
|
|
req.user_id,
|
|
None,
|
|
)
|
|
.await;
|
|
if result.success {
|
|
registered += 1;
|
|
} else if result.already_exists {
|
|
skipped += 1;
|
|
} else {
|
|
failed += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
return Ok(Json(RegisterFileResponse {
|
|
success: true,
|
|
file_uuid: format!("batch_{}_registered_{}_failed", registered, failed),
|
|
file_name: format!(
|
|
"{} files registered, {} skipped, {} failed",
|
|
registered, skipped, failed
|
|
),
|
|
file_path: file_path.clone(),
|
|
file_type: None,
|
|
duration: 0.0,
|
|
width: 0,
|
|
height: 0,
|
|
fps: 0.0,
|
|
total_frames: 0,
|
|
registration_time: None,
|
|
already_exists: false,
|
|
message: format!(
|
|
"Batch register: {} registered, {} skipped, {} failed",
|
|
registered, skipped, failed
|
|
),
|
|
}));
|
|
}
|
|
|
|
let resp = register_single_file(&state, &file_path, req.user_id, req.content_hash).await;
|
|
|
|
if resp.success
|
|
&& !resp.already_exists
|
|
&& resp.file_type.as_deref() == Some("video")
|
|
&& crate::core::config::get_auto_pipeline_enabled()
|
|
{
|
|
let auto_uuid = resp.file_uuid.clone();
|
|
let auto_state = state.clone();
|
|
tokio::spawn(async move {
|
|
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
|
|
let video_path: Option<String> = sqlx::query_scalar(&format!(
|
|
"SELECT file_path FROM {} WHERE file_uuid = $1",
|
|
schema::table_name("videos")
|
|
))
|
|
.bind(&auto_uuid)
|
|
.fetch_optional(auto_state.db.pool())
|
|
.await
|
|
.ok()
|
|
.flatten();
|
|
|
|
if let Some(ref vp) = video_path {
|
|
if let Ok(job) = auto_state.db.create_monitor_job(&auto_uuid, Some(vp)).await {
|
|
tracing::info!("[AUTO-PIPELINE] Job {} created for {}", job.id, auto_uuid);
|
|
let all_procs: Vec<&str> = vec![
|
|
"asr",
|
|
"cut",
|
|
"yolo",
|
|
"ocr",
|
|
"face",
|
|
"pose",
|
|
"asrx",
|
|
"visual_chunk",
|
|
"5w1h",
|
|
];
|
|
let total = sqlx::query_scalar::<_, i64>(&format!(
|
|
"SELECT COALESCE(total_frames, 0) FROM {} WHERE file_uuid = $1",
|
|
schema::table_name("videos")
|
|
))
|
|
.bind(&auto_uuid)
|
|
.fetch_one(auto_state.db.pool())
|
|
.await
|
|
.unwrap_or(0);
|
|
let _ = auto_state
|
|
.db
|
|
.init_processing_status(&auto_uuid, all_procs, total as u64)
|
|
.await;
|
|
let _ = sqlx::query(&format!(
|
|
"UPDATE {} SET status = 'processing' WHERE file_uuid = $1",
|
|
schema::table_name("videos")
|
|
))
|
|
.bind(&auto_uuid)
|
|
.execute(auto_state.db.pool())
|
|
.await;
|
|
tracing::info!("[AUTO-PIPELINE] Pipeline triggered for {}", auto_uuid);
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
Ok(Json(resp))
|
|
}
|
|
|
|
async fn probe_by_uuid(
|
|
State(state): State<AppState>,
|
|
Path(file_uuid): Path<String>,
|
|
) -> Result<Json<ProbeResponse>, (StatusCode, Json<serde_json::Value>)> {
|
|
let table = schema::table_name("videos");
|
|
let row: Option<(String, String)> = sqlx::query_as(&format!(
|
|
"SELECT file_name, file_path FROM {} WHERE file_uuid = $1",
|
|
table
|
|
))
|
|
.bind(&file_uuid)
|
|
.fetch_optional(state.db.pool())
|
|
.await
|
|
.map_err(|e| {
|
|
tracing::error!("DB error fetching video: {}", e);
|
|
(
|
|
StatusCode::INTERNAL_SERVER_ERROR,
|
|
Json(serde_json::json!({"error": format!("DB error: {}", e), "file_uuid": file_uuid})),
|
|
)
|
|
})?;
|
|
|
|
let (file_name, path) = row.ok_or_else(|| {
|
|
tracing::warn!("Video not found: {}", file_uuid);
|
|
(
|
|
StatusCode::NOT_FOUND,
|
|
Json(serde_json::json!({"error": "Video not found", "file_uuid": file_uuid})),
|
|
)
|
|
})?;
|
|
|
|
let probe_path = format!(
|
|
"{}/{}.probe.json",
|
|
crate::core::config::OUTPUT_DIR.as_str(),
|
|
file_uuid
|
|
);
|
|
|
|
let (probe_result, cached) = if let Ok(content) = std::fs::read_to_string(&probe_path) {
|
|
tracing::info!("Using cached probe.json: {}", probe_path);
|
|
let result: crate::core::probe::ProbeResult =
|
|
serde_json::from_str(&content).map_err(|e| {
|
|
tracing::error!("Failed to parse cached probe.json: {}", e);
|
|
(
|
|
StatusCode::INTERNAL_SERVER_ERROR,
|
|
Json(serde_json::json!({"error": format!("Failed to parse cached probe.json: {}", e), "file_uuid": file_uuid})),
|
|
)
|
|
})?;
|
|
(result, true)
|
|
} else {
|
|
let file_path = std::path::Path::new(&path);
|
|
if !file_path.exists() {
|
|
tracing::error!("File not found at path: {}", path);
|
|
return Err((
|
|
StatusCode::NOT_FOUND,
|
|
Json(
|
|
serde_json::json!({"error": "File does not exist at registered path", "file_uuid": file_uuid, "file_path": path}),
|
|
),
|
|
));
|
|
}
|
|
|
|
tracing::info!("Running ffprobe for: {}", path);
|
|
let result = crate::core::probe::probe_video(&path).map_err(|e| {
|
|
tracing::error!("ffprobe failed: {}", e);
|
|
(
|
|
StatusCode::INTERNAL_SERVER_ERROR,
|
|
Json(serde_json::json!({"error": format!("ffprobe failed: {}", e), "file_uuid": file_uuid, "file_path": path})),
|
|
)
|
|
})?;
|
|
|
|
let file_manager = FileManager::new(std::path::PathBuf::from(
|
|
crate::core::config::OUTPUT_DIR.as_str(),
|
|
));
|
|
let json_str = serde_json::to_string(&result).map_err(|e| {
|
|
tracing::error!("Failed to serialize probe result: {}", e);
|
|
(
|
|
StatusCode::INTERNAL_SERVER_ERROR,
|
|
Json(serde_json::json!({"error": format!("Failed to serialize probe result: {}", e), "file_uuid": file_uuid})),
|
|
)
|
|
})?;
|
|
file_manager
|
|
.save_json(&file_uuid, "probe", &json_str)
|
|
.map_err(|e| {
|
|
tracing::error!("Failed to save probe.json: {}", e);
|
|
(
|
|
StatusCode::INTERNAL_SERVER_ERROR,
|
|
Json(serde_json::json!({"error": format!("Failed to save probe.json: {}", e), "file_uuid": file_uuid})),
|
|
)
|
|
})?;
|
|
|
|
(result, false)
|
|
};
|
|
|
|
let duration = probe_result
|
|
.format
|
|
.duration
|
|
.as_ref()
|
|
.and_then(|s| s.parse::<f64>().ok())
|
|
.unwrap_or(0.0);
|
|
|
|
let mut width = 0u32;
|
|
let mut height = 0u32;
|
|
let mut fps = 0.0;
|
|
|
|
for stream in &probe_result.streams {
|
|
if stream.codec_type.as_deref() == Some("video") {
|
|
width = stream.width.unwrap_or(0);
|
|
height = stream.height.unwrap_or(0);
|
|
if let Some(fps_str) = &stream.r_frame_rate {
|
|
fps = if fps_str.contains('/') {
|
|
let parts: Vec<&str> = fps_str.split('/').collect();
|
|
if parts.len() == 2 {
|
|
let num: f64 = parts[0].parse().unwrap_or(0.0);
|
|
let den: f64 = parts[1].parse().unwrap_or(1.0);
|
|
if den > 0.0 {
|
|
num / den
|
|
} else {
|
|
0.0
|
|
}
|
|
} else {
|
|
0.0
|
|
}
|
|
} else {
|
|
fps_str.parse().unwrap_or(0.0)
|
|
};
|
|
}
|
|
}
|
|
}
|
|
|
|
let total_frames = probe_result
|
|
.streams
|
|
.iter()
|
|
.find(|s| s.codec_type.as_deref() == Some("video"))
|
|
.and_then(|s| s.nb_frames.as_ref())
|
|
.and_then(|n| n.parse::<i64>().ok())
|
|
.unwrap_or_else(|| (duration * fps).floor() as i64);
|
|
|
|
let _ = sqlx::query(&format!(
|
|
"UPDATE {} SET duration = $1, width = $2, height = $3, fps = $4, total_frames = $5 WHERE file_uuid = $6",
|
|
schema::table_name("videos")
|
|
))
|
|
.bind(duration)
|
|
.bind(width as i32)
|
|
.bind(height as i32)
|
|
.bind(fps)
|
|
.bind(total_frames)
|
|
.bind(&file_uuid)
|
|
.execute(state.db.pool())
|
|
.await;
|
|
|
|
let file_size = std::fs::metadata(&path).ok().map(|m| m.len() as i64);
|
|
|
|
let file_type = if width > 0 && height > 0 {
|
|
Some("video".to_string())
|
|
} else {
|
|
None
|
|
};
|
|
|
|
Ok(Json(ProbeResponse {
|
|
file_uuid,
|
|
file_name,
|
|
file_size,
|
|
duration,
|
|
width,
|
|
height,
|
|
fps,
|
|
total_frames: total_frames as u64,
|
|
file_type,
|
|
probe_result: if cached {
|
|
Some(serde_json::to_value(&probe_result).unwrap_or_default())
|
|
} else {
|
|
None
|
|
},
|
|
}))
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
struct UnregisterResponse {
|
|
success: bool,
|
|
message: String,
|
|
file_uuid: String,
|
|
deleted_face_detections: u64,
|
|
deleted_processor_results: u64,
|
|
deleted_chunks: u64,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
struct UnregisterRequest {
|
|
file_uuid: Option<String>,
|
|
file_path: Option<String>,
|
|
}
|
|
|
|
fn delete_output_files(uuid: &str) {
|
|
let output_dir = config::OUTPUT_DIR.to_string();
|
|
if let Ok(entries) = std::fs::read_dir(&output_dir) {
|
|
for entry in entries.flatten() {
|
|
let path = entry.path();
|
|
if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
|
|
if name.starts_with(uuid) {
|
|
let _ = std::fs::remove_file(&path);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn unregister(
|
|
State(state): State<AppState>,
|
|
Json(req): Json<UnregisterRequest>,
|
|
) -> Result<Json<UnregisterResponse>, StatusCode> {
|
|
let uuid = match req.file_uuid {
|
|
Some(u) => u,
|
|
None => return Err(StatusCode::BAD_REQUEST),
|
|
};
|
|
|
|
tracing::info!("[UNREGISTER] Unregistering file: {}", uuid);
|
|
|
|
let videos_table = schema::table_name("videos");
|
|
let face_table = schema::table_name("face_detections");
|
|
let processor_table = schema::table_name("processor_results");
|
|
let chunks_table = schema::table_name("chunk");
|
|
let parent_chunks_table = schema::table_name("parent_chunks");
|
|
|
|
let deleted_faces: i64 =
|
|
sqlx::query(&format!("DELETE FROM {} WHERE file_uuid = $1", face_table))
|
|
.bind(&uuid)
|
|
.execute(state.db.pool())
|
|
.await
|
|
.map_err(|e| {
|
|
tracing::error!("[unregister] Failed to delete faces: {}", e);
|
|
StatusCode::INTERNAL_SERVER_ERROR
|
|
})?
|
|
.rows_affected() as i64;
|
|
|
|
let deleted_processors: i64 = sqlx::query(&format!(
|
|
"DELETE FROM {} WHERE file_uuid = $1",
|
|
processor_table
|
|
))
|
|
.bind(&uuid)
|
|
.execute(state.db.pool())
|
|
.await
|
|
.map_err(|e| {
|
|
tracing::error!("[unregister] Failed to delete processors: {}", e);
|
|
StatusCode::INTERNAL_SERVER_ERROR
|
|
})?
|
|
.rows_affected() as i64;
|
|
|
|
let deleted_parent_chunks: i64 = sqlx::query(&format!(
|
|
"DELETE FROM {} WHERE uuid = $1",
|
|
parent_chunks_table
|
|
))
|
|
.bind(&uuid)
|
|
.execute(state.db.pool())
|
|
.await
|
|
.map_err(|e| {
|
|
tracing::error!("[unregister] Failed to delete parent chunks: {}", e);
|
|
StatusCode::INTERNAL_SERVER_ERROR
|
|
})?
|
|
.rows_affected() as i64;
|
|
|
|
let deleted_chunks: i64 = sqlx::query(&format!("DELETE FROM {} WHERE file_uuid = $1", chunks_table))
|
|
.bind(&uuid)
|
|
.execute(state.db.pool())
|
|
.await
|
|
.map_err(|e| {
|
|
tracing::error!("[unregister] Failed to delete chunks: {}", e);
|
|
StatusCode::INTERNAL_SERVER_ERROR
|
|
})?
|
|
.rows_affected() as i64;
|
|
|
|
// Delete pre_chunks
|
|
let pre_chunks_table = schema::table_name("pre_chunks");
|
|
let deleted_pre_chunks: i64 = sqlx::query(&format!(
|
|
"DELETE FROM {} WHERE file_uuid = $1",
|
|
pre_chunks_table
|
|
))
|
|
.bind(&uuid)
|
|
.execute(state.db.pool())
|
|
.await
|
|
.map_err(|e| {
|
|
tracing::error!("[unregister] Failed to delete pre_chunks: {}", e);
|
|
StatusCode::INTERNAL_SERVER_ERROR
|
|
})?
|
|
.rows_affected() as i64;
|
|
|
|
sqlx::query(&format!(
|
|
"DELETE FROM {} WHERE file_uuid = $1",
|
|
videos_table
|
|
))
|
|
.bind(&uuid)
|
|
.execute(state.db.pool())
|
|
.await
|
|
.map_err(|e| {
|
|
tracing::error!("[unregister] Failed: {}", e);
|
|
StatusCode::INTERNAL_SERVER_ERROR
|
|
})?;
|
|
|
|
delete_output_files(&uuid);
|
|
|
|
Ok(Json(UnregisterResponse {
|
|
success: true,
|
|
message: format!("File {} unregistered successfully.", uuid),
|
|
file_uuid: uuid,
|
|
deleted_face_detections: deleted_faces as u64,
|
|
deleted_processor_results: deleted_processors as u64,
|
|
deleted_chunks: (deleted_chunks + deleted_parent_chunks) as u64,
|
|
}))
|
|
}
|
|
|
|
pub fn file_routes() -> Router<AppState> {
|
|
Router::new()
|
|
.route("/api/v1/files/register", post(register_file))
|
|
.route("/api/v1/files/lookup", get(lookup_file_by_name))
|
|
.route("/api/v1/unregister", post(unregister))
|
|
.route("/api/v1/file/:file_uuid/probe", get(probe_by_uuid))
|
|
}
|