warren/momentry_core
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

feat: ASRX hybrid pipeline, identity history, worker fixes, checkpoint system

Browse Source
This commit is contained in:
Accusys
2026-06-02 07:13:23 +08:00
parent e3066c3f49
commit e1572907ae
198 changed files with 43705 additions and 8910 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/api/files.rs
+121 -77
View File
@@ -12,7 +12,7 @@ use std::collections::HashMap;
use super::types::AppState;
use crate::core::config;
use crate::core::db::schema;
use crate::core::db::{Database, PostgresDb};
use crate::core::db::{Database, PostgresDb, QdrantDb, RedisClient};
use crate::core::storage::content_hash;
use crate::FileManager;
@@ -767,17 +767,7 @@ async fn register_file(
if let Some(ref vp) = video_path {
if let Ok(job) = auto_state.db.create_monitor_job(&auto_uuid, Some(vp)).await {
tracing::info!("[AUTO-PIPELINE] Job {} created for {}", job.id, auto_uuid);
let all_procs: Vec<&str> = vec![
"asr",
"cut",
"yolo",
"ocr",
"face",
"pose",
"asrx",
"visual_chunk",
"5w1h",
];
let all_procs: Vec<&str> = vec!["cut", "yolo", "ocr", "face", "pose", "asrx"];
let total = sqlx::query_scalar::<_, i64>(&format!(
"SELECT COALESCE(total_frames, 0) FROM {} WHERE file_uuid = $1",
schema::table_name("videos")
@@ -986,6 +976,10 @@ struct UnregisterResponse {
deleted_face_detections: u64,
deleted_processor_results: u64,
deleted_chunks: u64,
deleted_tkg_nodes: u64,
deleted_qdrant_vectors: Option<u64>,
deleted_redis_keys: Option<u64>,
deleted_output_files: u64,
}
#[derive(Debug, Deserialize)]
@@ -994,18 +988,30 @@ struct UnregisterRequest {
file_path: Option<String>,
}
fn delete_output_files(uuid: &str) {
let output_dir = config::OUTPUT_DIR.to_string();
if let Ok(entries) = std::fs::read_dir(&output_dir) {
for entry in entries.flatten() {
let path = entry.path();
if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
if name.starts_with(uuid) {
let _ = std::fs::remove_file(&path);
fn delete_output_files(uuid: &str) -> u64 {
let mut deleted_count = 0u64;
let output_dirs = [
config::OUTPUT_DIR.to_string(),
"/Users/accusys/momentry/output_dev".to_string(),
"/Users/accusys/momentry/output".to_string(),
];
for output_dir in &output_dirs {
if let Ok(entries) = std::fs::read_dir(output_dir) {
for entry in entries.flatten() {
let path = entry.path();
if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
if name.starts_with(uuid) && name.ends_with(".json") {
if std::fs::remove_file(&path).is_ok() {
deleted_count += 1;
tracing::info!("[UNREGISTER] Deleted output file: {}", name);
}
}
}
}
}
}
deleted_count
}
async fn unregister(
@@ -1024,65 +1030,54 @@ async fn unregister(
let processor_table = schema::table_name("processor_results");
let chunks_table = schema::table_name("chunk");
let parent_chunks_table = schema::table_name("parent_chunks");
let deleted_faces: i64 =
sqlx::query(&format!("DELETE FROM {} WHERE file_uuid = $1", face_table))
.bind(&uuid)
.execute(state.db.pool())
.await
.map_err(|e| {
tracing::error!("[unregister] Failed to delete faces: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?
.rows_affected() as i64;
let deleted_processors: i64 = sqlx::query(&format!(
"DELETE FROM {} WHERE file_uuid = $1",
processor_table
))
.bind(&uuid)
.execute(state.db.pool())
.await
.map_err(|e| {
tracing::error!("[unregister] Failed to delete processors: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?
.rows_affected() as i64;
let deleted_parent_chunks: i64 = sqlx::query(&format!(
"DELETE FROM {} WHERE uuid = $1",
parent_chunks_table
))
.bind(&uuid)
.execute(state.db.pool())
.await
.map_err(|e| {
tracing::error!("[unregister] Failed to delete parent chunks: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?
.rows_affected() as i64;
let deleted_chunks: i64 = sqlx::query(&format!("DELETE FROM {} WHERE file_uuid = $1", chunks_table))
.bind(&uuid)
.execute(state.db.pool())
.await
.map_err(|e| {
tracing::error!("[unregister] Failed to delete chunks: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?
.rows_affected() as i64;
// Delete pre_chunks
let pre_chunks_table = schema::table_name("pre_chunks");
let deleted_pre_chunks: i64 = sqlx::query(&format!(
"DELETE FROM {} WHERE file_uuid = $1",
pre_chunks_table
let tkg_nodes_table = schema::table_name("tkg_nodes");
let cuts_table = schema::table_name("cuts");
let strangers_table = schema::table_name("strangers");
let chunk_vectors_table = schema::table_name("chunk_vectors");
let monitor_jobs_table = schema::table_name("monitor_jobs");
let frames_table = schema::table_name("frames");
let mut tx = state.db.pool().begin().await.map_err(|e| {
tracing::error!("[unregister] Failed to start transaction: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
macro_rules! delete_safe {
($table:expr, $where:expr, $bind:expr, $label:expr) => {{
sqlx::query(&format!("DELETE FROM {} WHERE {}", $table, $where))
.bind($bind)
.execute(&mut *tx)
.await
.map_err(|e| {
tracing::error!("[unregister] Failed to delete {}: {}", $label, e);
StatusCode::INTERNAL_SERVER_ERROR
})?
.rows_affected() as i64
}};
}
let deleted_faces = delete_safe!(face_table, "file_uuid = $1", &uuid, "faces");
let deleted_processors = delete_safe!(processor_table, "file_uuid = $1", &uuid, "processors");
let deleted_parent_chunks =
delete_safe!(parent_chunks_table, "uuid = $1", &uuid, "parent chunks");
let deleted_chunks = delete_safe!(chunks_table, "file_uuid = $1", &uuid, "chunks");
let deleted_pre_chunks = delete_safe!(pre_chunks_table, "file_uuid = $1", &uuid, "pre_chunks");
let deleted_tkg_nodes = delete_safe!(tkg_nodes_table, "file_uuid = $1", &uuid, "TKG nodes");
let deleted_cuts = delete_safe!(cuts_table, "file_uuid = $1", &uuid, "cuts");
let deleted_strangers = delete_safe!(strangers_table, "file_uuid = $1", &uuid, "strangers");
let deleted_chunk_vectors =
delete_safe!(chunk_vectors_table, "uuid = $1", &uuid, "chunk vectors");
let deleted_monitor_jobs = delete_safe!(monitor_jobs_table, "uuid = $1", &uuid, "monitor jobs");
let deleted_frames: i64 = sqlx::query(&format!(
"DELETE FROM {} WHERE file_id = (SELECT id FROM {} WHERE file_uuid = $1)",
frames_table, videos_table
))
.bind(&uuid)
.execute(state.db.pool())
.execute(&mut *tx)
.await
.map_err(|e| {
tracing::error!("[unregister] Failed to delete pre_chunks: {}", e);
tracing::error!("[unregister] Failed to delete frames: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?
.rows_affected() as i64;
@@ -1092,14 +1087,59 @@ async fn unregister(
videos_table
))
.bind(&uuid)
.execute(state.db.pool())
.execute(&mut *tx)
.await
.map_err(|e| {
tracing::error!("[unregister] Failed: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
delete_output_files(&uuid);
tx.commit().await.map_err(|e| {
tracing::error!("[unregister] Failed to commit transaction: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
tracing::info!(
"[UNREGISTER] Deleted: {} faces, {} processors, {} parent_chunks, {} chunks, {} pre_chunks, {} tkg_nodes, {} cuts, {} strangers, {} chunk_vectors, {} monitor_jobs, {} frames",
deleted_faces, deleted_processors, deleted_parent_chunks, deleted_chunks,
deleted_pre_chunks, deleted_tkg_nodes, deleted_cuts, deleted_strangers,
deleted_chunk_vectors, deleted_monitor_jobs, deleted_frames
);
let deleted_output_files = delete_output_files(&uuid);
let deleted_qdrant_vectors = {
let qdrant = QdrantDb::new();
match qdrant.delete_by_uuid(&uuid).await {
Ok(_) => {
tracing::info!("[UNREGISTER] Deleted Qdrant vectors for {}", uuid);
Some(1)
}
Err(e) => {
tracing::warn!("[UNREGISTER] Failed to delete Qdrant vectors: {}", e);
None
}
}
};
let deleted_redis_keys = {
match RedisClient::new() {
Ok(redis) => match redis.delete_worker_job(&uuid).await {
Ok(_) => {
tracing::info!("[UNREGISTER] Deleted Redis keys for {}", uuid);
Some(1)
}
Err(e) => {
tracing::warn!("[UNREGISTER] Failed to delete Redis keys: {}", e);
None
}
},
Err(e) => {
tracing::warn!("[UNREGISTER] Failed to create Redis client: {}", e);
None
}
}
};
Ok(Json(UnregisterResponse {
success: true,
@@ -1107,7 +1147,11 @@ async fn unregister(
file_uuid: uuid,
deleted_face_detections: deleted_faces as u64,
deleted_processor_results: deleted_processors as u64,
deleted_chunks: (deleted_chunks + deleted_parent_chunks) as u64,
deleted_chunks: (deleted_chunks + deleted_parent_chunks + deleted_pre_chunks) as u64,
deleted_tkg_nodes: deleted_tkg_nodes as u64,
deleted_qdrant_vectors,
deleted_redis_keys,
deleted_output_files,
}))
}