warren/momentry_core
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

refactor: cargo fmt across 27 files + behavioral fixes

Browse Source 
Behavioral changes:
- postgres_db: reorder default processors (cut first), prevent overwriting completed status
- qdrant_db: fix scroll pagination exit condition (next.is_none())
- job_worker: add idempotency guards for face trace / TKG build; better error logging
- main.rs: add LineWriter for stdout buffering

Remaining diff is cargo fmt reformatting (line wrapping, import ordering).
This commit is contained in:
Accusys
2026-07-11 02:03:28 +08:00
parent 701727fd08
commit b98a362de5
27 changed files with 1216 additions and 546 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/api/scan.rs
+254 -81
View File
@@ -460,7 +460,10 @@ async fn get_ingestion_status(
{"key": "file_uuid", "match": {"value": file_uuid}}
]
});
let points = qdrant.scroll_all_points("_faces", face_filter, 1000).await.unwrap_or_default();
let points = qdrant
.scroll_all_points("_faces", face_filter, 1000)
.await
.unwrap_or_default();
let face_total = points.len() as i64;
let mut trace_ids: std::collections::HashSet<i64> = std::collections::HashSet::new();
@@ -564,7 +567,11 @@ async fn get_ingestion_status(
// Get related identities from Qdrant _faces
let related_identity_ids: Vec<i64> = identity_ids.into_iter().collect();
let related_identities: Vec<IdentityRef> = if !related_identity_ids.is_empty() {
let id_list: String = related_identity_ids.iter().map(|id| id.to_string()).collect::<Vec<_>>().join(",");
let id_list: String = related_identity_ids
.iter()
.map(|id| id.to_string())
.collect::<Vec<_>>()
.join(",");
match sqlx::query_as::<_, (String, String)>(&format!(
"SELECT DISTINCT uuid::text, name FROM {identities} \
WHERE id IN ({id_list}) ORDER BY name"
@@ -592,21 +599,25 @@ async fn get_ingestion_status(
// Check if job is completed - if so, all ingestion steps are considered done
let mj_table = schema::table_name("monitor_jobs");
let job_completed: bool = sqlx::query_scalar::<_, String>(&format!(
"SELECT status FROM {mj_table} WHERE uuid = $1"
))
.bind(&file_uuid)
.fetch_optional(pool)
.await
.unwrap_or(None)
.map(|s| s == "completed")
.unwrap_or(false);
let job_completed: bool =
sqlx::query_scalar::<_, String>(&format!("SELECT status FROM {mj_table} WHERE uuid = $1"))
.bind(&file_uuid)
.fetch_optional(pool)
.await
.unwrap_or(None)
.map(|s| s == "completed")
.unwrap_or(false);
macro_rules! step {
($name:expr, $done:expr, $detail:expr) => {
IngestionStep {
name: $name.into(),
status: if $done || job_completed { "done" } else { "pending" }.into(),
status: if $done || job_completed {
"done"
} else {
"pending"
}
.into(),
detail: $detail,
}
};
@@ -639,24 +650,92 @@ async fn get_ingestion_status(
Some(format!("{trace_count} traces / {face_total} detections"))
),
// TKG Nodes
step!("tkg_face_track", face_track_nodes > 0, Some(format!("{face_track_nodes} nodes"))),
step!("tkg_gaze_track", gaze_track_nodes > 0, Some(format!("{gaze_track_nodes} nodes"))),
step!("tkg_lip_track", lip_track_nodes > 0, Some(format!("{lip_track_nodes} nodes"))),
step!("tkg_text_region", text_region_nodes > 0, Some(format!("{text_region_nodes} nodes"))),
step!("tkg_appearance", appearance_nodes > 0, Some(format!("{appearance_nodes} nodes"))),
step!("tkg_accessory", accessory_nodes > 0, Some(format!("{accessory_nodes} nodes"))),
step!("tkg_object", object_nodes > 0, Some(format!("{object_nodes} nodes"))),
step!("tkg_hand", hand_nodes > 0, Some(format!("{hand_nodes} nodes"))),
step!("tkg_speaker", speaker_nodes > 0, Some(format!("{speaker_nodes} nodes"))),
step!(
"tkg_face_track",
face_track_nodes > 0,
Some(format!("{face_track_nodes} nodes"))
),
step!(
"tkg_gaze_track",
gaze_track_nodes > 0,
Some(format!("{gaze_track_nodes} nodes"))
),
step!(
"tkg_lip_track",
lip_track_nodes > 0,
Some(format!("{lip_track_nodes} nodes"))
),
step!(
"tkg_text_region",
text_region_nodes > 0,
Some(format!("{text_region_nodes} nodes"))
),
step!(
"tkg_appearance",
appearance_nodes > 0,
Some(format!("{appearance_nodes} nodes"))
),
step!(
"tkg_accessory",
accessory_nodes > 0,
Some(format!("{accessory_nodes} nodes"))
),
step!(
"tkg_object",
object_nodes > 0,
Some(format!("{object_nodes} nodes"))
),
step!(
"tkg_hand",
hand_nodes > 0,
Some(format!("{hand_nodes} nodes"))
),
step!(
"tkg_speaker",
speaker_nodes > 0,
Some(format!("{speaker_nodes} nodes"))
),
// TKG Edges
step!("tkg_co_occurrence", co_occurrence_edges > 0, Some(format!("{co_occurrence_edges} edges"))),
step!("tkg_speaker_face", speaker_face_edges > 0, Some(format!("{speaker_face_edges} edges"))),
step!("tkg_face_face", face_face_edges > 0, Some(format!("{face_face_edges} edges"))),
step!("tkg_mutual_gaze", mutual_gaze_edges > 0, Some(format!("{mutual_gaze_edges} edges"))),
step!("tkg_lip_sync", lip_sync_edges > 0, Some(format!("{lip_sync_edges} edges"))),
step!("tkg_has_appearance", has_appearance_edges > 0, Some(format!("{has_appearance_edges} edges"))),
step!("tkg_wears", wears_edges > 0, Some(format!("{wears_edges} edges"))),
step!("tkg_hand_object", hand_object_edges > 0, Some(format!("{hand_object_edges} edges"))),
step!(
"tkg_co_occurrence",
co_occurrence_edges > 0,
Some(format!("{co_occurrence_edges} edges"))
),
step!(
"tkg_speaker_face",
speaker_face_edges > 0,
Some(format!("{speaker_face_edges} edges"))
),
step!(
"tkg_face_face",
face_face_edges > 0,
Some(format!("{face_face_edges} edges"))
),
step!(
"tkg_mutual_gaze",
mutual_gaze_edges > 0,
Some(format!("{mutual_gaze_edges} edges"))
),
step!(
"tkg_lip_sync",
lip_sync_edges > 0,
Some(format!("{lip_sync_edges} edges"))
),
step!(
"tkg_has_appearance",
has_appearance_edges > 0,
Some(format!("{has_appearance_edges} edges"))
),
step!(
"tkg_wears",
wears_edges > 0,
Some(format!("{wears_edges} edges"))
),
step!(
"tkg_hand_object",
hand_object_edges > 0,
Some(format!("{hand_object_edges} edges"))
),
// Rule 2
step!(
"rule2_relationship",
@@ -703,34 +782,45 @@ async fn get_file_stats(
.unwrap_or_else(|| (file_uuid.clone(), None, None));
// 2. Get processor status from processing_status JSONB
let processing_status: serde_json::Value =
sqlx::query_scalar(&format!(
"SELECT processing_status FROM {} WHERE file_uuid = $1",
videos_table
))
.bind(&file_uuid)
.fetch_optional(pool)
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
.unwrap_or(serde_json::json!({}));
let processing_status: serde_json::Value = sqlx::query_scalar(&format!(
"SELECT processing_status FROM {} WHERE file_uuid = $1",
videos_table
))
.bind(&file_uuid)
.fetch_optional(pool)
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
.unwrap_or(serde_json::json!({}));
let processors: Vec<ProcessorStatus> = processing_status
let progress_map = processing_status
.get("progress")
.and_then(|p| p.as_object())
.map(|progress| {
progress
.iter()
.filter_map(|(name, info)| {
.cloned()
.unwrap_or_default();
let active_order: Vec<String> = processing_status
.get("active_processors")
.and_then(|a| a.as_array())
.map(|arr| {
arr.iter()
.filter_map(|v| v.as_str().map(|s| s.to_uppercase()))
.collect()
})
.unwrap_or_default();
let processors: Vec<ProcessorStatus> = if !active_order.is_empty() {
active_order
.iter()
.filter_map(|name| {
progress_map.get(name).and_then(|info| {
info.as_object().map(|obj| {
let status = obj
.get("status")
.and_then(|s| s.as_str())
.unwrap_or("pending")
.to_string();
let progress_val = obj
.get("percentage")
.and_then(|p| p.as_u64())
.unwrap_or(0) as u32;
let progress_val =
obj.get("percentage").and_then(|p| p.as_u64()).unwrap_or(0) as u32;
let message = obj
.get("message")
.and_then(|m| m.as_str())
@@ -743,9 +833,34 @@ async fn get_file_stats(
}
})
})
.collect()
})
.unwrap_or_default();
})
.collect()
} else {
progress_map
.iter()
.filter_map(|(name, info)| {
info.as_object().map(|obj| {
let status = obj
.get("status")
.and_then(|s| s.as_str())
.unwrap_or("pending")
.to_string();
let progress_val =
obj.get("percentage").and_then(|p| p.as_u64()).unwrap_or(0) as u32;
let message = obj
.get("message")
.and_then(|m| m.as_str())
.map(|s| s.to_string());
ProcessorStatus {
name: name.clone(),
status,
progress: progress_val,
message,
}
})
})
.collect()
};
// 3. Get PostgreSQL counts
let chunk_table = schema::table_name("chunk");
@@ -853,10 +968,22 @@ async fn get_file_stats(
let tkg_nodes_table = schema::table_name("tkg_nodes");
let tkg_edges_table = schema::table_name("tkg_edges");
let tkg_nodes_total: i64 = sqlx::query_scalar::<_, i64>(&format!("SELECT COUNT(*) FROM {} WHERE file_uuid = $1", tkg_nodes_table))
.bind(&file_uuid).fetch_one(pool).await.unwrap_or(0);
let tkg_edges_total: i64 = sqlx::query_scalar::<_, i64>(&format!("SELECT COUNT(*) FROM {} WHERE file_uuid = $1", tkg_edges_table))
.bind(&file_uuid).fetch_one(pool).await.unwrap_or(0);
let tkg_nodes_total: i64 = sqlx::query_scalar::<_, i64>(&format!(
"SELECT COUNT(*) FROM {} WHERE file_uuid = $1",
tkg_nodes_table
))
.bind(&file_uuid)
.fetch_one(pool)
.await
.unwrap_or(0);
let tkg_edges_total: i64 = sqlx::query_scalar::<_, i64>(&format!(
"SELECT COUNT(*) FROM {} WHERE file_uuid = $1",
tkg_edges_table
))
.bind(&file_uuid)
.fetch_one(pool)
.await
.unwrap_or(0);
let tkg = TkgFileStats {
total_nodes: tkg_nodes_total,
@@ -870,12 +997,14 @@ async fn get_file_stats(
object_nodes: count_nodes(pool, &tkg_nodes_table, &file_uuid, "yolo_object").await,
hand_nodes: count_nodes(pool, &tkg_nodes_table, &file_uuid, "hand").await,
speaker_nodes: count_nodes(pool, &tkg_nodes_table, &file_uuid, "speaker").await,
co_occurrence_edges: count_edges(pool, &tkg_edges_table, &file_uuid, "CO_OCCURS_WITH").await,
co_occurrence_edges: count_edges(pool, &tkg_edges_table, &file_uuid, "CO_OCCURS_WITH")
.await,
speaker_face_edges: count_edges(pool, &tkg_edges_table, &file_uuid, "SPEAKS_AS").await,
face_face_edges: count_edges(pool, &tkg_edges_table, &file_uuid, "FACE_TO_FACE").await,
mutual_gaze_edges: count_edges(pool, &tkg_edges_table, &file_uuid, "MUTUAL_GAZE").await,
lip_sync_edges: count_edges(pool, &tkg_edges_table, &file_uuid, "LIP_SYNC").await,
has_appearance_edges: count_edges(pool, &tkg_edges_table, &file_uuid, "HAS_APPEARANCE").await,
has_appearance_edges: count_edges(pool, &tkg_edges_table, &file_uuid, "HAS_APPEARANCE")
.await,
wears_edges: count_edges(pool, &tkg_edges_table, &file_uuid, "WEARS").await,
hand_object_edges: count_edges(pool, &tkg_edges_table, &file_uuid, "HAND_OBJECT").await,
..Default::default()
@@ -895,7 +1024,8 @@ async fn get_file_stats(
let identity_agent = IdentityAgentStats {
clusters: 0, // From face_clustered.json if available
identities_created: face_identities.len() as i64,
tmdb_matches: seed_points.iter()
tmdb_matches: seed_points
.iter()
.filter(|p| p["payload"]["source"].as_str() == Some("tmdb"))
.count() as i64,
speaker_bindings: speaker_points.len() as i64,
@@ -946,10 +1076,7 @@ pub fn scan_routes() -> Router<AppState> {
"/api/v1/stats/ingestion-status/:file_uuid",
get(get_ingestion_status),
)
.route(
"/api/v1/stats/file/:file_uuid",
get(get_file_stats),
)
.route("/api/v1/stats/file/:file_uuid", get(get_file_stats))
.route(
"/api/v1/stats/pipeline/:file_uuid",
get(get_pipeline_progress_handler),
@@ -969,9 +1096,13 @@ async fn get_pipeline_progress_handler(
let mj_table = schema::table_name("monitor_jobs");
// Compute actual progress from DB state
let sentence_count: i64 = sqlx::query_scalar::<_, i64>(
&format!("SELECT COUNT(*) FROM {chunk_table} WHERE file_uuid = $1 AND chunk_type = 'sentence'")
).bind(&file_uuid).fetch_one(pool).await.unwrap_or(0);
let sentence_count: i64 = sqlx::query_scalar::<_, i64>(&format!(
"SELECT COUNT(*) FROM {chunk_table} WHERE file_uuid = $1 AND chunk_type = 'sentence'"
))
.bind(&file_uuid)
.fetch_one(pool)
.await
.unwrap_or(0);
let sentence_embedded: i64 = sqlx::query_scalar::<_, i64>(
&format!("SELECT COUNT(*) FROM {chunk_table} WHERE file_uuid = $1 AND chunk_type = 'sentence' AND embedding IS NOT NULL")
@@ -981,17 +1112,29 @@ async fn get_pipeline_progress_handler(
&format!("SELECT COUNT(*) FROM {pr_table} pr JOIN {mj_table} mj ON pr.job_id = mj.id WHERE mj.uuid = $1 AND pr.processor = 'face' AND pr.status = 'completed'")
).bind(&file_uuid).fetch_one(pool).await.unwrap_or(0);
let tkg_node_count: i64 = sqlx::query_scalar::<_, i64>(
&format!("SELECT COUNT(*) FROM {tkg_nodes_table} WHERE file_uuid = $1")
).bind(&file_uuid).fetch_one(pool).await.unwrap_or(0);
let tkg_node_count: i64 = sqlx::query_scalar::<_, i64>(&format!(
"SELECT COUNT(*) FROM {tkg_nodes_table} WHERE file_uuid = $1"
))
.bind(&file_uuid)
.fetch_one(pool)
.await
.unwrap_or(0);
let tkg_edge_count: i64 = sqlx::query_scalar::<_, i64>(
&format!("SELECT COUNT(*) FROM {tkg_edges_table} WHERE file_uuid = $1")
).bind(&file_uuid).fetch_one(pool).await.unwrap_or(0);
let tkg_edge_count: i64 = sqlx::query_scalar::<_, i64>(&format!(
"SELECT COUNT(*) FROM {tkg_edges_table} WHERE file_uuid = $1"
))
.bind(&file_uuid)
.fetch_one(pool)
.await
.unwrap_or(0);
let relationship_count: i64 = sqlx::query_scalar::<_, i64>(
&format!("SELECT COUNT(*) FROM {chunk_table} WHERE file_uuid = $1 AND chunk_type = 'relationship'")
).bind(&file_uuid).fetch_one(pool).await.unwrap_or(0);
let relationship_count: i64 = sqlx::query_scalar::<_, i64>(&format!(
"SELECT COUNT(*) FROM {chunk_table} WHERE file_uuid = $1 AND chunk_type = 'relationship'"
))
.bind(&file_uuid)
.fetch_one(pool)
.await
.unwrap_or(0);
let asrx_completed: i64 = sqlx::query_scalar::<_, i64>(
&format!("SELECT COUNT(*) FROM {pr_table} pr JOIN {mj_table} mj ON pr.job_id = mj.id WHERE mj.uuid = $1 AND pr.processor = 'asrx' AND pr.status = 'completed'")
@@ -1007,7 +1150,10 @@ async fn get_pipeline_progress_handler(
}
if sentence_count > 0 {
let detail = if sentence_embedded > 0 {
Some(format!("{} chunks, {} embedded", sentence_count, sentence_embedded))
Some(format!(
"{} chunks, {} embedded",
sentence_count, sentence_embedded
))
} else {
Some(format!("{} chunks", sentence_count))
};
@@ -1017,13 +1163,28 @@ async fn get_pipeline_progress_handler(
pp.update_stage("face_tracing", 1.0, "completed", None);
}
if tkg_node_count > 0 {
pp.update_stage("tkg_nodes", 1.0, "completed", Some(format!("{} nodes", tkg_node_count)));
pp.update_stage(
"tkg_nodes",
1.0,
"completed",
Some(format!("{} nodes", tkg_node_count)),
);
}
if tkg_edge_count > 0 {
pp.update_stage("tkg_edges", 1.0, "completed", Some(format!("{} edges", tkg_edge_count)));
pp.update_stage(
"tkg_edges",
1.0,
"completed",
Some(format!("{} edges", tkg_edge_count)),
);
}
if relationship_count > 0 {
pp.update_stage("rule2_ingestion", 1.0, "completed", Some(format!("{} chunks", relationship_count)));
pp.update_stage(
"rule2_ingestion",
1.0,
"completed",
Some(format!("{} chunks", relationship_count)),
);
}
// Check identity agent from _seeds
@@ -1031,11 +1192,23 @@ async fn get_pipeline_progress_handler(
use serde_json::json;
let qdrant = QdrantDb::new();
let schema = std::env::var("DATABASE_SCHEMA").unwrap_or_else(|_| "dev".to_string());
let seeds_collection = if schema == "public" { "momentry_public_speaker" } else { &format!("momentry_{}_speaker", schema) };
let seeds_collection = if schema == "public" {
"momentry_public_speaker"
} else {
&format!("momentry_{}_speaker", schema)
};
let seeds_filter = json!({"must": [{"key": "file_uuid", "match": {"value": &file_uuid}}]});
let seed_points = qdrant.scroll_all_points("_seeds", seeds_filter, 100).await.unwrap_or_default();
let seed_points = qdrant
.scroll_all_points("_seeds", seeds_filter, 100)
.await
.unwrap_or_default();
if !seed_points.is_empty() {
pp.update_stage("identity_agent", 1.0, "completed", Some(format!("{} seeds", seed_points.len())));
pp.update_stage(
"identity_agent",
1.0,
"completed",
Some(format!("{} seeds", seed_points.len())),
);
}
Ok(Json(pp))