feat: add cluster-agent endpoint, fix OCR labeling, fix ingestion blocking

- Add POST /api/v1/file/:file_uuid/cluster-agent endpoint for on-demand face clustering
- Fix OCR chunks being labeled as ASRX: use ChunkType.as_str() instead of {:?}
- Rule 1 now deletes old chunks before re-inserting to avoid stale data
- Add fallback face_traced.json when store_traced_faces.py fails
- ingestion_complete now handles status='error' to unblock jobs
- VLM describe tool now uses trace_id with pre-extracted face crops
This commit is contained in:
Accusys
2026-07-20 21:48:27 +08:00
parent 244af51edf
commit 7dc910e2de
8 changed files with 654 additions and 57 deletions
+14 -1
View File
@@ -96,7 +96,9 @@ const SYSTEM_PROMPT: &str = r#"你是 Momentry 影片分析助手。回答用戶
7. 人物台詞內容使用 tkg_query 的 speaker_dialogue
8. 用文字反查人物使用 identity_text(輸入關鍵字→找出誰說/提到這段話)
9. 畫面分析使用 analyze_frame — 可以分析影片中的任何畫面內容(場景、人物表情、動作、物件等)
10. **可以同時呼叫多個工具,但需符合以下條件:**
10. **人物外貌/衣著/顏色問題使用 vlm_describe** — 用 trace_id 查人臉裁切圖,經 VLM 分析描述衣著顏色、款式、配件等
11. **vlm_describe 使用流程**:先用 tkg_query top_identities/identity_traces 找到該人的 trace_id,再呼叫 vlm_describe(file_uuid + trace_id)
12. **可以同時呼叫多個工具,但需符合以下條件:**
- ✅ 查詢多部影片的相同資訊(如:3部影片的人物列表)
- ✅ 需要組合多個來源的資訊才能回答(如:file_info + tkg_query)
- ❌ 不要為了「嘗試所有可能」而盲目並行呼叫
@@ -261,6 +263,16 @@ fn make_tools(pool: &sqlx::PgPool) -> Vec<ToolDef> {
}),
vec!["file_uuid", "color"],
),
function_calling::make_tool(
"vlm_describe",
"Describe a person's appearance (clothing, colors, accessories) by face trace ID using Vision Language Model (LLaVA). Uses pre-extracted face crop images.",
serde_json::json!({
"file_uuid": {"type": "string", "description": "UUID of the video file"},
"trace_id": {"type": "integer", "description": "Face trace ID to analyze"},
"prompt": {"type": "string", "description": "Specific question about the person's appearance (optional, default: describes clothing colors and style)"}
}),
vec!["file_uuid", "trace_id"],
),
]
}
@@ -296,6 +308,7 @@ async fn execute_tool(pool: &sqlx::PgPool, tool_call: &ToolCall) -> (String, Str
"get_representative_frame" => tools::exec_get_representative_frame(pool, &args).await,
"analyze_frame" => tools::exec_analyze_frame(pool, &args).await,
"search_by_appearance" => tools::exec_search_by_appearance(pool, &args).await,
"vlm_describe" => tools::exec_vlm_describe(pool, &args).await,
_ => Err(format!("Unknown tool: {}", name)),
};
let content = match result {
+238
View File
@@ -0,0 +1,238 @@
use axum::{
Extension, Json,
extract::{Path, State},
http::StatusCode,
routing::post,
Router,
};
use serde::Serialize;
use std::collections::HashMap;
use std::path::PathBuf;
use super::middleware::UserAuth;
use super::types::AppState;
use crate::core::config::OUTPUT_DIR;
use crate::core::db::schema;
use crate::core::processor::{process_face_cluster, FaceClusterResult};
#[derive(Debug, Serialize)]
pub struct ClusterAgentResponse {
success: bool,
file_uuid: String,
message: String,
clusters: usize,
total_traces: usize,
}
pub fn cluster_agent_routes() -> Router<AppState> {
Router::new().route(
"/api/v1/file/:file_uuid/cluster-agent",
post(trigger_cluster_agent),
)
}
pub async fn trigger_cluster_agent(
State(state): State<AppState>,
Extension(_auth): Extension<UserAuth>,
Path(file_uuid): Path<String>,
) -> Result<Json<ClusterAgentResponse>, (StatusCode, Json<serde_json::Value>)> {
let videos_table = schema::table_name("videos");
let row = sqlx::query(&format!(
"SELECT file_path, COALESCE(total_frames, 0) as total_frames, COALESCE(fps, 25.0) as fps, probe_json FROM {} WHERE file_uuid = $1",
videos_table
))
.bind(&file_uuid)
.fetch_optional(state.db.pool())
.await
.map_err(|e| {
tracing::error!("[ClusterAgent] DB error: {}", e);
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})?;
let (file_path, total_frames, fps) = match row {
Some(r) => {
use sqlx::Row;
let p: String = r.get("file_path");
let tf: i64 = r.get("total_frames");
let f: f64 = r.get("fps");
let mut frames = tf;
if frames <= 0 {
if let Ok(meta_val) = r.try_get::<serde_json::Value, _>("probe_json") {
if let Some(streams) = meta_val.get("streams").and_then(|v| v.as_array()) {
for s in streams {
if s.get("codec_type").and_then(|v| v.as_str()) == Some("video") {
if let Some(nb) = s.get("nb_frames").and_then(|v| v.as_str()) {
if let Ok(n) = nb.parse::<i64>() {
frames = n;
break;
}
}
}
}
}
}
}
(p, frames, f)
}
None => {
return Err((
StatusCode::NOT_FOUND,
Json(serde_json::json!({"error": "File not found"})),
))
}
};
let output_path =
PathBuf::from(OUTPUT_DIR.as_str()).join(format!("{}.face_cluster.json", file_uuid));
tracing::info!("[ClusterAgent] Starting face clustering for {}", file_uuid);
let result = process_face_cluster(
&file_path,
output_path.to_str().ok_or_else(|| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": "Invalid output path"})),
)
})?,
Some(&file_uuid),
None,
)
.await
.map_err(|e| {
tracing::error!("[ClusterAgent] Clustering failed: {}", e);
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": format!("Clustering failed: {}", e)})),
)
})?;
update_trace_labels_from_cluster(&state, &file_uuid, &result)
.await
.map_err(|e| {
tracing::error!("[ClusterAgent] Failed to update trace labels: {}", e);
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})?;
let total_traces = count_total_traces(&state, &file_uuid)
.await
.map_err(|e| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({"error": e.to_string()})),
)
})?;
tracing::info!(
"[ClusterAgent] Completed for {}: {} clusters, {} traces",
file_uuid,
result.clusters.len(),
total_traces
);
Ok(Json(ClusterAgentResponse {
success: true,
file_uuid,
message: "Clustering completed".to_string(),
clusters: result.clusters.len(),
total_traces,
}))
}
async fn update_trace_labels_from_cluster(
state: &AppState,
file_uuid: &str,
result: &FaceClusterResult,
) -> anyhow::Result<()> {
let scroll_filter = serde_json::json!({
"must": [
{"key": "file_uuid", "match": {"value": file_uuid}}
]
});
let all_faces = state
.qdrant
.scroll_all_points("_faces", scroll_filter, 1000)
.await?;
let mut frame_to_traces: HashMap<u64, Vec<i64>> = HashMap::new();
for point in &all_faces {
let payload = &point["payload"];
if let (Some(frame), Some(trace_id)) =
(payload["frame"].as_u64(), payload["trace_id"].as_i64())
{
if trace_id > 0 {
frame_to_traces.entry(frame).or_default().push(trace_id);
}
}
}
let mut trace_to_cluster: HashMap<i64, String> = HashMap::new();
for frame_data in &result.frames {
let frame = frame_data.frame;
if let Some(trace_ids) = frame_to_traces.get(&frame) {
for (idx, face) in frame_data.faces.iter().enumerate() {
if idx < trace_ids.len() {
trace_to_cluster.insert(trace_ids[idx], face.cluster_id.clone());
}
}
}
}
let tkg_table = schema::table_name("tkg_nodes");
for (trace_id, cluster_id) in &trace_to_cluster {
let external_id = format!("trace_{}", trace_id);
let label = cluster_id.clone();
let rows_updated = sqlx::query(&format!(
"UPDATE {} SET label = $1 WHERE file_uuid = $2 AND node_type = 'face_track' AND external_id = $3",
tkg_table
))
.bind(&label)
.bind(file_uuid)
.bind(&external_id)
.execute(state.db.pool())
.await?
.rows_affected();
if rows_updated == 0 {
sqlx::query(&format!(
"INSERT INTO {} (file_uuid, node_type, external_id, label, properties) VALUES ($1, 'face_track', $2, $3, $4)",
tkg_table
))
.bind(file_uuid)
.bind(&external_id)
.bind(&label)
.bind(serde_json::json!({"trace_id": trace_id}))
.execute(state.db.pool())
.await?;
}
}
tracing::info!(
"[ClusterAgent] Updated {} trace labels for {}",
trace_to_cluster.len(),
file_uuid
);
Ok(())
}
async fn count_total_traces(state: &AppState, file_uuid: &str) -> anyhow::Result<usize> {
let tkg_table = schema::table_name("tkg_nodes");
let count: (i64,) = sqlx::query_as(&format!(
"SELECT COUNT(*) FROM {} WHERE file_uuid = $1 AND node_type = 'face_track'",
tkg_table
))
.bind(file_uuid)
.fetch_one(state.db.pool())
.await?;
Ok(count.0 as usize)
}
+1
View File
@@ -2,6 +2,7 @@ pub mod agent_api;
pub mod agent_search;
pub mod auth;
pub mod checkin_api;
pub mod cluster_agent;
pub mod docs;
pub mod files;
pub mod health;
+2
View File
@@ -13,6 +13,7 @@ use super::agent_api;
use super::agent_search;
use super::auth;
use super::checkin_api;
use super::cluster_agent;
use super::docs;
use super::files;
use super::health;
@@ -127,6 +128,7 @@ pub async fn start_server(host: &str, port: u16) -> anyhow::Result<()> {
.merge(universal_search_routes())
.merge(pipeline::pipeline_routes())
.merge(checkin_api::checkin_routes())
.merge(cluster_agent::cluster_agent_routes())
.merge(profile::profile_routes())
.layer(axum::middleware::from_fn_with_state(
state.api_state.clone(),