feat: add cluster-agent endpoint, fix OCR labeling, fix ingestion blocking
- Add POST /api/v1/file/:file_uuid/cluster-agent endpoint for on-demand face clustering
- Fix OCR chunks being labeled as ASRX: use ChunkType.as_str() instead of {:?}
- Rule 1 now deletes old chunks before re-inserting to avoid stale data
- Add fallback face_traced.json when store_traced_faces.py fails
- ingestion_complete now handles status='error' to unblock jobs
- VLM describe tool now uses trace_id with pre-extracted face crops
This commit is contained in:
+14
-1
@@ -96,7 +96,9 @@ const SYSTEM_PROMPT: &str = r#"你是 Momentry 影片分析助手。回答用戶
|
||||
7. 人物台詞內容使用 tkg_query 的 speaker_dialogue
|
||||
8. 用文字反查人物使用 identity_text(輸入關鍵字→找出誰說/提到這段話)
|
||||
9. 畫面分析使用 analyze_frame — 可以分析影片中的任何畫面內容(場景、人物表情、動作、物件等)
|
||||
10. **可以同時呼叫多個工具,但需符合以下條件:**
|
||||
10. **人物外貌/衣著/顏色問題使用 vlm_describe** — 用 trace_id 查人臉裁切圖,經 VLM 分析描述衣著顏色、款式、配件等
|
||||
11. **vlm_describe 使用流程**:先用 tkg_query top_identities/identity_traces 找到該人的 trace_id,再呼叫 vlm_describe(file_uuid + trace_id)
|
||||
12. **可以同時呼叫多個工具,但需符合以下條件:**
|
||||
- ✅ 查詢多部影片的相同資訊(如:3部影片的人物列表)
|
||||
- ✅ 需要組合多個來源的資訊才能回答(如:file_info + tkg_query)
|
||||
- ❌ 不要為了「嘗試所有可能」而盲目並行呼叫
|
||||
@@ -261,6 +263,16 @@ fn make_tools(pool: &sqlx::PgPool) -> Vec<ToolDef> {
|
||||
}),
|
||||
vec!["file_uuid", "color"],
|
||||
),
|
||||
function_calling::make_tool(
|
||||
"vlm_describe",
|
||||
"Describe a person's appearance (clothing, colors, accessories) by face trace ID using Vision Language Model (LLaVA). Uses pre-extracted face crop images.",
|
||||
serde_json::json!({
|
||||
"file_uuid": {"type": "string", "description": "UUID of the video file"},
|
||||
"trace_id": {"type": "integer", "description": "Face trace ID to analyze"},
|
||||
"prompt": {"type": "string", "description": "Specific question about the person's appearance (optional, default: describes clothing colors and style)"}
|
||||
}),
|
||||
vec!["file_uuid", "trace_id"],
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
@@ -296,6 +308,7 @@ async fn execute_tool(pool: &sqlx::PgPool, tool_call: &ToolCall) -> (String, Str
|
||||
"get_representative_frame" => tools::exec_get_representative_frame(pool, &args).await,
|
||||
"analyze_frame" => tools::exec_analyze_frame(pool, &args).await,
|
||||
"search_by_appearance" => tools::exec_search_by_appearance(pool, &args).await,
|
||||
"vlm_describe" => tools::exec_vlm_describe(pool, &args).await,
|
||||
_ => Err(format!("Unknown tool: {}", name)),
|
||||
};
|
||||
let content = match result {
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
use axum::{
|
||||
Extension, Json,
|
||||
extract::{Path, State},
|
||||
http::StatusCode,
|
||||
routing::post,
|
||||
Router,
|
||||
};
|
||||
use serde::Serialize;
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use super::middleware::UserAuth;
|
||||
use super::types::AppState;
|
||||
use crate::core::config::OUTPUT_DIR;
|
||||
use crate::core::db::schema;
|
||||
use crate::core::processor::{process_face_cluster, FaceClusterResult};
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct ClusterAgentResponse {
|
||||
success: bool,
|
||||
file_uuid: String,
|
||||
message: String,
|
||||
clusters: usize,
|
||||
total_traces: usize,
|
||||
}
|
||||
|
||||
pub fn cluster_agent_routes() -> Router<AppState> {
|
||||
Router::new().route(
|
||||
"/api/v1/file/:file_uuid/cluster-agent",
|
||||
post(trigger_cluster_agent),
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn trigger_cluster_agent(
|
||||
State(state): State<AppState>,
|
||||
Extension(_auth): Extension<UserAuth>,
|
||||
Path(file_uuid): Path<String>,
|
||||
) -> Result<Json<ClusterAgentResponse>, (StatusCode, Json<serde_json::Value>)> {
|
||||
let videos_table = schema::table_name("videos");
|
||||
let row = sqlx::query(&format!(
|
||||
"SELECT file_path, COALESCE(total_frames, 0) as total_frames, COALESCE(fps, 25.0) as fps, probe_json FROM {} WHERE file_uuid = $1",
|
||||
videos_table
|
||||
))
|
||||
.bind(&file_uuid)
|
||||
.fetch_optional(state.db.pool())
|
||||
.await
|
||||
.map_err(|e| {
|
||||
tracing::error!("[ClusterAgent] DB error: {}", e);
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Json(serde_json::json!({"error": e.to_string()})),
|
||||
)
|
||||
})?;
|
||||
|
||||
let (file_path, total_frames, fps) = match row {
|
||||
Some(r) => {
|
||||
use sqlx::Row;
|
||||
let p: String = r.get("file_path");
|
||||
let tf: i64 = r.get("total_frames");
|
||||
let f: f64 = r.get("fps");
|
||||
|
||||
let mut frames = tf;
|
||||
if frames <= 0 {
|
||||
if let Ok(meta_val) = r.try_get::<serde_json::Value, _>("probe_json") {
|
||||
if let Some(streams) = meta_val.get("streams").and_then(|v| v.as_array()) {
|
||||
for s in streams {
|
||||
if s.get("codec_type").and_then(|v| v.as_str()) == Some("video") {
|
||||
if let Some(nb) = s.get("nb_frames").and_then(|v| v.as_str()) {
|
||||
if let Ok(n) = nb.parse::<i64>() {
|
||||
frames = n;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
(p, frames, f)
|
||||
}
|
||||
None => {
|
||||
return Err((
|
||||
StatusCode::NOT_FOUND,
|
||||
Json(serde_json::json!({"error": "File not found"})),
|
||||
))
|
||||
}
|
||||
};
|
||||
|
||||
let output_path =
|
||||
PathBuf::from(OUTPUT_DIR.as_str()).join(format!("{}.face_cluster.json", file_uuid));
|
||||
|
||||
tracing::info!("[ClusterAgent] Starting face clustering for {}", file_uuid);
|
||||
let result = process_face_cluster(
|
||||
&file_path,
|
||||
output_path.to_str().ok_or_else(|| {
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Json(serde_json::json!({"error": "Invalid output path"})),
|
||||
)
|
||||
})?,
|
||||
Some(&file_uuid),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
tracing::error!("[ClusterAgent] Clustering failed: {}", e);
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Json(serde_json::json!({"error": format!("Clustering failed: {}", e)})),
|
||||
)
|
||||
})?;
|
||||
|
||||
update_trace_labels_from_cluster(&state, &file_uuid, &result)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
tracing::error!("[ClusterAgent] Failed to update trace labels: {}", e);
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Json(serde_json::json!({"error": e.to_string()})),
|
||||
)
|
||||
})?;
|
||||
|
||||
let total_traces = count_total_traces(&state, &file_uuid)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Json(serde_json::json!({"error": e.to_string()})),
|
||||
)
|
||||
})?;
|
||||
|
||||
tracing::info!(
|
||||
"[ClusterAgent] Completed for {}: {} clusters, {} traces",
|
||||
file_uuid,
|
||||
result.clusters.len(),
|
||||
total_traces
|
||||
);
|
||||
|
||||
Ok(Json(ClusterAgentResponse {
|
||||
success: true,
|
||||
file_uuid,
|
||||
message: "Clustering completed".to_string(),
|
||||
clusters: result.clusters.len(),
|
||||
total_traces,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn update_trace_labels_from_cluster(
|
||||
state: &AppState,
|
||||
file_uuid: &str,
|
||||
result: &FaceClusterResult,
|
||||
) -> anyhow::Result<()> {
|
||||
let scroll_filter = serde_json::json!({
|
||||
"must": [
|
||||
{"key": "file_uuid", "match": {"value": file_uuid}}
|
||||
]
|
||||
});
|
||||
|
||||
let all_faces = state
|
||||
.qdrant
|
||||
.scroll_all_points("_faces", scroll_filter, 1000)
|
||||
.await?;
|
||||
|
||||
let mut frame_to_traces: HashMap<u64, Vec<i64>> = HashMap::new();
|
||||
for point in &all_faces {
|
||||
let payload = &point["payload"];
|
||||
if let (Some(frame), Some(trace_id)) =
|
||||
(payload["frame"].as_u64(), payload["trace_id"].as_i64())
|
||||
{
|
||||
if trace_id > 0 {
|
||||
frame_to_traces.entry(frame).or_default().push(trace_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut trace_to_cluster: HashMap<i64, String> = HashMap::new();
|
||||
for frame_data in &result.frames {
|
||||
let frame = frame_data.frame;
|
||||
if let Some(trace_ids) = frame_to_traces.get(&frame) {
|
||||
for (idx, face) in frame_data.faces.iter().enumerate() {
|
||||
if idx < trace_ids.len() {
|
||||
trace_to_cluster.insert(trace_ids[idx], face.cluster_id.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let tkg_table = schema::table_name("tkg_nodes");
|
||||
for (trace_id, cluster_id) in &trace_to_cluster {
|
||||
let external_id = format!("trace_{}", trace_id);
|
||||
let label = cluster_id.clone();
|
||||
|
||||
let rows_updated = sqlx::query(&format!(
|
||||
"UPDATE {} SET label = $1 WHERE file_uuid = $2 AND node_type = 'face_track' AND external_id = $3",
|
||||
tkg_table
|
||||
))
|
||||
.bind(&label)
|
||||
.bind(file_uuid)
|
||||
.bind(&external_id)
|
||||
.execute(state.db.pool())
|
||||
.await?
|
||||
.rows_affected();
|
||||
|
||||
if rows_updated == 0 {
|
||||
sqlx::query(&format!(
|
||||
"INSERT INTO {} (file_uuid, node_type, external_id, label, properties) VALUES ($1, 'face_track', $2, $3, $4)",
|
||||
tkg_table
|
||||
))
|
||||
.bind(file_uuid)
|
||||
.bind(&external_id)
|
||||
.bind(&label)
|
||||
.bind(serde_json::json!({"trace_id": trace_id}))
|
||||
.execute(state.db.pool())
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
"[ClusterAgent] Updated {} trace labels for {}",
|
||||
trace_to_cluster.len(),
|
||||
file_uuid
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn count_total_traces(state: &AppState, file_uuid: &str) -> anyhow::Result<usize> {
|
||||
let tkg_table = schema::table_name("tkg_nodes");
|
||||
let count: (i64,) = sqlx::query_as(&format!(
|
||||
"SELECT COUNT(*) FROM {} WHERE file_uuid = $1 AND node_type = 'face_track'",
|
||||
tkg_table
|
||||
))
|
||||
.bind(file_uuid)
|
||||
.fetch_one(state.db.pool())
|
||||
.await?;
|
||||
|
||||
Ok(count.0 as usize)
|
||||
}
|
||||
@@ -2,6 +2,7 @@ pub mod agent_api;
|
||||
pub mod agent_search;
|
||||
pub mod auth;
|
||||
pub mod checkin_api;
|
||||
pub mod cluster_agent;
|
||||
pub mod docs;
|
||||
pub mod files;
|
||||
pub mod health;
|
||||
|
||||
@@ -13,6 +13,7 @@ use super::agent_api;
|
||||
use super::agent_search;
|
||||
use super::auth;
|
||||
use super::checkin_api;
|
||||
use super::cluster_agent;
|
||||
use super::docs;
|
||||
use super::files;
|
||||
use super::health;
|
||||
@@ -127,6 +128,7 @@ pub async fn start_server(host: &str, port: u16) -> anyhow::Result<()> {
|
||||
.merge(universal_search_routes())
|
||||
.merge(pipeline::pipeline_routes())
|
||||
.merge(checkin_api::checkin_routes())
|
||||
.merge(cluster_agent::cluster_agent_routes())
|
||||
.merge(profile::profile_routes())
|
||||
.layer(axum::middleware::from_fn_with_state(
|
||||
state.api_state.clone(),
|
||||
|
||||
Reference in New Issue
Block a user