feat: Phase 2.6 edges migration to Qdrant (TKG-only architecture)
Phase 2.6.1: co_occurrence_edges migration - build_co_occurrence_edges_from_qdrant() - Qdrant embeddings → frame grouping → YOLO objects - Result: 6679 edges (vs 6701 PostgreSQL) Phase 2.6.2: face_face_edges migration - build_face_face_edges_from_qdrant() - Qdrant embeddings → frame grouping → face pairs - mutual_gaze detection preserved - Result: 6 edges (exact match) Phase 2.6.3: speaker_face_edges migration - build_speaker_face_edges_from_qdrant() - Qdrant embeddings → trace_id frame ranges - SPEAKS_AS edge creation Architecture: - All edges use Qdrant payload (no face_detections queries) - PostgreSQL fallback for empty Qdrant - Estimated 3.6x performance improvement Testing: - Playground (3003): ✓ All Phase 2.6 logs verified - Edge counts: ✓ Close match with PostgreSQL - Fallback: ✓ Working Docs: - docs_v1.0/DESIGN/TKG_PHASE2_6_EDGES_MIGRATION.md - docs_v1.0/M4_workspace/2026-06-21_phase2_6_test.md
This commit is contained in:
Accusys
@@ -0,0 +1,861 @@
|
||||
use base64::{engine::general_purpose::STANDARD as BASE64, Engine};
|
||||
use serde_json;
|
||||
|
||||
use crate::core::db::schema;
|
||||
use crate::core::llm::function_calling::call_llm_vision;
|
||||
use crate::core::processor::tkg::query_auto_representative_frame;
|
||||
|
||||
fn t(name: &str) -> String {
|
||||
let schema = std::env::var("DATABASE_SCHEMA").unwrap_or_else(|_| "dev".to_string());
|
||||
if schema == "public" {
|
||||
name.to_string()
|
||||
} else {
|
||||
format!("{}.{}", schema, name)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn exec_find_file(pool: &sqlx::PgPool, args: &serde_json::Value) -> Result<String, String> {
|
||||
let query = args.get("query").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let videos = schema::table_name("videos");
|
||||
let fd_table = schema::table_name("face_detections");
|
||||
let like = format!("%{}%", query);
|
||||
let rows: Vec<(String, String, bool)> = sqlx::query_as(&format!(
|
||||
"SELECT v.file_uuid::text, v.file_name, \
|
||||
(SELECT COUNT(*) FROM {} fd WHERE fd.file_uuid = v.file_uuid) > 0 AS has_data \
|
||||
FROM {} v WHERE v.file_name ILIKE $1 \
|
||||
ORDER BY v.created_at DESC LIMIT 10",
|
||||
fd_table, videos
|
||||
))
|
||||
.bind(&like)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
if rows.is_empty() {
|
||||
return Ok(serde_json::json!({"found": false, "message": "No files match the query. Try different keywords."}).to_string());
|
||||
}
|
||||
let files: Vec<serde_json::Value> = rows
|
||||
.into_iter()
|
||||
.map(|(u, n, hd)| serde_json::json!({"file_uuid": u, "file_name": n, "has_data": hd}))
|
||||
.collect();
|
||||
Ok(serde_json::json!({"found": true, "files": files}).to_string())
|
||||
}
|
||||
|
||||
pub async fn exec_list_files(pool: &sqlx::PgPool, args: &serde_json::Value) -> Result<String, String> {
|
||||
let limit = args.get("limit").and_then(|v| v.as_i64()).unwrap_or(10);
|
||||
let videos = schema::table_name("videos");
|
||||
let fd_table = schema::table_name("face_detections");
|
||||
let rows: Vec<(String, String, bool)> = sqlx::query_as(&format!(
|
||||
"SELECT v.file_uuid::text, v.file_name, \
|
||||
(SELECT COUNT(*) FROM {} fd WHERE fd.file_uuid = v.file_uuid) > 0 AS has_data \
|
||||
FROM {} v ORDER BY v.created_at DESC LIMIT $1",
|
||||
fd_table, videos
|
||||
))
|
||||
.bind(limit)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let files: Vec<serde_json::Value> = rows
|
||||
.into_iter()
|
||||
.map(|(u, n, hd)| serde_json::json!({"file_uuid": u, "file_name": n, "has_data": hd}))
|
||||
.collect();
|
||||
Ok(serde_json::json!({"files": files}).to_string())
|
||||
}
|
||||
|
||||
pub async fn exec_tkg_query(pool: &sqlx::PgPool, args: &serde_json::Value) -> Result<String, String> {
|
||||
let file_uuid = args.get("file_uuid").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let query_type = args
|
||||
.get("query_type")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("");
|
||||
let identity_name = args.get("identity_name").and_then(|v| v.as_str());
|
||||
let identity_b = args.get("identity_b").and_then(|v| v.as_str());
|
||||
let limit = args.get("limit").and_then(|v| v.as_i64()).unwrap_or(5);
|
||||
|
||||
let id_table = schema::table_name("identities");
|
||||
let fd_table = schema::table_name("face_detections");
|
||||
let videos = schema::table_name("videos");
|
||||
let nodes = schema::table_name("tkg_nodes");
|
||||
let edges = schema::table_name("tkg_edges");
|
||||
|
||||
match query_type {
|
||||
"top_identities" => {
|
||||
let rows: Vec<(String, String, i64)> = sqlx::query_as(&format!(
|
||||
"SELECT i.uuid::text, i.name, COUNT(fd.id)::bigint AS face_count \
|
||||
FROM {} fd JOIN {} i ON i.id = fd.identity_id \
|
||||
WHERE fd.file_uuid = $1 AND fd.identity_id IS NOT NULL AND i.source = 'tmdb' \
|
||||
GROUP BY i.uuid, i.name ORDER BY face_count DESC LIMIT $2",
|
||||
fd_table, id_table
|
||||
))
|
||||
.bind(file_uuid)
|
||||
.bind(limit)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(serde_json::json!({"identities": rows}).to_string())
|
||||
}
|
||||
"first_cooccurrence" => {
|
||||
let name_a = identity_name.unwrap_or("");
|
||||
let name_b = identity_b.unwrap_or("");
|
||||
let row: Option<(i64, f64)> = sqlx::query_as(&format!(
|
||||
"SELECT MIN(fd_a.frame_number)::bigint, \
|
||||
ROUND(MIN(fd_a.frame_number)::numeric / GREATEST(MAX(v.fps)::numeric, 25.0), 2)::float8 \
|
||||
FROM {} fd_a JOIN {} fd_b ON fd_a.frame_number = fd_b.frame_number \
|
||||
JOIN {} v ON v.file_uuid = $1 \
|
||||
WHERE fd_a.file_uuid = $1 \
|
||||
AND fd_a.identity_id = (SELECT id FROM {} WHERE name ILIKE $2 LIMIT 1) \
|
||||
AND fd_b.identity_id = (SELECT id FROM {} WHERE name ILIKE $3 LIMIT 1)",
|
||||
fd_table, fd_table, videos, id_table, id_table
|
||||
))
|
||||
.bind(file_uuid).bind(name_a).bind(name_b)
|
||||
.fetch_optional(pool)
|
||||
.await.map_err(|e| e.to_string())?;
|
||||
Ok(serde_json::json!({"first_cooccurrence": row.map(|(f, t)| serde_json::json!({"frame": f, "timestamp_secs": t}))}).to_string())
|
||||
}
|
||||
"identity_details" => {
|
||||
let name = identity_name.unwrap_or("");
|
||||
let row: Option<(String, String, Option<i32>, i64)> = sqlx::query_as(&format!(
|
||||
"SELECT i.uuid::text, i.name, i.tmdb_id, \
|
||||
(SELECT COUNT(*) FROM {} fd WHERE fd.identity_id = i.id AND fd.file_uuid = $1)::bigint \
|
||||
FROM {} i WHERE i.name ILIKE $2 LIMIT 1",
|
||||
fd_table, id_table
|
||||
))
|
||||
.bind(file_uuid).bind(name)
|
||||
.fetch_optional(pool)
|
||||
.await.map_err(|e| e.to_string())?;
|
||||
Ok(serde_json::json!({"identity": row.map(|(u, n, tid, fc)| serde_json::json!({"uuid": u, "name": n, "tmdb_id": tid, "face_count": fc}))}).to_string())
|
||||
}
|
||||
"mutual_gaze" => {
|
||||
let name_a = identity_name.unwrap_or("");
|
||||
let name_b = identity_b.unwrap_or("");
|
||||
let row: Option<(i64, i64, f64, f64)> = sqlx::query_as(&format!(
|
||||
"SELECT (e.properties->>'first_frame')::bigint, \
|
||||
(e.properties->>'gaze_frame_count')::int::bigint, \
|
||||
(e.properties->>'yaw_a_avg')::float8, \
|
||||
(e.properties->>'yaw_b_avg')::float8 \
|
||||
FROM {} e \
|
||||
JOIN {} a ON a.id = e.source_node_id \
|
||||
JOIN {} b ON b.id = e.target_node_id \
|
||||
JOIN {} fd_a ON fd_a.file_uuid = $1 AND fd_a.trace_id = REPLACE(a.external_id, 'trace_', '')::int \
|
||||
JOIN {} fd_b ON fd_b.file_uuid = $1 AND fd_b.trace_id = REPLACE(b.external_id, 'trace_', '')::int \
|
||||
JOIN {} ia ON ia.id = fd_a.identity_id \
|
||||
JOIN {} ib ON ib.id = fd_b.identity_id \
|
||||
WHERE e.file_uuid = $1 AND ia.name ILIKE $2 AND ib.name ILIKE $3 \
|
||||
AND e.properties->>'mutual_gaze' = 'true' LIMIT 1",
|
||||
edges, nodes, nodes, fd_table, fd_table, id_table, id_table
|
||||
))
|
||||
.bind(file_uuid).bind(name_a).bind(name_b)
|
||||
.fetch_optional(pool)
|
||||
.await.map_err(|e| e.to_string())?;
|
||||
Ok(serde_json::json!({"mutual_gaze": row.map(|(f, gc, ya, yb)| serde_json::json!({"first_frame": f, "gaze_frame_count": gc, "yaw_a": ya, "yaw_b": yb}))}).to_string())
|
||||
}
|
||||
"interaction_network" => {
|
||||
let rows: Vec<(String, String, i64)> = sqlx::query_as(&format!(
|
||||
"SELECT ia.name, ib.name, COUNT(*)::bigint \
|
||||
FROM {} e \
|
||||
JOIN {} a ON a.id = e.source_node_id \
|
||||
JOIN {} b ON b.id = e.target_node_id \
|
||||
JOIN {} fd_a ON fd_a.trace_id = REPLACE(a.external_id, 'trace_', '')::int AND fd_a.file_uuid = $1 \
|
||||
JOIN {} fd_b ON fd_b.trace_id = REPLACE(b.external_id, 'trace_', '')::int AND fd_b.file_uuid = $1 \
|
||||
JOIN {} ia ON ia.id = fd_a.identity_id \
|
||||
JOIN {} ib ON ib.id = fd_b.identity_id \
|
||||
WHERE e.file_uuid = $1 AND e.edge_type = 'CO_OCCURS_WITH' \
|
||||
AND ia.name != ib.name AND ia.source = 'tmdb' AND ib.source = 'tmdb' \
|
||||
GROUP BY ia.name, ib.name \
|
||||
ORDER BY COUNT(*) DESC LIMIT $2",
|
||||
edges, nodes, nodes, fd_table, fd_table, id_table, id_table
|
||||
))
|
||||
.bind(file_uuid).bind(limit)
|
||||
.fetch_all(pool)
|
||||
.await.map_err(|e| e.to_string())?;
|
||||
Ok(serde_json::json!({"interaction_network": rows}).to_string())
|
||||
}
|
||||
"identity_traces" => {
|
||||
let name = identity_name.unwrap_or("");
|
||||
let rows: Vec<(i32, i64, i64, i64)> = sqlx::query_as(&format!(
|
||||
"SELECT fd.trace_id, COUNT(*)::bigint, MIN(fd.frame_number)::bigint, MAX(fd.frame_number)::bigint \
|
||||
FROM {} fd JOIN {} i ON i.id = fd.identity_id \
|
||||
WHERE fd.file_uuid = $1 AND i.name ILIKE $2 \
|
||||
GROUP BY fd.trace_id ORDER BY COUNT(*) DESC LIMIT $3",
|
||||
fd_table, id_table
|
||||
))
|
||||
.bind(file_uuid).bind(name).bind(limit)
|
||||
.fetch_all(pool)
|
||||
.await.map_err(|e| e.to_string())?;
|
||||
Ok(serde_json::json!({"traces": rows}).to_string())
|
||||
}
|
||||
"file_info" => {
|
||||
let row: Option<(String, f64, i32, i32, f64)> = sqlx::query_as(&format!(
|
||||
"SELECT file_name, duration, width, height, fps FROM {} WHERE file_uuid = $1",
|
||||
videos
|
||||
))
|
||||
.bind(file_uuid)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(serde_json::json!({"file_info": row.map(|(n, d, w, h, f)| serde_json::json!({"file_name": n, "duration_sec": d, "width": w, "height": h, "fps": f}))}).to_string())
|
||||
}
|
||||
"speaker_dialogue" => {
|
||||
let name = identity_name.unwrap_or("");
|
||||
let rows: Vec<(String, Option<String>)> = sqlx::query_as(&format!(
|
||||
"SELECT DISTINCT sn.external_id, sn.properties->>'full_text' AS full_text \
|
||||
FROM {} i \
|
||||
JOIN {} fd ON fd.identity_id = i.id AND ($2::text IS NULL OR fd.file_uuid = $2) \
|
||||
JOIN {} fn ON fn.file_uuid = fd.file_uuid \
|
||||
AND fn.node_type = 'face_trace' \
|
||||
AND fn.external_id = CONCAT('trace_', fd.trace_id) \
|
||||
JOIN {} e ON e.source_node_id = fn.id \
|
||||
AND e.edge_type = 'SPEAKS_AS' \
|
||||
AND ($2::text IS NULL OR e.file_uuid = $2) \
|
||||
JOIN {} sn ON sn.id = e.target_node_id \
|
||||
WHERE i.name ILIKE $1 \
|
||||
LIMIT $3",
|
||||
id_table, fd_table, nodes, edges, nodes
|
||||
))
|
||||
.bind(name)
|
||||
.bind(file_uuid)
|
||||
.bind(limit)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(
|
||||
serde_json::json!({"speakers": rows.iter().map(|(sid, text)| {
|
||||
serde_json::json!({"speaker_id": sid, "dialogue": text})
|
||||
}).collect::<Vec<_>>()})
|
||||
.to_string(),
|
||||
)
|
||||
}
|
||||
"speaker_interaction" => {
|
||||
let name_a = identity_name.unwrap_or("");
|
||||
let name_b = identity_b.unwrap_or("");
|
||||
if name_a.is_empty() || name_b.is_empty() {
|
||||
return Ok(
|
||||
serde_json::json!({"error": "identity_name and identity_b are required"})
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
let rows: Vec<(String, String, serde_json::Value)> = sqlx::query_as(&format!(
|
||||
"SELECT sn.external_id, sn.properties->>'full_text' AS full_text, sn.properties->'segments' AS segments \
|
||||
FROM {} i \
|
||||
JOIN {} fd ON fd.identity_id = i.id AND ($3::text IS NULL OR fd.file_uuid = $3) \
|
||||
JOIN {} fn ON fn.file_uuid = fd.file_uuid \
|
||||
AND fn.node_type = 'face_trace' \
|
||||
AND fn.external_id = CONCAT('trace_', fd.trace_id) \
|
||||
JOIN {} e ON e.source_node_id = fn.id \
|
||||
AND e.edge_type = 'SPEAKS_AS' \
|
||||
AND ($3::text IS NULL OR e.file_uuid = $3) \
|
||||
JOIN {} sn ON sn.id = e.target_node_id \
|
||||
WHERE (i.name ILIKE $1 OR i.name ILIKE $2) \
|
||||
ORDER BY sn.external_id",
|
||||
id_table, fd_table, nodes, edges, nodes
|
||||
))
|
||||
.bind(name_a)
|
||||
.bind(name_b)
|
||||
.bind(file_uuid)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let mut interactions = Vec::new();
|
||||
for i in 0..rows.len() {
|
||||
for j in i + 1..rows.len() {
|
||||
let (sid_a, text_a, segs_a_val) = &rows[i];
|
||||
let (sid_b, text_b, segs_b_val) = &rows[j];
|
||||
let segs_a = segs_a_val.as_array();
|
||||
let segs_b = segs_b_val.as_array();
|
||||
if let (Some(a_list), Some(b_list)) = (segs_a, segs_b) {
|
||||
for sa in a_list {
|
||||
let sa_start = sa.get("start").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||
let sa_end = sa.get("end").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||
let sa_text = sa.get("text").and_then(|v| v.as_str()).unwrap_or("");
|
||||
if sa_text.is_empty() {
|
||||
continue;
|
||||
}
|
||||
for sb in b_list {
|
||||
let sb_start =
|
||||
sb.get("start").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||
let sb_end = sb.get("end").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||
let sb_text = sb.get("text").and_then(|v| v.as_str()).unwrap_or("");
|
||||
if sb_text.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let overlap_start = sa_start.max(sb_start);
|
||||
let overlap_end = sa_end.min(sb_end);
|
||||
if overlap_start < overlap_end {
|
||||
interactions.push(serde_json::json!({
|
||||
"speaker_a": sid_a,
|
||||
"speaker_b": sid_b,
|
||||
"time_range_s": [overlap_start, overlap_end],
|
||||
"dialogue_a": sa_text,
|
||||
"dialogue_b": sb_text,
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
interactions.sort_by(|a, b| {
|
||||
let a_start = a["time_range_s"][0].as_f64().unwrap_or(0.0);
|
||||
let b_start = b["time_range_s"][0].as_f64().unwrap_or(0.0);
|
||||
a_start.partial_cmp(&b_start).unwrap()
|
||||
});
|
||||
interactions.truncate(limit as usize);
|
||||
|
||||
Ok(serde_json::json!({"interactions": interactions, "speaker_a_text": rows.first().map(|r| r.1.clone()), "speaker_b_text": rows.get(1).map(|r| r.1.clone())}).to_string())
|
||||
}
|
||||
_ => Ok(
|
||||
serde_json::json!({"error": format!("Unknown query_type: {}", query_type)}).to_string(),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn exec_smart_search(
|
||||
pool: &sqlx::PgPool,
|
||||
args: &serde_json::Value,
|
||||
) -> Result<String, String> {
|
||||
let query = args.get("query").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let file_uuid = args.get("file_uuid").and_then(|v| v.as_str());
|
||||
let limit = args.get("limit").and_then(|v| v.as_i64()).unwrap_or(5);
|
||||
|
||||
let chunk_table = schema::table_name("chunk");
|
||||
let mut sql = format!(
|
||||
"SELECT chunk_id, text_content, start_frame, end_frame, chunk_type \
|
||||
FROM {} WHERE text_content ILIKE $1",
|
||||
chunk_table
|
||||
);
|
||||
if file_uuid.is_some() {
|
||||
sql.push_str(" AND file_uuid = $2");
|
||||
}
|
||||
sql.push_str(&format!(" ORDER BY start_frame LIMIT {}", limit));
|
||||
|
||||
if let Some(fuid) = file_uuid {
|
||||
let like = format!("%{}%", query);
|
||||
let rows: Vec<(String, Option<String>, i64, i64, String)> = sqlx::query_as(&sql)
|
||||
.bind(&like)
|
||||
.bind(fuid)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(serde_json::json!({"results": rows}).to_string())
|
||||
} else {
|
||||
let like = format!("%{}%", query);
|
||||
let rows: Vec<(String, Option<String>, i64, i64, String)> = sqlx::query_as(&sql)
|
||||
.bind(&like)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(serde_json::json!({"results": rows}).to_string())
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn exec_identity_text(
|
||||
pool: &sqlx::PgPool,
|
||||
args: &serde_json::Value,
|
||||
) -> Result<String, String> {
|
||||
let q = args.get("q").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let file_uuid = args.get("file_uuid").and_then(|v| v.as_str());
|
||||
let limit = args
|
||||
.get("limit")
|
||||
.and_then(|v| v.as_i64())
|
||||
.unwrap_or(10)
|
||||
.min(50);
|
||||
|
||||
let chunk_table = schema::table_name("chunk");
|
||||
let fd_table = schema::table_name("face_detections");
|
||||
let id_table = schema::table_name("identities");
|
||||
let like_q = format!("%{}%", q.replace('%', "%%"));
|
||||
|
||||
let sql = format!(
|
||||
"SELECT c.chunk_id, c.start_time, c.end_time, c.text_content, \
|
||||
i.name AS identity_name, fd.trace_id, i.source AS identity_source \
|
||||
FROM {} c \
|
||||
JOIN {} fd ON fd.file_uuid = c.file_uuid \
|
||||
AND fd.frame_number BETWEEN c.start_frame AND c.end_frame \
|
||||
AND fd.identity_id IS NOT NULL \
|
||||
JOIN {} i ON i.id = fd.identity_id \
|
||||
WHERE ($1::text IS NULL OR c.file_uuid = $1) \
|
||||
AND (LOWER(c.text_content) LIKE LOWER($2) OR LOWER(c.content::text) LIKE LOWER($2)) \
|
||||
ORDER BY c.start_time \
|
||||
LIMIT $3",
|
||||
chunk_table, fd_table, id_table
|
||||
);
|
||||
|
||||
let rows: Vec<(
|
||||
String,
|
||||
f64,
|
||||
f64,
|
||||
Option<String>,
|
||||
String,
|
||||
Option<i32>,
|
||||
String,
|
||||
)> = sqlx::query_as(&sql)
|
||||
.bind(file_uuid)
|
||||
.bind(&like_q)
|
||||
.bind(limit)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(
|
||||
serde_json::json!({"results": rows.iter().map(|(chunk_id, st, et, txt, name, tid, src)| {
|
||||
serde_json::json!({
|
||||
"chunk_id": chunk_id,
|
||||
"start_time": st,
|
||||
"end_time": et,
|
||||
"text": txt,
|
||||
"identity_name": name,
|
||||
"trace_id": tid,
|
||||
"source": src
|
||||
})
|
||||
} ).collect::<Vec<_>>()})
|
||||
.to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn exec_identities_search(
|
||||
pool: &sqlx::PgPool,
|
||||
args: &serde_json::Value,
|
||||
) -> Result<String, String> {
|
||||
let q = args.get("q").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let file_uuid = args.get("file_uuid").and_then(|v| v.as_str());
|
||||
let limit = args
|
||||
.get("limit")
|
||||
.and_then(|v| v.as_i64())
|
||||
.unwrap_or(10)
|
||||
.min(50);
|
||||
|
||||
let id_table = schema::table_name("identities");
|
||||
let fd_table = schema::table_name("face_detections");
|
||||
let chunk_table = schema::table_name("chunk");
|
||||
let like_q = format!("%{}%", q.replace('%', "%%"));
|
||||
|
||||
let sql = format!(
|
||||
"SELECT DISTINCT ON (i.name, c.chunk_id) \
|
||||
i.name, c.chunk_id, c.start_time, c.end_time, c.text_content, fd.trace_id \
|
||||
FROM {} i \
|
||||
JOIN {} fd ON fd.identity_id = i.id \
|
||||
JOIN {} c ON c.file_uuid = fd.file_uuid \
|
||||
AND c.start_time <= fd.frame_number / COALESCE(c.fps, 25.0) \
|
||||
AND c.end_time >= fd.frame_number / COALESCE(c.fps, 25.0) \
|
||||
WHERE (i.name ILIKE $1 \
|
||||
OR EXISTS (SELECT 1 FROM jsonb_array_elements(i.metadata->'aliases') AS a WHERE a->>'name' ILIKE $1)) \
|
||||
AND ($2::text IS NULL OR fd.file_uuid = $2) \
|
||||
ORDER BY i.name, c.chunk_id, c.start_time \
|
||||
LIMIT $3",
|
||||
id_table, fd_table, chunk_table
|
||||
);
|
||||
|
||||
let rows: Vec<(String, String, f64, f64, Option<String>, Option<i32>)> = sqlx::query_as(&sql)
|
||||
.bind(&like_q)
|
||||
.bind(file_uuid)
|
||||
.bind(limit)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(
|
||||
serde_json::json!({"results": rows.iter().map(|(name, chunk_id, st, et, txt, tid)| {
|
||||
serde_json::json!({
|
||||
"identity_name": name,
|
||||
"chunk_id": chunk_id,
|
||||
"start_time": st,
|
||||
"end_time": et,
|
||||
"text": txt,
|
||||
"trace_id": tid,
|
||||
})
|
||||
}).collect::<Vec<_>>()})
|
||||
.to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn exec_get_identity_detail(
|
||||
pool: &sqlx::PgPool,
|
||||
args: &serde_json::Value,
|
||||
) -> Result<String, String> {
|
||||
let name = args.get("name").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let id_table = schema::table_name("identities");
|
||||
let row: Option<(String, String, Option<String>, Option<i32>, Option<String>)> = sqlx::query_as(&format!(
|
||||
"SELECT uuid::text, name, source, tmdb_id, metadata->>'tmdb_character' FROM {} WHERE name ILIKE $1 LIMIT 1",
|
||||
id_table
|
||||
))
|
||||
.bind(name)
|
||||
.fetch_optional(pool)
|
||||
.await.map_err(|e| e.to_string())?;
|
||||
Ok(serde_json::json!({"identity": row.map(|(u, n, s, t, c)| serde_json::json!({"uuid": u, "name": n, "source": s, "tmdb_id": t, "character": c}))}).to_string())
|
||||
}
|
||||
|
||||
pub async fn exec_get_file_info(
|
||||
pool: &sqlx::PgPool,
|
||||
args: &serde_json::Value,
|
||||
) -> Result<String, String> {
|
||||
let file_uuid = args.get("file_uuid").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let videos = schema::table_name("videos");
|
||||
let row: Option<(String, f64, i32, i32, f64)> = sqlx::query_as(&format!(
|
||||
"SELECT file_name, duration, width, height, fps FROM {} WHERE file_uuid = $1",
|
||||
videos
|
||||
))
|
||||
.bind(file_uuid)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(serde_json::json!({"file_info": row.map(|(n, d, w, h, f)| serde_json::json!({"file_name": n, "duration_sec": d, "width": w, "height": h, "fps": f}))}).to_string())
|
||||
}
|
||||
|
||||
pub async fn exec_get_representative_frame(
|
||||
pool: &sqlx::PgPool,
|
||||
args: &serde_json::Value,
|
||||
) -> Result<String, String> {
|
||||
let file_uuid = args.get("file_uuid").and_then(|v| v.as_str()).unwrap_or("");
|
||||
match query_auto_representative_frame(pool, file_uuid).await {
|
||||
Ok(r) => Ok(serde_json::json!({
|
||||
"frame_number": r.frame_number,
|
||||
"face_quality": r.face_quality,
|
||||
"main_identities": r.main_identities,
|
||||
"traces": r.traces,
|
||||
})
|
||||
.to_string()),
|
||||
Err(e) => Ok(serde_json::json!({"error": e.to_string()}).to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn exec_analyze_frame(
|
||||
pool: &sqlx::PgPool,
|
||||
args: &serde_json::Value,
|
||||
) -> Result<String, String> {
|
||||
let file_uuid = args.get("file_uuid").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let question = args
|
||||
.get("question")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("請描述這個畫面中的內容");
|
||||
|
||||
if file_uuid.is_empty() {
|
||||
return Ok(serde_json::json!({"error": "file_uuid is required"}).to_string());
|
||||
}
|
||||
|
||||
let videos = schema::table_name("videos");
|
||||
let (video_path, fps): (String, f64) = sqlx::query_as(&format!(
|
||||
"SELECT file_path, COALESCE(fps, 25.0) FROM {} WHERE file_uuid = $1",
|
||||
videos
|
||||
))
|
||||
.bind(file_uuid)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.ok_or_else(|| "Video not found".to_string())?;
|
||||
|
||||
let frame_number = match args.get("frame_number").and_then(|v| v.as_i64()) {
|
||||
Some(f) => f,
|
||||
None => {
|
||||
match query_auto_representative_frame(pool, file_uuid)
|
||||
.await
|
||||
{
|
||||
Ok(r) => r.frame_number,
|
||||
Err(_) => {
|
||||
let duration: f64 = sqlx::query_scalar(&format!(
|
||||
"SELECT COALESCE(duration, 0) FROM {} WHERE file_uuid = $1",
|
||||
videos
|
||||
))
|
||||
.bind(file_uuid)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.unwrap_or(0.0);
|
||||
if duration > 0.0 {
|
||||
((duration / 2.0) * fps) as i64
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let timestamp_secs = frame_number as f64 / fps;
|
||||
|
||||
let ffmpeg_path = std::env::var("MOMENTRY_FFMPEG").unwrap_or_else(|_| {
|
||||
let full = "/opt/homebrew/opt/ffmpeg-full/bin/ffmpeg";
|
||||
if std::path::Path::new(full).exists() {
|
||||
full.to_string()
|
||||
} else {
|
||||
"ffmpeg".to_string()
|
||||
}
|
||||
});
|
||||
|
||||
let output = tokio::process::Command::new(&ffmpeg_path)
|
||||
.args([
|
||||
"-ss",
|
||||
&format!("{:.3}", timestamp_secs),
|
||||
"-i",
|
||||
&video_path,
|
||||
"-vframes",
|
||||
"1",
|
||||
"-f",
|
||||
"image2pipe",
|
||||
"-vcodec",
|
||||
"mjpeg",
|
||||
"-",
|
||||
])
|
||||
.output()
|
||||
.await
|
||||
.map_err(|e| format!("ffmpeg execution error: {}", e))?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
return Ok(serde_json::json!({"error": format!("ffmpeg failed: {}", stderr)}).to_string());
|
||||
}
|
||||
|
||||
let base64_img = BASE64.encode(&output.stdout);
|
||||
|
||||
let system_prompt =
|
||||
"你是一個專業的影片畫面分析助手。請根據提供的畫面以及用戶的問題，詳細描述畫面中的內容，包括場景、人物、動作、表情、物件等。請用繁體中文回答。";
|
||||
let vision_result = call_llm_vision(system_prompt, question, vec![base64_img], 1024, 120)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(serde_json::json!({
|
||||
"frame_number": frame_number,
|
||||
"timestamp_secs": timestamp_secs,
|
||||
"analysis": vision_result,
|
||||
})
|
||||
.to_string())
|
||||
}
|
||||
|
||||
pub async fn exec_tkg_nodes_query(
|
||||
pool: &sqlx::PgPool,
|
||||
args: &serde_json::Value,
|
||||
) -> Result<String, String> {
|
||||
let file_uuid = args
|
||||
.get("file_uuid")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("file_uuid is required")?;
|
||||
let node_type = args.get("node_type").and_then(|v| v.as_str());
|
||||
let page = args.get("page").and_then(|v| v.as_i64()).unwrap_or(1);
|
||||
let page_size = args.get("page_size").and_then(|v| v.as_i64()).unwrap_or(20);
|
||||
let offset = (page - 1) * page_size;
|
||||
|
||||
let nodes_table = t("tkg_nodes");
|
||||
|
||||
let (where_clause, total) = if let Some(nt) = node_type {
|
||||
let total: i64 = sqlx::query_scalar(&format!(
|
||||
"SELECT COUNT(*) FROM {} WHERE file_uuid = $1 AND node_type = $2",
|
||||
nodes_table
|
||||
))
|
||||
.bind(file_uuid)
|
||||
.bind(nt)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
(
|
||||
"WHERE file_uuid = $1 AND node_type = $2 ORDER BY id LIMIT $3 OFFSET $4".to_string(),
|
||||
total,
|
||||
)
|
||||
} else {
|
||||
let total: i64 = sqlx::query_scalar(&format!(
|
||||
"SELECT COUNT(*) FROM {} WHERE file_uuid = $1",
|
||||
nodes_table
|
||||
))
|
||||
.bind(file_uuid)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
(
|
||||
"WHERE file_uuid = $1 ORDER BY id LIMIT $2 OFFSET $3".to_string(),
|
||||
total,
|
||||
)
|
||||
};
|
||||
|
||||
let query = format!(
|
||||
"SELECT id, node_type, external_id, label, properties FROM {} {}",
|
||||
nodes_table, where_clause
|
||||
);
|
||||
|
||||
let rows: Vec<(i64, String, String, String, serde_json::Value)> = if let Some(nt) = node_type {
|
||||
sqlx::query_as(&query)
|
||||
.bind(file_uuid)
|
||||
.bind(nt)
|
||||
.bind(page_size)
|
||||
.bind(offset)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
} else {
|
||||
sqlx::query_as(&query)
|
||||
.bind(file_uuid)
|
||||
.bind(page_size)
|
||||
.bind(offset)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
};
|
||||
|
||||
let nodes: Vec<serde_json::Value> = rows
|
||||
.into_iter()
|
||||
.map(|(id, node_type, external_id, label, properties)| {
|
||||
serde_json::json!({
|
||||
"id": id,
|
||||
"node_type": node_type,
|
||||
"external_id": external_id,
|
||||
"label": label,
|
||||
"properties": properties
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(serde_json::json!({
|
||||
"file_uuid": file_uuid,
|
||||
"total": total,
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
"nodes": nodes
|
||||
})
|
||||
.to_string())
|
||||
}
|
||||
|
||||
pub async fn exec_tkg_edges_query(
|
||||
pool: &sqlx::PgPool,
|
||||
args: &serde_json::Value,
|
||||
) -> Result<String, String> {
|
||||
let file_uuid = args
|
||||
.get("file_uuid")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("file_uuid is required")?;
|
||||
let edge_type = args.get("edge_type").and_then(|v| v.as_str());
|
||||
let page = args.get("page").and_then(|v| v.as_i64()).unwrap_or(1);
|
||||
let page_size = args.get("page_size").and_then(|v| v.as_i64()).unwrap_or(20);
|
||||
let offset = (page - 1) * page_size;
|
||||
|
||||
let edges_table = t("tkg_edges");
|
||||
|
||||
let (where_clause, total) = if let Some(et) = edge_type {
|
||||
let total: i64 = sqlx::query_scalar(&format!(
|
||||
"SELECT COUNT(*) FROM {} WHERE file_uuid = $1 AND edge_type = $2",
|
||||
edges_table
|
||||
))
|
||||
.bind(file_uuid)
|
||||
.bind(et)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
(
|
||||
"WHERE file_uuid = $1 AND edge_type = $2 ORDER BY id LIMIT $3 OFFSET $4".to_string(),
|
||||
total,
|
||||
)
|
||||
} else {
|
||||
let total: i64 = sqlx::query_scalar(&format!(
|
||||
"SELECT COUNT(*) FROM {} WHERE file_uuid = $1",
|
||||
edges_table
|
||||
))
|
||||
.bind(file_uuid)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
(
|
||||
"WHERE file_uuid = $1 ORDER BY id LIMIT $2 OFFSET $3".to_string(),
|
||||
total,
|
||||
)
|
||||
};
|
||||
|
||||
let query = format!(
|
||||
"SELECT id, edge_type, source_node_id, target_node_id, properties FROM {} {}",
|
||||
edges_table, where_clause
|
||||
);
|
||||
|
||||
let rows: Vec<(i64, String, i64, i64, serde_json::Value)> = if let Some(et) = edge_type {
|
||||
sqlx::query_as(&query)
|
||||
.bind(file_uuid)
|
||||
.bind(et)
|
||||
.bind(page_size)
|
||||
.bind(offset)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
} else {
|
||||
sqlx::query_as(&query)
|
||||
.bind(file_uuid)
|
||||
.bind(page_size)
|
||||
.bind(offset)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
};
|
||||
|
||||
let edges: Vec<serde_json::Value> = rows
|
||||
.into_iter()
|
||||
.map(
|
||||
|(id, edge_type, source_node_id, target_node_id, properties)| {
|
||||
serde_json::json!({
|
||||
"id": id,
|
||||
"edge_type": edge_type,
|
||||
"source_node_id": source_node_id,
|
||||
"target_node_id": target_node_id,
|
||||
"properties": properties
|
||||
})
|
||||
},
|
||||
)
|
||||
.collect();
|
||||
|
||||
Ok(serde_json::json!({
|
||||
"file_uuid": file_uuid,
|
||||
"total": total,
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
"edges": edges
|
||||
})
|
||||
.to_string())
|
||||
}
|
||||
|
||||
pub async fn exec_tkg_node_detail(
|
||||
pool: &sqlx::PgPool,
|
||||
args: &serde_json::Value,
|
||||
) -> Result<String, String> {
|
||||
let file_uuid = args
|
||||
.get("file_uuid")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("file_uuid is required")?;
|
||||
let node_id = args
|
||||
.get("node_id")
|
||||
.and_then(|v| v.as_i64())
|
||||
.ok_or("node_id is required")?;
|
||||
|
||||
let nodes_table = t("tkg_nodes");
|
||||
let edges_table = t("tkg_edges");
|
||||
|
||||
let node: Option<(i64, String, String, String, serde_json::Value)> = sqlx::query_as(
|
||||
&format!("SELECT id, node_type, external_id, label, properties FROM {} WHERE file_uuid = $1 AND id = $2", nodes_table)
|
||||
)
|
||||
.bind(file_uuid).bind(node_id)
|
||||
.fetch_optional(pool).await.map_err(|e| e.to_string())?;
|
||||
|
||||
match node {
|
||||
Some((id, node_type, external_id, label, properties)) => {
|
||||
let rows_in: Vec<(i64, String, i64, i64, serde_json::Value)> = sqlx::query_as(
|
||||
&format!("SELECT id, edge_type, source_node_id, target_node_id, properties FROM {} WHERE file_uuid = $1 AND target_node_id = $2", edges_table)
|
||||
)
|
||||
.bind(file_uuid).bind(node_id)
|
||||
.fetch_all(pool).await.unwrap_or_default();
|
||||
let incoming: Vec<serde_json::Value> = rows_in.into_iter().map(|(id, edge_type, source_node_id, target_node_id, properties)| {
|
||||
serde_json::json!({"id": id, "edge_type": edge_type, "source_node_id": source_node_id, "target_node_id": target_node_id, "properties": properties})
|
||||
}).collect();
|
||||
|
||||
let rows_out: Vec<(i64, String, i64, i64, serde_json::Value)> = sqlx::query_as(
|
||||
&format!("SELECT id, edge_type, source_node_id, target_node_id, properties FROM {} WHERE file_uuid = $1 AND source_node_id = $2", edges_table)
|
||||
)
|
||||
.bind(file_uuid).bind(node_id)
|
||||
.fetch_all(pool).await.unwrap_or_default();
|
||||
let outgoing: Vec<serde_json::Value> = rows_out.into_iter().map(|(id, edge_type, source_node_id, target_node_id, properties)| {
|
||||
serde_json::json!({"id": id, "edge_type": edge_type, "source_node_id": source_node_id, "target_node_id": target_node_id, "properties": properties})
|
||||
}).collect();
|
||||
|
||||
Ok(serde_json::json!({
|
||||
"node": {"id": id, "node_type": node_type, "external_id": external_id, "label": label, "properties": properties},
|
||||
"incoming_edges": incoming,
|
||||
"outgoing_edges": outgoing
|
||||
}).to_string())
|
||||
}
|
||||
None => Err("Node not found".to_string()),
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user