warren/momentry_core
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

release: v1.3.0 - TKG node type renaming

Browse Source 
Changes:
- Rust: face_trace → face_track (45 occurrences in 8 files)
- Rust: gaze_trace → gaze_track, lip_trace → lip_track
- Python: tkg_builder.py unified + pipeline_checklist.py fixed
- Swift: swift_hand.swift hand state detection (empty vs holding)

Node type changes:
  face_trace    → face_track
  person_trace  → body_track
  gaze_trace    → gaze_track
  lip_trace     → lip_track
  hand_trace    → hand_track
  speaker       → speaker_segment
  object        → detected_object
  text_trace    → text_region

Migration:
  PUBLIC schema: 12970 + 892 + 305 rows updated
This commit is contained in:
Accusys
2026-06-22 07:18:21 +08:00
parent bce9435823
commit 7e548f8b08
35 changed files with 2789 additions and 481 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/processor/executor.rs
+89 -8
View File
@@ -8,6 +8,8 @@ use tokio::io::{AsyncBufReadExt, BufReader};
use tokio::process::Command;
use tokio::time::{sleep, timeout};
use crate::core::config::{DATABASE_SCHEMA, OUTPUT_DIR, REDIS_KEY_PREFIX};
#[derive(Debug, Clone)]
pub struct RetryConfig {
pub max_attempts: u32,
@@ -292,6 +294,10 @@ impl PythonExecutor {
}
let mut cmd = Command::new(&self.python_path);
cmd.env("MOMENTRY_OUTPUT_DIR", &*OUTPUT_DIR);
cmd.env("DATABASE_SCHEMA", &*DATABASE_SCHEMA);
cmd.env("MOMENTRY_DB_SCHEMA", &*DATABASE_SCHEMA);
cmd.env("MOMENTRY_REDIS_PREFIX", &*REDIS_KEY_PREFIX);
cmd.arg(&script_path);
for arg in args {
@@ -302,11 +308,18 @@ impl PythonExecutor {
cmd.arg("--uuid").arg(u);
}
// Pass frame list for 8Hz sampling
// Pass frame list for 8Hz sampling (only if non-empty)
if let Some(frames) = frames {
let frames_str = Self::format_frames_arg(frames);
cmd.arg("--frames").arg(&frames_str);
tracing::info!("[{}] 8Hz sampling: {} frames", log_prefix, frames.len());
if !frames.is_empty() {
let frames_str = Self::format_frames_arg(frames);
cmd.arg("--frames").arg(&frames_str);
tracing::info!("[{}] 8Hz sampling: {} frames", log_prefix, frames.len());
} else {
tracing::info!(
"[{}] 8Hz sampling: 0 frames (skipping --frames arg)",
log_prefix
);
}
}
cmd.stdout(Stdio::piped());
@@ -419,6 +432,10 @@ impl PythonExecutor {
}
let mut cmd = Command::new(&self.python_path);
cmd.env("MOMENTRY_OUTPUT_DIR", &*OUTPUT_DIR);
cmd.env("DATABASE_SCHEMA", &*DATABASE_SCHEMA);
cmd.env("MOMENTRY_DB_SCHEMA", &*DATABASE_SCHEMA);
cmd.env("MOMENTRY_REDIS_PREFIX", &*REDIS_KEY_PREFIX);
cmd.arg(&script_path);
for arg in args {
@@ -593,6 +610,10 @@ impl PythonExecutor {
}
let mut cmd = Command::new(&self.python_path);
cmd.env("MOMENTRY_OUTPUT_DIR", &*OUTPUT_DIR);
cmd.env("DATABASE_SCHEMA", &*DATABASE_SCHEMA);
cmd.env("MOMENTRY_DB_SCHEMA", &*DATABASE_SCHEMA);
cmd.env("MOMENTRY_REDIS_PREFIX", &*REDIS_KEY_PREFIX);
cmd.arg(&script_path);
for arg in args {
@@ -603,11 +624,18 @@ impl PythonExecutor {
cmd.arg("--uuid").arg(u);
}
// Pass frame list for 8Hz sampling
// Pass frame list for 8Hz sampling (only if non-empty)
if let Some(frames) = frames {
let frames_str = Self::format_frames_arg(frames);
cmd.arg("--frames").arg(&frames_str);
tracing::info!("[{}] 8Hz sampling: {} frames", log_prefix, frames.len());
if !frames.is_empty() {
let frames_str = Self::format_frames_arg(frames);
cmd.arg("--frames").arg(&frames_str);
tracing::info!("[{}] 8Hz sampling: {} frames", log_prefix, frames.len());
} else {
tracing::info!(
"[{}] 8Hz sampling: 0 frames (skipping --frames arg)",
log_prefix
);
}
}
cmd.stdout(Stdio::piped());
@@ -826,6 +854,59 @@ impl Default for PythonExecutor {
#[cfg(test)]
mod tests {
use super::*;
use std::process::Stdio;
#[tokio::test]
async fn test_executor_passes_env_vars() {
let executor = PythonExecutor::new().unwrap();
let mut cmd = Command::new(&executor.python_path);
cmd.env("MOMENTRY_OUTPUT_DIR", &*OUTPUT_DIR);
cmd.env("DATABASE_SCHEMA", &*DATABASE_SCHEMA);
cmd.env("MOMENTRY_DB_SCHEMA", &*DATABASE_SCHEMA);
cmd.env("MOMENTRY_REDIS_PREFIX", &*REDIS_KEY_PREFIX);
cmd.args([
"-c",
"import os; print(f'ENV_DATABASE_SCHEMA={os.environ.get(\"DATABASE_SCHEMA\",\"\")}'); print(f'ENV_MOMENTRY_DB_SCHEMA={os.environ.get(\"MOMENTRY_DB_SCHEMA\",\"\")}'); print(f'ENV_MOMENTRY_OUTPUT_DIR={os.environ.get(\"MOMENTRY_OUTPUT_DIR\",\"\")}'); print(f'ENV_MOMENTRY_REDIS_PREFIX={os.environ.get(\"MOMENTRY_REDIS_PREFIX\",\"\")}');",
]);
cmd.stdout(Stdio::piped());
cmd.stderr(Stdio::piped());
let output = cmd.output().await.expect("Failed to run inline Python");
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
println!("stdout: {}", stdout);
if !stderr.is_empty() {
println!("stderr: {}", stderr);
}
assert!(
output.status.success(),
"Python inline script failed: {}",
stderr
);
assert!(
stdout.contains(&format!("ENV_DATABASE_SCHEMA={}", *DATABASE_SCHEMA)),
"DATABASE_SCHEMA mismatch:\n{}",
stdout
);
assert!(
stdout.contains(&format!("ENV_MOMENTRY_DB_SCHEMA={}", *DATABASE_SCHEMA)),
"MOMENTRY_DB_SCHEMA mismatch:\n{}",
stdout
);
assert!(
stdout.contains(&format!("ENV_MOMENTRY_OUTPUT_DIR={}", *OUTPUT_DIR)),
"MOMENTRY_OUTPUT_DIR mismatch:\n{}",
stdout
);
assert!(
stdout.contains(&format!("ENV_MOMENTRY_REDIS_PREFIX={}", *REDIS_KEY_PREFIX)),
"MOMENTRY_REDIS_PREFIX mismatch:\n{}",
stdout
);
}
#[test]
fn test_python_executor_new() {
src/core/processor/tkg.rs
+227 -127
View File
@@ -26,7 +26,7 @@ async fn populate_face_detections_from_face_json(
use tracing::info;
let fd_table = t("face_detections");
// Check if trace_id is already populated
let traced_count: i64 = sqlx::query_scalar(&format!(
"SELECT COUNT(*) FROM {} WHERE file_uuid = $1 AND trace_id IS NOT NULL",
@@ -37,7 +37,10 @@ async fn populate_face_detections_from_face_json(
.await?;
if traced_count > 0 {
info!("[TKG-Phase0] face_detections already traced for {} ({} rows with trace_id)", file_uuid, traced_count);
info!(
"[TKG-Phase0] face_detections already traced for {} ({} rows with trace_id)",
file_uuid, traced_count
);
return Ok(());
}
@@ -50,11 +53,17 @@ async fn populate_face_detections_from_face_json(
.await?;
if total_count == 0 {
info!("[TKG-Phase0] No face_detections for {}, need face processor first", file_uuid);
info!(
"[TKG-Phase0] No face_detections for {}, need face processor first",
file_uuid
);
return Ok(());
}
info!("[TKG-Phase0] {} faces exist but trace_id=NULL, calling store_traced_faces.py...", total_count);
info!(
"[TKG-Phase0] {} faces exist but trace_id=NULL, calling store_traced_faces.py...",
total_count
);
let executor = PythonExecutor::new()?;
@@ -77,11 +86,17 @@ async fn populate_face_detections_from_face_json(
.bind(file_uuid)
.fetch_one(pool)
.await?;
info!("[TKG-Phase0] Traced {} face_detections for {}", new_traced_count, file_uuid);
info!(
"[TKG-Phase0] Traced {} face_detections for {}",
new_traced_count, file_uuid
);
Ok(())
}
Err(e) => {
info!("[TKG-Phase0] Failed to trace face_detections: {} (continuing with TKG build)", e);
info!(
"[TKG-Phase0] Failed to trace face_detections: {} (continuing with TKG build)",
e
);
Ok(())
}
}
@@ -103,7 +118,11 @@ async fn populate_face_embeddings_to_qdrant(
// Check if embeddings already exist
let existing = face_db.get_all_embeddings_for_file(file_uuid).await?;
if !existing.is_empty() {
info!("[TKG-Phase1] {} embeddings already in Qdrant for {}", existing.len(), file_uuid);
info!(
"[TKG-Phase1] {} embeddings already in Qdrant for {}",
existing.len(),
file_uuid
);
return Ok(existing.len());
}
@@ -129,8 +148,8 @@ async fn populate_face_embeddings_to_qdrant(
let mut points: Vec<(String, Vec<f32>, FaceEmbeddingPayload)> = Vec::new();
for (trace_id, frame, x, y, w, h, confidence, embedding) in &rows {
if let Some(emb) = embedding {
let (yaw, pitch, roll) = get_pose_for_face(*frame, *x, *y, *w, *h, &pose_data)
.unwrap_or((0.0, 0.0, 0.0));
let (yaw, pitch, roll) =
get_pose_for_face(*frame, *x, *y, *w, *h, &pose_data).unwrap_or((0.0, 0.0, 0.0));
// Generate unique numeric point ID (trace_id * 100000 + frame)
let point_id = format!("{}", (*trace_id as u64) * 100000 + (*frame as u64));
@@ -152,7 +171,10 @@ async fn populate_face_embeddings_to_qdrant(
}
let count = face_db.batch_upsert(points).await?;
info!("[TKG-Phase1] Stored {} face embeddings in Qdrant for {}", count, file_uuid);
info!(
"[TKG-Phase1] Stored {} face embeddings in Qdrant for {}",
count, file_uuid
);
Ok(count)
}
@@ -461,10 +483,10 @@ struct FaceDetectionRow {
// ── Public API ────────────────────────────────────────────────────
pub struct TkgResult {
pub face_trace_nodes: usize,
pub gaze_trace_nodes: usize,
pub lip_trace_nodes: usize,
pub text_trace_nodes: usize,
pub face_track_nodes: usize,
pub gaze_track_nodes: usize,
pub lip_track_nodes: usize,
pub text_region_nodes: usize,
pub appearance_trace_nodes: usize,
pub skin_tone_trace_nodes: usize,
pub accessory_nodes: usize,
@@ -486,28 +508,36 @@ pub async fn build_tkg(db: &PostgresDb, file_uuid: &str, output_dir: &str) -> Re
// Phase 0: Populate face_detections from face.json (if not exists)
if let Err(e) = populate_face_detections_from_face_json(pool, output_dir, file_uuid).await {
tracing::warn!("[TKG-Phase0] populate_face_detections failed: {} (continuing)", e);
tracing::warn!(
"[TKG-Phase0] populate_face_detections failed: {} (continuing)",
e
);
}
// Phase 1: Populate face embeddings to Qdrant (for TKG-only migration)
if let Err(e) = populate_face_embeddings_to_qdrant(pool, output_dir, file_uuid).await {
tracing::warn!("[TKG-Phase1] populate_face_embeddings failed: {} (continuing)", e);
tracing::warn!(
"[TKG-Phase1] populate_face_embeddings failed: {} (continuing)",
e
);
}
let pose_data = load_face_pose_data(output_dir, file_uuid).map_err(|e| {
tracing::error!("[TKG] Failed to load face pose data: {}", e);
e
}).unwrap_or_default();
let pose_data = load_face_pose_data(output_dir, file_uuid)
.map_err(|e| {
tracing::error!("[TKG] Failed to load face pose data: {}", e);
e
})
.unwrap_or_default();
tracing::info!(
"[TKG] Loaded {} pose entries from face.json (output_dir={})",
pose_data.len(),
output_dir
);
let n_face = build_face_trace_nodes(pool, file_uuid, &pose_data).await?;
let n_gaze = build_gaze_trace_nodes(pool, file_uuid, &pose_data).await?;
let n_lip = build_lip_trace_nodes(pool, file_uuid, output_dir, &pose_data).await?;
let n_text = build_text_trace_nodes(pool, file_uuid).await?;
let n_face = build_face_track_nodes(pool, file_uuid, &pose_data).await?;
let n_gaze = build_gaze_track_nodes(pool, file_uuid, &pose_data).await?;
let n_lip = build_lip_track_nodes(pool, file_uuid, output_dir, &pose_data).await?;
let n_text = build_text_region_nodes(pool, file_uuid).await?;
let n_appearance =
build_appearance_trace_nodes(pool, file_uuid, output_dir, &pose_data).await?;
let n_skin = build_skin_tone_trace_nodes(pool, file_uuid, output_dir, &pose_data).await?;
@@ -524,10 +554,10 @@ pub async fn build_tkg(db: &PostgresDb, file_uuid: &str, output_dir: &str) -> Re
let e_w = build_wears_edges(pool, file_uuid).await?;
Ok(TkgResult {
face_trace_nodes: n_face,
gaze_trace_nodes: n_gaze,
lip_trace_nodes: n_lip,
text_trace_nodes: n_text,
face_track_nodes: n_face,
gaze_track_nodes: n_gaze,
lip_track_nodes: n_lip,
text_region_nodes: n_text,
appearance_trace_nodes: n_appearance,
skin_tone_trace_nodes: n_skin,
accessory_nodes: n_accessories,
@@ -545,7 +575,7 @@ pub async fn build_tkg(db: &PostgresDb, file_uuid: &str, output_dir: &str) -> Re
// ── Node builders ─────────────────────────────────────────────────
async fn build_face_trace_nodes(
async fn build_face_track_nodes(
pool: &PgPool,
file_uuid: &str,
pose_data: &[FacePose],
@@ -557,20 +587,28 @@ async fn build_face_trace_nodes(
let qdrant_embeddings = face_db.get_all_embeddings_for_file(file_uuid).await?;
if !qdrant_embeddings.is_empty() {
tracing::info!("[TKG-Phase2] Building face_trace nodes from Qdrant ({} embeddings)", qdrant_embeddings.len());
return build_face_trace_nodes_from_qdrant(pool, file_uuid, pose_data, qdrant_embeddings).await;
tracing::info!(
"[TKG-Phase2] Building face_track nodes from Qdrant ({} embeddings)",
qdrant_embeddings.len()
);
return build_face_track_nodes_from_qdrant(pool, file_uuid, pose_data, qdrant_embeddings)
.await;
}
// Fallback to PostgreSQL
tracing::info!("[TKG-Phase2] No Qdrant embeddings, falling back to PostgreSQL");
build_face_trace_nodes_from_pg(pool, file_uuid, pose_data).await
build_face_track_nodes_from_pg(pool, file_uuid, pose_data).await
}
async fn build_face_trace_nodes_from_qdrant(
async fn build_face_track_nodes_from_qdrant(
pool: &PgPool,
file_uuid: &str,
pose_data: &[FacePose],
qdrant_embeddings: Vec<(String, Vec<f32>, crate::core::db::face_embedding_db::FaceEmbeddingPayload)>,
qdrant_embeddings: Vec<(
String,
Vec<f32>,
crate::core::db::face_embedding_db::FaceEmbeddingPayload,
)>,
) -> Result<usize> {
use crate::core::db::face_embedding_db::FaceEmbeddingPayload;
let nodes_table = t("tkg_nodes");
@@ -598,7 +636,7 @@ async fn build_face_trace_nodes_from_qdrant(
// Build aggregates
let mut count = 0;
for (tid, frames) in &trace_frames {
let external_id = format!("trace_{}", tid);
let external_id = format!("face_track_{}", tid);
let label = format!("Face Trace {}", tid);
let frame_count = frames.len() as i64;
@@ -625,7 +663,11 @@ async fn build_face_trace_nodes_from_qdrant(
}
let (avg_yaw, avg_pitch, avg_roll) = if pose_count > 0 {
(yaw_sum / pose_count as f64, pitch_sum / pose_count as f64, roll_sum / pose_count as f64)
(
yaw_sum / pose_count as f64,
pitch_sum / pose_count as f64,
roll_sum / pose_count as f64,
)
} else {
(0.0, 0.0, 0.0)
};
@@ -653,7 +695,7 @@ async fn build_face_trace_nodes_from_qdrant(
nodes_table
))
.bind(file_uuid)
.bind("face_trace")
.bind("face_track")
.bind(&external_id)
.bind(&label)
.bind(serde_json::to_string(&props)?)
@@ -663,11 +705,11 @@ async fn build_face_trace_nodes_from_qdrant(
count += 1;
}
tracing::info!("[TKG-Phase2] Built {} face_trace nodes from Qdrant", count);
tracing::info!("[TKG-Phase2] Built {} face_track nodes from Qdrant", count);
Ok(count)
}
async fn build_face_trace_nodes_from_pg(
async fn build_face_track_nodes_from_pg(
pool: &PgPool,
file_uuid: &str,
pose_data: &[FacePose],
@@ -720,7 +762,7 @@ async fn build_face_trace_nodes_from_pg(
let mut count = 0;
for row in &rows {
let tid = row.trace_id;
let external_id = format!("trace_{}", tid);
let external_id = format!("face_track_{}", tid);
let label = format!("Face Trace {}", tid);
// Compute average pose for this trace
@@ -779,7 +821,7 @@ async fn build_face_trace_nodes_from_pg(
"#,
nodes_table
))
.bind("face_trace")
.bind("face_track")
.bind(&external_id)
.bind(file_uuid)
.bind(&label)
@@ -944,7 +986,13 @@ async fn build_co_occurrence_edges(
"[TKG-Phase2.6.1] Building co_occurrence edges from Qdrant ({} embeddings)",
qdrant_embeddings.len()
);
return build_co_occurrence_edges_from_qdrant(pool, file_uuid, output_dir, qdrant_embeddings).await;
return build_co_occurrence_edges_from_qdrant(
pool,
file_uuid,
output_dir,
qdrant_embeddings,
)
.await;
}
tracing::info!("[TKG-Phase2.6.1] No Qdrant embeddings, falling back to PostgreSQL");
@@ -955,7 +1003,11 @@ async fn build_co_occurrence_edges_from_qdrant(
pool: &PgPool,
file_uuid: &str,
output_dir: &str,
qdrant_embeddings: Vec<(String, Vec<f32>, crate::core::db::face_embedding_db::FaceEmbeddingPayload)>,
qdrant_embeddings: Vec<(
String,
Vec<f32>,
crate::core::db::face_embedding_db::FaceEmbeddingPayload,
)>,
) -> Result<usize> {
use crate::core::db::face_embedding_db::FaceEmbeddingPayload;
@@ -974,10 +1026,13 @@ async fn build_co_occurrence_edges_from_qdrant(
for (_, _, payload) in &qdrant_embeddings {
let frame = payload.frame;
let trace_id = payload.trace_id as i64;
frame_faces
.entry(frame)
.or_default()
.push((trace_id, payload.bbox_x, payload.bbox_y, payload.bbox_w, payload.bbox_h));
frame_faces.entry(frame).or_default().push((
trace_id,
payload.bbox_x,
payload.bbox_y,
payload.bbox_w,
payload.bbox_h,
));
}
let mut edge_count = 0;
@@ -999,9 +1054,9 @@ async fn build_co_occurrence_edges_from_qdrant(
}
for (trace_id, _, _, _, _) in faces {
let external_id = format!("trace_{}", trace_id);
let external_id = format!("face_track_{}", trace_id);
let face_node: Option<(i64,)> = sqlx::query_as(&format!(
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_trace' AND external_id=$2",
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_track' AND external_id=$2",
nodes_table
))
.bind(file_uuid)
@@ -1113,9 +1168,9 @@ async fn build_co_occurrence_edges_from_pg(
continue;
}
let external_id = format!("trace_{}", face.trace_id);
let external_id = format!("face_track_{}", face.trace_id);
let face_node: Option<(i64,)> = sqlx::query_as(&format!(
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_trace' AND external_id=$2",
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_track' AND external_id=$2",
nodes_table
))
.bind(file_uuid)
@@ -1196,7 +1251,13 @@ async fn build_speaker_face_edges(
"[TKG-Phase2.6.3] Building speaker_face edges from Qdrant ({} embeddings)",
qdrant_embeddings.len()
);
return build_speaker_face_edges_from_qdrant(pool, file_uuid, output_dir, qdrant_embeddings).await;
return build_speaker_face_edges_from_qdrant(
pool,
file_uuid,
output_dir,
qdrant_embeddings,
)
.await;
}
tracing::info!("[TKG-Phase2.6.3] No Qdrant embeddings, falling back to PostgreSQL");
@@ -1207,7 +1268,11 @@ async fn build_speaker_face_edges_from_qdrant(
pool: &PgPool,
file_uuid: &str,
output_dir: &str,
qdrant_embeddings: Vec<(String, Vec<f32>, crate::core::db::face_embedding_db::FaceEmbeddingPayload)>,
qdrant_embeddings: Vec<(
String,
Vec<f32>,
crate::core::db::face_embedding_db::FaceEmbeddingPayload,
)>,
) -> Result<usize> {
use crate::core::db::face_embedding_db::FaceEmbeddingPayload;
@@ -1245,9 +1310,9 @@ async fn build_speaker_face_edges_from_qdrant(
let mut edge_count = 0;
for (tid, (sf, ef)) in &trace_ranges {
let face_ext_id = format!("trace_{}", tid);
let face_ext_id = format!("face_track_{}", tid);
let face_node: Option<(i64,)> = sqlx::query_as(&format!(
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_trace' AND external_id=$2",
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_track' AND external_id=$2",
nodes_table
))
.bind(file_uuid)
@@ -1370,9 +1435,9 @@ async fn build_speaker_face_edges_from_pg(
let mut edge_count = 0;
for (tid, sf, ef) in &traces {
let face_ext_id = format!("trace_{}", tid);
let face_ext_id = format!("face_track_{}", tid);
let face_node: Option<(i64,)> = sqlx::query_as(&format!(
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_trace' AND external_id=$2",
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_track' AND external_id=$2",
nodes_table
))
.bind(file_uuid)
@@ -1469,7 +1534,8 @@ async fn build_face_face_edges(
"[TKG-Phase2.6.2] Building face_face edges from Qdrant ({} embeddings)",
qdrant_embeddings.len()
);
return build_face_face_edges_from_qdrant(pool, file_uuid, pose_data, qdrant_embeddings).await;
return build_face_face_edges_from_qdrant(pool, file_uuid, pose_data, qdrant_embeddings)
.await;
}
tracing::info!("[TKG-Phase2.6.2] No Qdrant embeddings, falling back to PostgreSQL");
@@ -1480,7 +1546,11 @@ async fn build_face_face_edges_from_qdrant(
pool: &PgPool,
file_uuid: &str,
pose_data: &[FacePose],
qdrant_embeddings: Vec<(String, Vec<f32>, crate::core::db::face_embedding_db::FaceEmbeddingPayload)>,
qdrant_embeddings: Vec<(
String,
Vec<f32>,
crate::core::db::face_embedding_db::FaceEmbeddingPayload,
)>,
) -> Result<usize> {
use crate::core::db::face_embedding_db::FaceEmbeddingPayload;
@@ -1489,20 +1559,31 @@ async fn build_face_face_edges_from_qdrant(
let mut frame_faces: HashMap<i64, Vec<FaceEmbeddingPayload>> = HashMap::new();
for (_, _, payload) in &qdrant_embeddings {
frame_faces.entry(payload.frame).or_default().push(payload.clone());
frame_faces
.entry(payload.frame)
.or_default()
.push(payload.clone());
}
let mut frame_map: HashMap<(i64, i64), (f64, f64, f64, f64)> = HashMap::new();
for (_, _, payload) in &qdrant_embeddings {
let trace_id = payload.trace_id as i64;
let frame = payload.frame;
frame_map.insert((trace_id, frame), (payload.bbox_x, payload.bbox_y, payload.bbox_w, payload.bbox_h));
frame_map.insert(
(trace_id, frame),
(
payload.bbox_x,
payload.bbox_y,
payload.bbox_w,
payload.bbox_h,
),
);
}
let mut rows: Vec<(i64, i64, i64)> = Vec::new();
for (frame, faces) in frame_faces.iter() {
for i in 0..faces.len() {
for j in (i+1)..faces.len() {
for j in (i + 1)..faces.len() {
let tid_a = faces[i].trace_id as i64;
let tid_b = faces[j].trace_id as i64;
let min_tid = tid_a.min(tid_b);
@@ -1536,14 +1617,14 @@ async fn build_face_face_edges_from_qdrant(
let mut edge_count = 0;
let mut node_id_cache: HashMap<i64, i64> = HashMap::new();
for ((tid_a, tid_b), frame_data) in &pair_frames {
let ext_a = format!("trace_{}", tid_a);
let ext_b = format!("trace_{}", tid_b);
let ext_a = format!("face_track_{}", tid_a);
let ext_b = format!("face_track_{}", tid_b);
let n_a_id = match node_id_cache.get(tid_a) {
Some(id) => *id,
None => {
if let Some((id,)) = sqlx::query_as::<_, (i64,)>(&format!(
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_trace' AND external_id=$2",
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_track' AND external_id=$2",
nodes_table
))
.bind(file_uuid).bind(&ext_a).fetch_optional(pool).await?
@@ -1558,7 +1639,7 @@ async fn build_face_face_edges_from_qdrant(
Some(id) => *id,
None => {
if let Some((id,)) = sqlx::query_as::<_, (i64,)>(&format!(
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_trace' AND external_id=$2",
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_track' AND external_id=$2",
nodes_table
))
.bind(file_uuid).bind(&ext_b).fetch_optional(pool).await?
@@ -1711,14 +1792,14 @@ async fn build_face_face_edges_from_pg(
let mut edge_count = 0;
let mut node_id_cache: HashMap<i64, i64> = HashMap::new();
for ((tid_a, tid_b), frame_data) in &pair_frames {
let ext_a = format!("trace_{}", tid_a);
let ext_b = format!("trace_{}", tid_b);
let ext_a = format!("face_track_{}", tid_a);
let ext_b = format!("face_track_{}", tid_b);
let n_a_id = match node_id_cache.get(tid_a) {
Some(id) => *id,
None => {
if let Some((id,)) = sqlx::query_as::<_, (i64,)>(&format!(
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_trace' AND external_id=$2",
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_track' AND external_id=$2",
nodes_table
))
.bind(file_uuid).bind(&ext_a).fetch_optional(pool).await?
@@ -1733,7 +1814,7 @@ async fn build_face_face_edges_from_pg(
Some(id) => *id,
None => {
if let Some((id,)) = sqlx::query_as::<_, (i64,)>(&format!(
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_trace' AND external_id=$2",
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_track' AND external_id=$2",
nodes_table
))
.bind(file_uuid).bind(&ext_b).fetch_optional(pool).await?
@@ -1820,7 +1901,7 @@ async fn build_face_face_edges_from_pg(
// ── Gaze Trace Nodes ──────────────────────────────────────────────
async fn build_gaze_trace_nodes(
async fn build_gaze_track_nodes(
pool: &PgPool,
file_uuid: &str,
pose_data: &[FacePose],
@@ -1832,19 +1913,27 @@ async fn build_gaze_trace_nodes(
let qdrant_embeddings = face_db.get_all_embeddings_for_file(file_uuid).await?;
if !qdrant_embeddings.is_empty() {
tracing::info!("[TKG-Phase2.5] Building gaze_trace nodes from Qdrant ({} embeddings)", qdrant_embeddings.len());
return build_gaze_trace_nodes_from_qdrant(pool, file_uuid, pose_data, qdrant_embeddings).await;
tracing::info!(
"[TKG-Phase2.5] Building gaze_track nodes from Qdrant ({} embeddings)",
qdrant_embeddings.len()
);
return build_gaze_track_nodes_from_qdrant(pool, file_uuid, pose_data, qdrant_embeddings)
.await;
}
tracing::info!("[TKG-Phase2.5] No Qdrant embeddings, falling back to PostgreSQL");
build_gaze_trace_nodes_from_pg(pool, file_uuid, pose_data).await
build_gaze_track_nodes_from_pg(pool, file_uuid, pose_data).await
}
async fn build_gaze_trace_nodes_from_qdrant(
async fn build_gaze_track_nodes_from_qdrant(
pool: &PgPool,
file_uuid: &str,
pose_data: &[FacePose],
qdrant_embeddings: Vec<(String, Vec<f32>, crate::core::db::face_embedding_db::FaceEmbeddingPayload)>,
qdrant_embeddings: Vec<(
String,
Vec<f32>,
crate::core::db::face_embedding_db::FaceEmbeddingPayload,
)>,
) -> Result<usize> {
use crate::core::db::face_embedding_db::FaceEmbeddingPayload;
let nodes_table = t("tkg_nodes");
@@ -1873,11 +1962,11 @@ async fn build_gaze_trace_nodes_from_qdrant(
for (tid, frames) in &trace_frames {
let external_id = format!("gaze_{}", tid);
// Phase 2.7: Query face_trace identity_id
let face_ext_id = format!("trace_{}", tid);
// Phase 2.7: Query face_track identity_id
let face_ext_id = format!("face_track_{}", tid);
let face_identity_id: Option<i64> = sqlx::query_scalar(&format!(
"SELECT (properties->>'identity_id')::bigint FROM {}
WHERE file_uuid=$1 AND node_type='face_trace' AND external_id=$2",
WHERE file_uuid=$1 AND node_type='face_track' AND external_id=$2",
nodes_table
))
.bind(file_uuid)
@@ -1969,7 +2058,7 @@ async fn build_gaze_trace_nodes_from_qdrant(
"#,
nodes_table
))
.bind("gaze_trace")
.bind("gaze_track")
.bind(&external_id)
.bind(file_uuid)
.bind(&external_id)
@@ -1980,11 +2069,14 @@ async fn build_gaze_trace_nodes_from_qdrant(
count += 1;
}
tracing::info!("[TKG-Phase2.5] Built {} gaze_trace nodes from Qdrant", count);
tracing::info!(
"[TKG-Phase2.5] Built {} gaze_track nodes from Qdrant",
count
);
Ok(count)
}
async fn build_gaze_trace_nodes_from_pg(
async fn build_gaze_track_nodes_from_pg(
pool: &PgPool,
file_uuid: &str,
pose_data: &[FacePose],
@@ -2103,7 +2195,7 @@ async fn build_gaze_trace_nodes_from_pg(
"#,
nodes_table
))
.bind("gaze_trace")
.bind("gaze_track")
.bind(&external_id)
.bind(file_uuid)
.bind(&format!("Gaze Trace {}", tid))
@@ -2203,15 +2295,15 @@ async fn build_mutual_gaze_edges(
let mut node_id_cache: HashMap<i64, i64> = HashMap::new();
for ((tid_a, tid_b), frames) in &pair_gaze_frames {
let ext_a = format!("trace_{}", tid_a);
let ext_b = format!("trace_{}", tid_b);
let ext_a = format!("face_track_{}", tid_a);
let ext_b = format!("face_track_{}", tid_b);
// Get node IDs
let n_a_id = match node_id_cache.get(tid_a) {
Some(id) => *id,
None => {
if let Some((id,)) = sqlx::query_as::<_, (i64,)>(&format!(
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_trace' AND external_id=$2",
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_track' AND external_id=$2",
nodes_table
))
.bind(file_uuid).bind(&ext_a).fetch_optional(pool).await?
@@ -2226,7 +2318,7 @@ async fn build_mutual_gaze_edges(
Some(id) => *id,
None => {
if let Some((id,)) = sqlx::query_as::<_, (i64,)>(&format!(
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_trace' AND external_id=$2",
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_track' AND external_id=$2",
nodes_table
))
.bind(file_uuid).bind(&ext_b).fetch_optional(pool).await?
@@ -2284,7 +2376,7 @@ async fn build_mutual_gaze_edges(
// ── Lip Trace Nodes ───────────────────────────────────────────────
async fn build_lip_trace_nodes(
async fn build_lip_track_nodes(
pool: &PgPool,
file_uuid: &str,
output_dir: &str,
@@ -2297,20 +2389,31 @@ async fn build_lip_trace_nodes(
let qdrant_embeddings = face_db.get_all_embeddings_for_file(file_uuid).await?;
if !qdrant_embeddings.is_empty() {
tracing::info!("[TKG-Phase2.5] Building lip_trace nodes from Qdrant + face.json");
return build_lip_trace_nodes_from_qdrant(pool, file_uuid, output_dir, pose_data, qdrant_embeddings).await;
tracing::info!("[TKG-Phase2.5] Building lip_track nodes from Qdrant + face.json");
return build_lip_track_nodes_from_qdrant(
pool,
file_uuid,
output_dir,
pose_data,
qdrant_embeddings,
)
.await;
}
tracing::info!("[TKG-Phase2.5] No Qdrant embeddings, falling back to PostgreSQL");
build_lip_trace_nodes_from_pg(pool, file_uuid, output_dir, pose_data).await
build_lip_track_nodes_from_pg(pool, file_uuid, output_dir, pose_data).await
}
async fn build_lip_trace_nodes_from_qdrant(
async fn build_lip_track_nodes_from_qdrant(
pool: &PgPool,
file_uuid: &str,
output_dir: &str,
pose_data: &[FacePose],
qdrant_embeddings: Vec<(String, Vec<f32>, crate::core::db::face_embedding_db::FaceEmbeddingPayload)>,
qdrant_embeddings: Vec<(
String,
Vec<f32>,
crate::core::db::face_embedding_db::FaceEmbeddingPayload,
)>,
) -> Result<usize> {
use crate::core::db::face_embedding_db::FaceEmbeddingPayload;
let nodes_table = t("tkg_nodes");
@@ -2328,16 +2431,13 @@ async fn build_lip_trace_nodes_from_qdrant(
// Build trace_id mapping from Qdrant: frame → Vec<(trace_id, bbox)>
let mut frame_trace_map: HashMap<i64, Vec<(i64, f64, f64, f64, f64)>> = HashMap::new();
for (_, _, payload) in &qdrant_embeddings {
frame_trace_map
.entry(payload.frame)
.or_default()
.push((
payload.trace_id as i64,
payload.bbox_x,
payload.bbox_y,
payload.bbox_w,
payload.bbox_h,
));
frame_trace_map.entry(payload.frame).or_default().push((
payload.trace_id as i64,
payload.bbox_x,
payload.bbox_y,
payload.bbox_w,
payload.bbox_h,
));
}
// Helper function to match trace_id by bbox distance
@@ -2411,11 +2511,11 @@ async fn build_lip_trace_nodes_from_qdrant(
for (tid, frames) in &lip_data {
let external_id = format!("lip_{}", tid);
// Phase 2.7: Query face_trace identity_id
let face_ext_id = format!("trace_{}", tid);
// Phase 2.7: Query face_track identity_id
let face_ext_id = format!("face_track_{}", tid);
let face_identity_id: Option<i64> = sqlx::query_scalar(&format!(
"SELECT (properties->>'identity_id')::bigint FROM {}
WHERE file_uuid=$1 AND node_type='face_trace' AND external_id=$2",
WHERE file_uuid=$1 AND node_type='face_track' AND external_id=$2",
nodes_table
))
.bind(file_uuid)
@@ -2500,7 +2600,7 @@ async fn build_lip_trace_nodes_from_qdrant(
"#,
nodes_table
))
.bind("lip_trace")
.bind("lip_track")
.bind(&external_id)
.bind(file_uuid)
.bind(&format!("Lip Trace {}", tid))
@@ -2511,11 +2611,11 @@ async fn build_lip_trace_nodes_from_qdrant(
count += 1;
}
tracing::info!("[TKG-Phase2.5] Built {} lip_trace nodes from Qdrant", count);
tracing::info!("[TKG-Phase2.5] Built {} lip_track nodes from Qdrant", count);
Ok(count)
}
async fn build_lip_trace_nodes_from_pg(
async fn build_lip_track_nodes_from_pg(
pool: &PgPool,
file_uuid: &str,
output_dir: &str,
@@ -2658,7 +2758,7 @@ async fn build_lip_trace_nodes_from_pg(
"#,
nodes_table
))
.bind("lip_trace")
.bind("lip_track")
.bind(&external_id)
.bind(file_uuid)
.bind(&format!("Lip Trace {}", tid))
@@ -2750,7 +2850,7 @@ async fn get_trace_for_face(
// ── Text/Sentence Trace Nodes ─────────────────────────────────────
async fn build_text_trace_nodes(pool: &PgPool, file_uuid: &str) -> Result<usize> {
async fn build_text_region_nodes(pool: &PgPool, file_uuid: &str) -> Result<usize> {
let chunk_table = t("chunk");
let nodes_table = t("tkg_nodes");
@@ -2827,14 +2927,14 @@ async fn build_lip_sync_edges(
let edges_table = t("tkg_edges");
// Get lip traces
let lip_traces: Vec<(i64, String, i64, i64, i64, f64)> = sqlx::query_as(&format!(
let lip_tracks: Vec<(i64, String, i64, i64, i64, f64)> = sqlx::query_as(&format!(
r#"
SELECT id::bigint, external_id,
(properties->>'start_frame')::bigint,
(properties->>'end_frame')::bigint,
(properties->>'speaking_frames')::bigint,
(properties->>'avg_openness')::float8
FROM {} WHERE file_uuid = $1 AND node_type = 'lip_trace'
FROM {} WHERE file_uuid = $1 AND node_type = 'lip_track'
"#,
nodes_table
))
@@ -2843,13 +2943,13 @@ async fn build_lip_sync_edges(
.await?;
// Get text traces
let text_traces: Vec<(i64, String, i64, i64, Option<String>)> = sqlx::query_as(&format!(
let text_regions: Vec<(i64, String, i64, i64, Option<String>)> = sqlx::query_as(&format!(
r#"
SELECT id::bigint, external_id,
(properties->>'start_frame')::bigint,
(properties->>'end_frame')::bigint,
properties->>'speaker_id'
FROM {} WHERE file_uuid = $1 AND node_type = 'text_trace'
FROM {} WHERE file_uuid = $1 AND node_type = 'text_region'
"#,
nodes_table
))
@@ -2860,8 +2960,8 @@ async fn build_lip_sync_edges(
let mut edge_count = 0;
let mut node_id_cache: HashMap<String, i64> = HashMap::new();
for (lip_id, lip_ext, lip_start, lip_end, lip_speaking, lip_openness) in &lip_traces {
for (text_id, text_ext, text_start, text_end, speaker_id) in &text_traces {
for (lip_id, lip_ext, lip_start, lip_end, lip_speaking, lip_openness) in &lip_tracks {
for (text_id, text_ext, text_start, text_end, speaker_id) in &text_regions {
// Check time overlap
let overlap_start = lip_start.max(text_start);
let overlap_end = lip_end.min(text_end);
@@ -2887,7 +2987,7 @@ async fn build_lip_sync_edges(
Some(id) => *id,
None => {
if let Some((id,)) = sqlx::query_as::<_, (i64,)>(&format!(
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='lip_trace' AND external_id=$2",
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='lip_track' AND external_id=$2",
nodes_table
))
.bind(file_uuid).bind(lip_ext).fetch_optional(pool).await?
@@ -2902,7 +3002,7 @@ async fn build_lip_sync_edges(
Some(id) => *id,
None => {
if let Some((id,)) = sqlx::query_as::<_, (i64,)>(&format!(
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='text_trace' AND external_id=$2",
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='text_region' AND external_id=$2",
nodes_table
))
.bind(file_uuid).bind(text_ext).fetch_optional(pool).await?
@@ -3245,7 +3345,7 @@ async fn build_has_appearance_edges(pool: &PgPool, file_uuid: &str) -> Result<us
let nodes_table = t("tkg_nodes");
let edges_table = t("tkg_edges");
// Match appearance_trace to face_trace via trace_id
// Match appearance_trace to face_track via trace_id
let appearance_traces: Vec<(i64, String, Option<i64>)> = sqlx::query_as(&format!(
r#"
SELECT id::bigint, external_id,
@@ -3263,14 +3363,14 @@ async fn build_has_appearance_edges(pool: &PgPool, file_uuid: &str) -> Result<us
for (app_id, app_ext, trace_id) in &appearance_traces {
if let Some(tid) = trace_id {
let face_ext = format!("trace_{}", tid);
let face_ext = format!("face_track_{}", tid);
// Get face trace node ID
let face_node_id = match node_id_cache.get(&face_ext) {
Some(id) => *id,
None => {
if let Some((id,)) = sqlx::query_as::<_, (i64,)>(&format!(
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_trace' AND external_id=$2",
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_track' AND external_id=$2",
nodes_table
))
.bind(file_uuid).bind(&face_ext).fetch_optional(pool).await?
@@ -3636,10 +3736,10 @@ mod tests {
#[test]
fn test_tkg_result() {
let r = TkgResult {
face_trace_nodes: 5,
gaze_trace_nodes: 5,
lip_trace_nodes: 4,
text_trace_nodes: 20,
face_track_nodes: 5,
gaze_track_nodes: 5,
lip_track_nodes: 4,
text_region_nodes: 20,
appearance_trace_nodes: 3,
skin_tone_trace_nodes: 5,
accessory_nodes: 0,
@@ -3653,7 +3753,7 @@ mod tests {
has_appearance_edges: 3,
wears_edges: 0,
};
assert_eq!(r.face_trace_nodes, 5);
assert_eq!(r.face_track_nodes, 5);
assert_eq!(r.object_nodes, 10);
assert_eq!(r.speaker_nodes, 3);
}