warren/momentry_core
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

release: v1.3.0 - TKG node type renaming

Browse Source 
Changes:
- Rust: face_trace → face_track (45 occurrences in 8 files)
- Rust: gaze_trace → gaze_track, lip_trace → lip_track
- Python: tkg_builder.py unified + pipeline_checklist.py fixed
- Swift: swift_hand.swift hand state detection (empty vs holding)

Node type changes:
  face_trace    → face_track
  person_trace  → body_track
  gaze_trace    → gaze_track
  lip_trace     → lip_track
  hand_trace    → hand_track
  speaker       → speaker_segment
  object        → detected_object
  text_trace    → text_region

Migration:
  PUBLIC schema: 12970 + 892 + 305 rows updated
This commit is contained in:
Accusys
2026-06-22 07:18:21 +08:00
parent bce9435823
commit 7e548f8b08
35 changed files with 2789 additions and 481 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/api/identity_agent_api.rs
+66 -63
View File
@@ -200,7 +200,7 @@ async fn match_from_photo(
// 4. Find best matching trace (highest similarity, no threshold)
let fd_table = schema::table_name("face_detections");
let best_match: Option<(i32, i32, f64)> = sqlx::query_as(&format!(
r#"SELECT id, trace_id,
r#"SELECT id, face_track_id,
1 - (embedding::vector <=> $1::vector) as similarity
FROM {}
WHERE file_uuid = $2 AND embedding IS NOT NULL
@@ -242,7 +242,7 @@ async fn match_from_photo(
matches: 1,
traces_matched,
message: format!(
"Best trace: trace_id={}, similarity={:.4}",
"Best trace: face_track_id={}, similarity={:.4}",
fb_trace, fb_sim
),
}))
@@ -276,7 +276,7 @@ async fn match_from_trace(
let fd_table = schema::table_name("face_detections");
let all_faces: Vec<(Vec<f32>, i64)> = sqlx::query_as::<_, (Vec<f32>, i64)>(&format!(
"SELECT embedding, frame_number FROM {} \
WHERE file_uuid = $1 AND trace_id = $2 AND embedding IS NOT NULL \
WHERE file_uuid = $1 AND face_track_id = $2 AND embedding IS NOT NULL \
ORDER BY frame_number ASC",
fd_table
))
@@ -313,7 +313,7 @@ async fn match_from_trace(
// Get width*height info if available (not all pipelines store it)
let face_sizes: Vec<(i64, i32)> = sqlx::query_as::<_, (i64, i32)>(&format!(
"SELECT frame_number, COALESCE(width, 0) * COALESCE(height, 0) AS area \
FROM {} WHERE file_uuid = $1 AND trace_id = $2 AND embedding IS NOT NULL \
FROM {} WHERE file_uuid = $1 AND face_track_id = $2 AND embedding IS NOT NULL \
ORDER BY frame_number ASC",
fd_table
))
@@ -352,7 +352,7 @@ async fn match_from_trace(
for qemb in &query_embeddings {
let top = sqlx::query_as::<_, (i32, i32, f64)>(&format!(
r#"SELECT id, trace_id,
r#"SELECT id, face_track_id,
1 - (embedding::vector <=> $1::vector) as similarity
FROM {}
WHERE file_uuid = $2
@@ -374,9 +374,9 @@ async fn match_from_trace(
)
})?;
if let Some((cface_id, c_trace_id, c_sim)) = top {
if seen_trace_ids.insert(c_trace_id) {
validated.push((cface_id, c_trace_id, c_sim));
if let Some((cface_id, c_face_track_id, c_sim)) = top {
if seen_trace_ids.insert(c_face_track_id) {
validated.push((cface_id, c_face_track_id, c_sim));
}
}
}
@@ -411,7 +411,7 @@ async fn match_from_trace(
// 4. Update matched face_detections
let mut traces_matched: Vec<i32> = Vec::new();
for (id, trace_id, _similarity) in &validated {
for (id, face_track_id, _similarity) in &validated {
if let Err(e) = sqlx::query(&format!(
"UPDATE {} SET identity_id = $1 WHERE id = $2",
fd_table
@@ -427,15 +427,15 @@ async fn match_from_trace(
e
);
} else {
if !traces_matched.contains(trace_id) {
traces_matched.push(*trace_id);
if !traces_matched.contains(face_track_id) {
traces_matched.push(*face_track_id);
}
}
}
// 5. Also bind the source trace itself
let _ = sqlx::query(&format!(
"UPDATE {} SET identity_id = $1 WHERE file_uuid = $2 AND trace_id = $3",
"UPDATE {} SET identity_id = $1 WHERE file_uuid = $2 AND face_track_id = $3",
fd_table
))
.bind(identity_id)
@@ -452,7 +452,7 @@ async fn match_from_trace(
let _ = crate::core::identity::storage::save_identity_file(&*state.db, &uuid_clean).await;
let match_count = validated.len() + 1;
let trace_count = traces_matched.len();
let face_track_count = traces_matched.len();
Ok(Json(MatchFromPhotoResponse {
success: true,
identity_uuid: uuid_clean,
@@ -461,7 +461,7 @@ async fn match_from_trace(
traces_matched,
message: format!(
"Matched {} faces ({} unique traces)",
match_count, trace_count
match_count, face_track_count
),
}))
}
@@ -647,22 +647,25 @@ async fn match_faces_iterative(pool: &sqlx::PgPool, file_uuid: &str) -> anyhow::
let qdrant_embeddings = face_db.get_all_embeddings_for_file(file_uuid).await?;
if qdrant_embeddings.is_empty() {
tracing::warn!("[FaceMatch-Qdrant] No face embeddings in Qdrant for {}", file_uuid);
tracing::warn!(
"[FaceMatch-Qdrant] No face embeddings in Qdrant for {}",
file_uuid
);
return match_faces_iterative_pg(pool, file_uuid).await; // Fallback to PG
}
// Group: trace_id → Vec<(frame, embedding)>
let mut trace_faces_raw: HashMap<i32, Vec<(i64, Vec<f32>)>> = HashMap::new();
let mut face_track_faces_raw: HashMap<i32, Vec<(i64, Vec<f32>)>> = HashMap::new();
for (_, emb, payload) in &qdrant_embeddings {
trace_faces_raw
face_track_faces_raw
.entry(payload.trace_id)
.or_default()
.push((payload.frame, emb.clone()));
}
// Sample 3 embeddings per trace (front, mid, back)
let mut trace_samples: HashMap<i32, Vec<Vec<f32>>> = HashMap::new();
for (tid, mut faces) in trace_faces_raw {
let mut face_track_samples: HashMap<i32, Vec<Vec<f32>>> = HashMap::new();
for (tid, mut faces) in face_track_faces_raw {
faces.sort_by_key(|(frame, _)| *frame);
let n = faces.len();
let indices = if n <= 3 {
@@ -671,11 +674,11 @@ async fn match_faces_iterative(pool: &sqlx::PgPool, file_uuid: &str) -> anyhow::
vec![0, n / 2, n - 1]
};
let samples: Vec<Vec<f32>> = indices.iter().map(|&i| faces[i].1.clone()).collect();
trace_samples.insert(tid, samples);
face_track_samples.insert(tid, samples);
}
let total_traces = trace_samples.len();
let sample_count: usize = trace_samples.values().map(|v| v.len()).sum();
let total_traces = face_track_samples.len();
let sample_count: usize = face_track_samples.values().map(|v| v.len()).sum();
tracing::info!(
"[FaceMatch-Qdrant] Loaded {} traces, sampled {} embeddings",
total_traces,
@@ -687,7 +690,7 @@ async fn match_faces_iterative(pool: &sqlx::PgPool, file_uuid: &str) -> anyhow::
let tmdb_seeds: Vec<(i32, String, Vec<f32>)> = tmdb_rows;
let mut matched: HashMap<i32, String> = HashMap::new();
for (&tid, samples) in &trace_samples {
for (&tid, samples) in &face_track_samples {
let mut best_name = String::new();
let mut best_sim = 0.0f32;
for (_, ref name, ref tmdb_emb) in &tmdb_seeds {
@@ -711,19 +714,19 @@ async fn match_faces_iterative(pool: &sqlx::PgPool, file_uuid: &str) -> anyhow::
// Round 2+: Propagate
let mut round = 2;
while matched.len() < trace_samples.len() {
while matched.len() < face_track_samples.len() {
let prev_count = matched.len();
// Collect new matches in separate HashMap
let mut new_matches: HashMap<i32, String> = HashMap::new();
for (&tid, samples) in &trace_samples {
for (&tid, samples) in &face_track_samples {
if matched.contains_key(&tid) {
continue;
}
for (matched_tid, matched_name) in &matched {
if let Some(matched_embs) = trace_samples.get(matched_tid) {
if let Some(matched_embs) = face_track_samples.get(matched_tid) {
for face_emb in samples {
for ref_emb in matched_embs {
let s = cosine_similarity(face_emb, ref_emb);
@@ -776,7 +779,7 @@ async fn match_faces_iterative(pool: &sqlx::PgPool, file_uuid: &str) -> anyhow::
let identity_id = identities_map.get(name);
if let Some(id) = identity_id {
let rows = sqlx::query(&format!(
"UPDATE {} SET identity_id = $1 WHERE file_uuid = $2 AND trace_id = $3",
"UPDATE {} SET identity_id = $1 WHERE file_uuid = $2 AND face_track_id = $3",
fd_table
))
.bind(*id)
@@ -788,13 +791,13 @@ async fn match_faces_iterative(pool: &sqlx::PgPool, file_uuid: &str) -> anyhow::
updated += rows as usize;
// Phase 3: Also update TKG node
let external_id = format!("trace_{}", tid);
let external_id = format!("face_track_{}", tid);
let identity_name = identity_names.get(id);
let _ = sqlx::query(&format!(
"UPDATE {} SET properties = jsonb_set(\
jsonb_set(properties, '{{identity_id}}', $1::jsonb, false),\
'{{identity_name}}', $2::jsonb, false)\
WHERE file_uuid = $3 AND node_type = 'face_trace' AND external_id = $4",
WHERE file_uuid = $3 AND node_type = 'face_track' AND external_id = $4",
nodes_table
))
.bind(*id)
@@ -828,12 +831,12 @@ async fn match_faces_iterative_pg(pool: &sqlx::PgPool, file_uuid: &str) -> anyho
tmdb_rows.len()
);
// Step 2: 載入所有 face_detections（含 frame_number），按 trace_id 分組
// Step 2: 載入所有 face_detections（含 frame_number），按 face_track_id 分組
let fd_table = schema::table_name("face_detections");
let fd_rows = sqlx::query_as::<_, (i32, i64, Vec<f32>)>(&format!(
"SELECT trace_id, frame_number, embedding FROM {} \
WHERE file_uuid=$1 AND trace_id IS NOT NULL AND embedding IS NOT NULL \
ORDER BY trace_id, frame_number",
"SELECT face_track_id, frame_number, embedding FROM {} \
WHERE file_uuid=$1 AND face_track_id IS NOT NULL AND embedding IS NOT NULL \
ORDER BY face_track_id, frame_number",
fd_table
))
.bind(file_uuid)
@@ -845,19 +848,19 @@ async fn match_faces_iterative_pg(pool: &sqlx::PgPool, file_uuid: &str) -> anyho
return Ok(0);
}
// 分組：trace_id → (frame_number, embedding)
// 分組：face_track_id → (frame_number, embedding)
use std::collections::HashMap;
let mut trace_faces_raw: HashMap<i32, Vec<(i64, Vec<f32>)>> = HashMap::new();
let mut face_track_faces_raw: HashMap<i32, Vec<(i64, Vec<f32>)>> = HashMap::new();
for (tid, frame, emb) in &fd_rows {
trace_faces_raw
face_track_faces_raw
.entry(*tid)
.or_insert_with(Vec::new)
.push((*frame, emb.clone()));
}
// 從每個 trace 選取不同角度的 3 個 face embedding
let mut trace_samples: HashMap<i32, Vec<Vec<f32>>> = HashMap::new();
for (tid, mut faces) in trace_faces_raw {
let mut face_track_samples: HashMap<i32, Vec<Vec<f32>>> = HashMap::new();
for (tid, mut faces) in face_track_faces_raw {
faces.sort_by_key(|(frame, _)| *frame);
let n = faces.len();
let indices = if n <= 3 {
@@ -867,11 +870,11 @@ async fn match_faces_iterative_pg(pool: &sqlx::PgPool, file_uuid: &str) -> anyho
vec![0, mid, n - 1]
};
let samples: Vec<Vec<f32>> = indices.iter().map(|&i| faces[i].1.clone()).collect();
trace_samples.insert(tid, samples);
face_track_samples.insert(tid, samples);
}
let total_traces = trace_samples.len();
let sample_count: usize = trace_samples.values().map(|v| v.len()).sum();
let total_traces = face_track_samples.len();
let sample_count: usize = face_track_samples.values().map(|v| v.len()).sum();
tracing::info!(
"[FaceMatch-PG] Loaded {} traces, sampled {} embeddings (3-angle)",
total_traces,
@@ -883,10 +886,10 @@ async fn match_faces_iterative_pg(pool: &sqlx::PgPool, file_uuid: &str) -> anyho
// Step 4: 迭代匹配
const TH: f32 = 0.50;
let mut matched: HashMap<i32, String> = HashMap::new(); // trace_id → identity_name
let mut matched: HashMap<i32, String> = HashMap::new(); // face_track_id → identity_name
// Round 1: 用 3-angle samples 比對 TMDb
for (&tid, samples) in &trace_samples {
for (&tid, samples) in &face_track_samples {
let mut best_name = String::new();
let mut best_sim = 0.0f32;
for (_, ref name, ref tmdb_emb) in &tmdb_seeds {
@@ -924,7 +927,7 @@ async fn match_faces_iterative_pg(pool: &sqlx::PgPool, file_uuid: &str) -> anyho
.await?;
if let Some(identity_id) = id_opt {
let _ = sqlx::query(&format!(
"UPDATE {} SET identity_id=$1 WHERE file_uuid=$2 AND trace_id=$3",
"UPDATE {} SET identity_id=$1 WHERE file_uuid=$2 AND face_track_id=$3",
fd_table
))
.bind(identity_id)
@@ -934,12 +937,12 @@ async fn match_faces_iterative_pg(pool: &sqlx::PgPool, file_uuid: &str) -> anyho
.await;
// Phase 3: Also update TKG node
let external_id = format!("trace_{}", tid);
let external_id = format!("face_track_{}", tid);
let _ = sqlx::query(&format!(
"UPDATE {} SET properties = jsonb_set(\
jsonb_set(properties, '{{identity_id}}', $1::jsonb, false),\
'{{identity_name}}', $2::jsonb, false)\
WHERE file_uuid = $3 AND node_type = 'face_trace' AND external_id = $4",
WHERE file_uuid = $3 AND node_type = 'face_track' AND external_id = $4",
nodes_table
))
.bind(identity_id)
@@ -961,7 +964,7 @@ async fn match_faces_iterative_pg(pool: &sqlx::PgPool, file_uuid: &str) -> anyho
// 建立 seed pool: name → Vec<embedding>
let mut seed_pool: HashMap<String, Vec<&Vec<f32>>> = HashMap::new();
for (&tid, name) in &matched {
if let Some(samples) = trace_samples.get(&tid) {
if let Some(samples) = face_track_samples.get(&tid) {
seed_pool
.entry(name.clone())
.or_default()
@@ -970,7 +973,7 @@ async fn match_faces_iterative_pg(pool: &sqlx::PgPool, file_uuid: &str) -> anyho
}
let mut new_matches: Vec<(i32, String)> = Vec::new();
for (&tid, samples) in &trace_samples {
for (&tid, samples) in &face_track_samples {
if matched.contains_key(&tid) {
continue;
}
@@ -1014,11 +1017,11 @@ async fn match_faces_iterative_pg(pool: &sqlx::PgPool, file_uuid: &str) -> anyho
// Step 6: 未匹配的 trace 設 stranger_id = strangers.id (FK)
// First: ensure strangers records exist
let _ = sqlx::query(&format!(
"INSERT INTO {} (file_uuid, trace_id) \
SELECT $1, fd.trace_id FROM {} fd \
WHERE fd.file_uuid = $1 AND fd.trace_id IS NOT NULL \
"INSERT INTO {} (file_uuid, face_track_id) \
SELECT $1, fd.face_track_id FROM {} fd \
WHERE fd.file_uuid = $1 AND fd.face_track_id IS NOT NULL \
AND fd.identity_id IS NULL \
ON CONFLICT (file_uuid, trace_id) DO NOTHING",
ON CONFLICT (file_uuid, face_track_id) DO NOTHING",
strangers_table, fd_table
))
.bind(file_uuid)
@@ -1029,9 +1032,9 @@ async fn match_faces_iterative_pg(pool: &sqlx::PgPool, file_uuid: &str) -> anyho
let stranger_update = sqlx::query(&format!(
"UPDATE {} fd SET stranger_id = s.id \
FROM {} s \
WHERE s.file_uuid = fd.file_uuid AND s.trace_id = fd.trace_id \
WHERE s.file_uuid = fd.file_uuid AND s.face_track_id = fd.face_track_id \
AND fd.file_uuid = $1 AND fd.identity_id IS NULL \
AND fd.trace_id IS NOT NULL AND fd.stranger_id IS NULL",
AND fd.face_track_id IS NOT NULL AND fd.stranger_id IS NULL",
fd_table, strangers_table
))
.bind(file_uuid)
@@ -1069,16 +1072,16 @@ async fn match_faces_iterative_pg(pool: &sqlx::PgPool, file_uuid: &str) -> anyho
}
/// Bind ASRX speakers to face traces based on temporal overlap.
/// Reads face_detections (trace_id, identity_id, frame_number) and ASRX
/// Reads face_detections (face_track_id, identity_id, frame_number) and ASRX
/// segments (speaker_id, start_time, end_time), computes overlap,
/// and stores bindings in identity_bindings table.
pub async fn bind_speakers(pool: &sqlx::PgPool, file_uuid: &str) -> anyhow::Result<usize> {
// Load face traces with identity_id and frame numbers
let fd_table = schema::table_name("face_detections");
let traces = sqlx::query_as::<_, (i32, Vec<i32>)>(&format!(
"SELECT trace_id, array_agg(frame_number ORDER BY frame_number) \
FROM {} WHERE file_uuid=$1 AND trace_id IS NOT NULL AND identity_id IS NOT NULL \
GROUP BY trace_id",
"SELECT face_track_id, array_agg(frame_number ORDER BY frame_number) \
FROM {} WHERE file_uuid=$1 AND face_track_id IS NOT NULL AND identity_id IS NOT NULL \
GROUP BY face_track_id",
fd_table
))
.bind(file_uuid)
@@ -1141,7 +1144,7 @@ pub async fn bind_speakers(pool: &sqlx::PgPool, file_uuid: &str) -> anyhow::Resu
// For each trace, compute overlap with each speaker
let mut bindings = 0usize;
for (trace_id, frames) in &traces {
for (face_track_id, frames) in &traces {
if frames.is_empty() {
continue;
}
@@ -1149,9 +1152,9 @@ pub async fn bind_speakers(pool: &sqlx::PgPool, file_uuid: &str) -> anyhow::Resu
// Get identity_id for this trace
let fd_table = schema::table_name("face_detections");
let identity_id: Option<i32> = sqlx::query_scalar(
&format!("SELECT identity_id FROM {} WHERE file_uuid=$1 AND trace_id=$2 AND identity_id IS NOT NULL LIMIT 1", fd_table)
&format!("SELECT identity_id FROM {} WHERE file_uuid=$1 AND face_track_id=$2 AND identity_id IS NOT NULL LIMIT 1", fd_table)
)
.bind(file_uuid).bind(trace_id)
.bind(file_uuid).bind(face_track_id)
.fetch_optional(pool).await?.flatten();
if identity_id.is_none() {
@@ -1184,7 +1187,7 @@ pub async fn bind_speakers(pool: &sqlx::PgPool, file_uuid: &str) -> anyhow::Resu
let overlap_ratio = best_overlap as f64 / frames.len() as f64;
if overlap_ratio > 0.3 && !best_speaker.is_empty() {
let metadata = serde_json::json!({
"trace_id": trace_id,
"trace_id": face_track_id,
"overlap_frames": best_overlap,
"total_frames": frames.len(),
"overlap_ratio": overlap_ratio,
@@ -1278,7 +1281,7 @@ pub async fn run_identity_agent(db: &PostgresDb, file_uuid: &str) -> anyhow::Res
"reasoning": identities[0].reasoning,
});
let _ = sqlx::query(&format!(
"INSERT INTO {} (file_uuid, trace_id, metadata) \
"INSERT INTO {} (file_uuid, face_track_id, metadata) \
VALUES ($1, NULL, $2::jsonb) ON CONFLICT DO NOTHING",
schema::table_name("strangers")
))