feat: progressive multi-round face matching + pending person API
- Identity agent: per-face max matching, multi-round with derived seeds from high-confidence faces, angle diversity filter (cosine sim < 0.90) - Pending person API: POST /file/:file_uuid/pending-person + GET /file/:file_uuid/pending-persons with status=pending, source=manual - Update API docs (07_identity.md)
This commit is contained in:
@@ -0,0 +1,75 @@
|
||||
use anyhow::{Context, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::Duration;
|
||||
|
||||
use super::executor::PythonExecutor;
|
||||
|
||||
const FACE_CLUSTER_TIMEOUT: Duration = Duration::from_secs(3600);
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
pub struct FaceClusterResult {
|
||||
pub clusters: Vec<FaceClusterInfo>,
|
||||
pub frames: Vec<FaceClusterFrame>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
pub struct FaceClusterInfo {
|
||||
pub cluster_id: String,
|
||||
pub face_count: usize,
|
||||
pub representative_face: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
pub struct FaceClusterFrame {
|
||||
pub frame: u64,
|
||||
pub timestamp: f64,
|
||||
pub faces: Vec<ClusteredFace>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
pub struct ClusteredFace {
|
||||
pub face_id: String,
|
||||
pub cluster_id: String,
|
||||
pub confidence: f32,
|
||||
}
|
||||
|
||||
pub async fn process_face_cluster(
|
||||
video_path: &str,
|
||||
output_path: &str,
|
||||
uuid: Option<&str>,
|
||||
frames: Option<&[i64]>,
|
||||
) -> Result<FaceClusterResult> {
|
||||
let executor = PythonExecutor::new()?;
|
||||
let script_path = executor.script_path("fast_face_clustering_processor.py");
|
||||
|
||||
tracing::info!("[FACE_CLUSTER] Starting face clustering: {}", video_path);
|
||||
|
||||
if !script_path.exists() {
|
||||
tracing::warn!("[FACE_CLUSTER] Script not found, returning empty result");
|
||||
return Ok(FaceClusterResult {
|
||||
clusters: vec![],
|
||||
frames: vec![],
|
||||
});
|
||||
}
|
||||
|
||||
executor
|
||||
.run_with_frames(
|
||||
"fast_face_clustering_processor.py",
|
||||
&[video_path, output_path],
|
||||
uuid,
|
||||
"FACE_CLUSTER",
|
||||
Some(FACE_CLUSTER_TIMEOUT),
|
||||
frames,
|
||||
)
|
||||
.await
|
||||
.with_context(|| format!("Failed to run face clustering script"))?;
|
||||
|
||||
let json_str = std::fs::read_to_string(output_path).context("Failed to read FACE_CLUSTER output")?;
|
||||
|
||||
let result: FaceClusterResult =
|
||||
serde_json::from_str(&json_str).context("Failed to parse FACE_CLUSTER output")?;
|
||||
|
||||
tracing::info!("[FACE_CLUSTER] Result: {} clusters, {} frames", result.clusters.len(), result.frames.len());
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
@@ -7,6 +7,7 @@ pub mod clip;
|
||||
pub mod cut;
|
||||
pub mod executor;
|
||||
pub mod face;
|
||||
pub mod face_clustering;
|
||||
pub mod face_recognition;
|
||||
pub mod hand;
|
||||
pub mod heuristic_scene;
|
||||
@@ -32,6 +33,9 @@ pub use clip::{
|
||||
pub use cut::{process_cut, CutResult, CutScene};
|
||||
pub use executor::{validate_python_env, PythonExecutor, RetryConfig};
|
||||
pub use face::{process_face, Face, FaceFrame, FaceResult};
|
||||
pub use face_clustering::{
|
||||
process_face_cluster, ClusteredFace, FaceClusterFrame, FaceClusterInfo, FaceClusterResult,
|
||||
};
|
||||
pub use face_recognition::{
|
||||
process_face_recognition, register_face, FaceAttributes, FaceCluster, FaceIdentity, FacePose,
|
||||
FaceRecognitionFrame, FaceRecognitionResult, FaceRegistrationResult, RecognizedFace,
|
||||
|
||||
+26
-25
@@ -129,7 +129,7 @@ async fn populate_face_embeddings_to_qdrant(
|
||||
// Load from face_detections table
|
||||
let fd_table = t("face_detections");
|
||||
let rows: Vec<(i32, i64, f64, f64, f64, f64, f64, Option<Vec<f32>>)> = sqlx::query_as(&format!(
|
||||
"SELECT trace_id::int, frame_number::bigint, x::float8, y::float8, width::float8, height::float8, confidence::float8, embedding \
|
||||
"SELECT trace_id::int, frame_number::bigint, x::float8, y::float8, width::float8, height::float8, confidence::float8, embedding::float4[] \
|
||||
FROM {} WHERE file_uuid = $1 AND trace_id IS NOT NULL AND embedding IS NOT NULL",
|
||||
fd_table
|
||||
))
|
||||
@@ -165,11 +165,20 @@ async fn populate_face_embeddings_to_qdrant(
|
||||
yaw,
|
||||
pitch,
|
||||
roll,
|
||||
identity_uuid: None,
|
||||
identity_ref: None,
|
||||
stranger_ref: None,
|
||||
r#type: None,
|
||||
};
|
||||
points.push((point_id, emb.clone(), payload));
|
||||
}
|
||||
}
|
||||
|
||||
info!(
|
||||
"[TKG-Phase1] Attempting to store {} face embeddings in Qdrant for {}",
|
||||
points.len(),
|
||||
file_uuid
|
||||
);
|
||||
let count = face_db.batch_upsert(points).await?;
|
||||
info!(
|
||||
"[TKG-Phase1] Stored {} face embeddings in Qdrant for {}",
|
||||
@@ -401,19 +410,7 @@ fn detect_mutual_gaze(
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct YoloJson {
|
||||
#[serde(default)]
|
||||
frames: Vec<YoloFrameData>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct YoloFrameData {
|
||||
#[serde(default)]
|
||||
frame: u32,
|
||||
#[serde(default)]
|
||||
timestamp: f64,
|
||||
#[serde(default)]
|
||||
detections: Vec<YoloDetEntry>,
|
||||
#[serde(default)]
|
||||
objects: Vec<YoloDetEntry>,
|
||||
frames: HashMap<String, YoloFrameEntry>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
@@ -1033,7 +1030,7 @@ async fn build_yolo_object_nodes(
|
||||
.with_context(|| format!("Failed to parse {:?}", yolo_path))?;
|
||||
|
||||
let mut class_counts: HashMap<String, i64> = HashMap::new();
|
||||
for fdata in &yolo.frames {
|
||||
for fdata in yolo.frames.values() {
|
||||
let dets = if !fdata.detections.is_empty() {
|
||||
&fdata.detections
|
||||
} else {
|
||||
@@ -1277,9 +1274,9 @@ async fn build_co_occurrence_edges_from_qdrant(
|
||||
|
||||
let mut edge_count = 0;
|
||||
for (frame, faces) in frame_faces.iter() {
|
||||
let yolo_frame = match yolo.frames.iter().find(|f| f.frame == *frame as u32) {
|
||||
Some(f) => f,
|
||||
None => continue,
|
||||
let yolo_frame = match yolo.frames.get(&frame.to_string()) {
|
||||
Some(f) => f,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let dets = if !yolo_frame.detections.is_empty() {
|
||||
@@ -1391,9 +1388,9 @@ async fn build_co_occurrence_edges_from_pg(
|
||||
|
||||
let mut edge_count = 0;
|
||||
for face in &face_rows {
|
||||
let yolo_frame = match yolo.frames.iter().find(|f| f.frame == face.frame_number as u32) {
|
||||
Some(f) => f,
|
||||
None => continue,
|
||||
let yolo_frame = match yolo.frames.get(&face.frame_number.to_string()) {
|
||||
Some(f) => f,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let dets = if !yolo_frame.detections.is_empty() {
|
||||
@@ -2411,7 +2408,9 @@ async fn build_gaze_track_nodes_from_face_json(
|
||||
let nodes_table = t("tkg_nodes");
|
||||
sqlx::query(&format!(
|
||||
"INSERT INTO {} (file_uuid, external_id, label, node_type, properties, created_at) \
|
||||
VALUES ($1, $2, $3, 'gaze_track', $4, NOW())",
|
||||
VALUES ($1, $2, $3, 'gaze_track', $4, NOW()) \
|
||||
ON CONFLICT (file_uuid, node_type, external_id) \
|
||||
DO UPDATE SET properties = COALESCE(EXCLUDED.properties, tkg_nodes.properties)",
|
||||
nodes_table
|
||||
))
|
||||
.bind(file_uuid)
|
||||
@@ -3063,7 +3062,9 @@ async fn build_lip_track_nodes_from_face_json(
|
||||
let nodes_table = t("tkg_nodes");
|
||||
sqlx::query(&format!(
|
||||
"INSERT INTO {} (file_uuid, external_id, label, node_type, properties, created_at) \
|
||||
VALUES ($1, $2, $3, 'lip_track', $4, NOW())",
|
||||
VALUES ($1, $2, $3, 'lip_track', $4, NOW()) \
|
||||
ON CONFLICT (file_uuid, node_type, external_id) \
|
||||
DO UPDATE SET properties = COALESCE(EXCLUDED.properties, tkg_nodes.properties)",
|
||||
nodes_table
|
||||
))
|
||||
.bind(file_uuid)
|
||||
@@ -3814,10 +3815,10 @@ async fn build_hand_object_edges(pool: &PgPool, file_uuid: &str, output_dir: &st
|
||||
|
||||
let yolo_frames: HashMap<u64, &Vec<YoloDetEntry>> = yolo.frames
|
||||
.iter()
|
||||
.filter_map(|f| {
|
||||
.filter_map(|(frame_key, f)| {
|
||||
let objs = if !f.objects.is_empty() { &f.objects } else { &f.detections };
|
||||
if !objs.is_empty() {
|
||||
Some((f.frame as u64, objs))
|
||||
frame_key.parse::<u64>().ok().map(|n| (n, objs))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user