fix: ASRX duplication, TKG edges, trace ingest, and add pipeline progress publishing

- ASRX handler no longer stores duplicate 'asr' pre_chunks
- Pre_chunks storage made idempotent (delete-before-insert)
- Rule 1 + trace_ingest changed to query 'asrx' not 'asr'
- Trace chunks removed (dynamic from TKG/Qdrant)
- TKG scroll_face_points fixed: trace_id >= 1 (not == 1)
- TKG AsrxSegmentEntry: start/end -> start_time/end_time (match ASRX JSON)
- Unregister error handling: log instead of silent discard
- Add publish_pipeline_progress calls at each pipeline stage
  (processors, rule1, face_trace, identity_agent, TKG, rule2, completion)

src/api/universal_search.rs

@@ -10,6 +10,7 @@ use axum::{
 };
 use serde::{Deserialize, Serialize};
 
+use crate::core::db::qdrant_db::QdrantDb;
 use crate::core::db::{schema, Database, PostgresDb};
 
 #[derive(Debug, Deserialize)]
@@ -590,76 +591,162 @@ async fn search_persons_internal(
     req: &UniversalSearchRequest,
 ) -> Result<Vec<SearchResult>, anyhow::Error> {
     let id_table = schema::table_name("identities");
-    let fd_table = schema::table_name("face_detections");
-    let mut sql = format!(
-        "SELECT i.id, i.uuid::text, i.name, COUNT(fd.id) AS appearance_count, \
-         MIN(fd.timestamp_secs) AS first_time, MAX(fd.timestamp_secs) AS last_time, \
-         fd.file_uuid \
-         FROM {} i JOIN {} fd ON fd.identity_id = i.id WHERE 1=1",
-        id_table, fd_table
+
+    // Query matching identities from PostgreSQL
+    let mut id_sql = format!(
+        "SELECT id, uuid::text, name FROM {} WHERE name IS NOT NULL",
+        id_table
     );
-
-    if let Some(uuid) = &req.file_uuid {
-        sql.push_str(&format!(
-            " AND fd.file_uuid = '{}'",
-            uuid.replace('\'', "''")
-        ));
-    }
-
     if !req.query.is_empty() {
         let q = req.query.replace('\'', "''");
-        sql.push_str(&format!(" AND i.name ILIKE '%{}%'", q));
+        id_sql.push_str(&format!(" AND name ILIKE '%{}%'", q));
+    }
+    id_sql.push_str(" ORDER BY name ASC");
+
+    let identities: Vec<(i32, String, Option<String>)> =
+        sqlx::query_as(&id_sql).fetch_all(db.pool()).await?;
+
+    if identities.is_empty() {
+        return Ok(Vec::new());
     }
 
-    sql.push_str(" GROUP BY i.id, i.uuid, i.name, fd.file_uuid");
-    sql.push_str(" ORDER BY appearance_count DESC");
-    sql.push_str(&format!(" LIMIT {}", req.page_size.unwrap_or(20)));
+    // For each identity, scroll _faces points from Qdrant and aggregate per file
+    let qdrant = QdrantDb::new();
+    let limit = req.page_size.unwrap_or(20);
 
-    let rows: Vec<(
-        i32,
-        String,
-        Option<String>,
-        i64,
-        Option<f64>,
-        Option<f64>,
-        String,
-    )> = sqlx::query_as(&sql).fetch_all(db.pool()).await?;
+    // Aggregate frame ranges per (identity_id, file_uuid)
+    use std::collections::HashMap;
+    let mut agg: HashMap<(i32, String), (i64, i64, i64)> = HashMap::new(); // (id, fu) -> (count, min_frame, max_frame)
 
-    let results: Vec<SearchResult> = rows
-        .into_iter()
-        .map(
-            |(
-                identity_id,
-                identity_uuid,
-                name,
-                appearance_count,
-                first_time,
-                last_time,
-                file_uuid,
-            )| {
-                let score = if !req.query.is_empty()
-                    && name.as_ref().map_or(false, |n| {
-                        n.to_lowercase().contains(&req.query.to_lowercase())
-                    }) {
-                    0.95
-                } else {
-                    0.5
-                };
+    for (id, _uuid, _name) in &identities {
+        let scroll_filter = serde_json::json!({
+            "must": [
+                {"key": "identity_id", "match": {"value": id}}
+            ]
+        });
 
-                SearchResult::Person {
-                    file_uuid: Some(file_uuid),
-                    identity_id,
-                    identity_uuid,
-                    name,
-                    appearance_count: appearance_count as i32,
-                    score,
-                    first_appearance_time: first_time,
-                    last_appearance_time: last_time,
+        let points = match qdrant
+            .scroll_all_points("_faces", scroll_filter, 1000)
+            .await
+        {
+            Ok(p) => p,
+            Err(e) => {
+                tracing::warn!("Qdrant scroll failed for identity {}: {}", id, e);
+                continue;
+            }
+        };
+
+        for point in &points {
+            let payload = &point["payload"];
+            let file_uuid = match payload["file_uuid"].as_str() {
+                Some(f) => f.to_string(),
+                None => continue,
+            };
+
+            // Apply file_uuid filter if specified
+            if let Some(ref filter_fu) = req.file_uuid {
+                if &file_uuid != filter_fu {
+                    continue;
                 }
-            },
-        )
+            }
+
+            let frame = payload["frame"].as_i64().unwrap_or(0);
+            let entry = agg
+                .entry((*id, file_uuid))
+                .or_insert((0, i64::MAX, i64::MIN));
+            entry.0 += 1;
+            if frame < entry.1 {
+                entry.1 = frame;
+            }
+            if frame > entry.2 {
+                entry.2 = frame;
+            }
+        }
+    }
+
+    // Cache FPS per file_uuid for frame→second conversion
+    use std::collections::HashSet;
+    let file_uuids: HashSet<&str> = agg.keys().map(|(_, fu)| fu.as_str()).collect();
+    let video_table = crate::core::db::schema::table_name("videos");
+    let mut fps_cache: HashMap<String, f64> = HashMap::new();
+    for fu in file_uuids {
+        let fps: f64 = sqlx::query_scalar(&format!(
+            "SELECT COALESCE(fps, 30.0) FROM {} WHERE file_uuid = $1",
+            video_table
+        ))
+        .bind(fu)
+        .fetch_optional(db.pool())
+        .await?
+        .unwrap_or(30.0);
+        fps_cache.insert(fu.to_string(), fps);
+    }
+
+    // Build results
+    let q_lower = req.query.to_lowercase();
+    let mut results: Vec<SearchResult> = identities
+        .iter()
+        .flat_map(|(id, uuid, name)| {
+            let name_str = name.as_deref().unwrap_or("");
+            let name_match = !req.query.is_empty() && name_str.to_lowercase().contains(&q_lower);
+            let score = if name_match { 0.95 } else { 0.5 };
+            // Yield entries for this identity's files
+            let files: Vec<String> = agg
+                .keys()
+                .filter(|(iid, _)| iid == id)
+                .map(|(_, fu)| fu.clone())
+                .collect();
+            if files.is_empty() {
+                vec![]
+            } else {
+                files
+                    .into_iter()
+                    .map(|fu| {
+                        let (count, min_fr, max_fr) = agg[&(*id, fu.clone())];
+                        let fps = fps_cache.get(&fu).copied().unwrap_or(30.0);
+                        let first = if min_fr == i64::MAX {
+                            None
+                        } else {
+                            Some(min_fr as f64 / fps)
+                        };
+                        let last = if max_fr == i64::MIN {
+                            None
+                        } else {
+                            Some(max_fr as f64 / fps)
+                        };
+                        SearchResult::Person {
+                            file_uuid: Some(fu),
+                            identity_id: *id,
+                            identity_uuid: uuid.clone(),
+                            name: name.clone(),
+                            appearance_count: count as i32,
+                            score,
+                            first_appearance_time: first,
+                            last_appearance_time: last,
+                        }
+                    })
+                    .collect::<Vec<_>>()
+            }
+        })
         .collect();
 
+    // Sort by appearance_count descending, then limit
+    results.sort_by(|a, b| {
+        let a_count = match a {
+            SearchResult::Person {
+                appearance_count, ..
+            } => *appearance_count,
+            _ => 0,
+        };
+        let b_count = match b {
+            SearchResult::Person {
+                appearance_count, ..
+            } => *appearance_count,
+            _ => 0,
+        };
+        b_count.cmp(&a_count)
+    });
+    results.truncate(limit);
+
     Ok(results)
 }
 
@@ -752,49 +839,105 @@ async fn search_persons_by_query(
     limit: usize,
 ) -> Result<Vec<PersonResult>, anyhow::Error> {
     let id_table = schema::table_name("identities");
-    let fd_table = schema::table_name("face_detections");
-    let mut sql = format!(
-        "SELECT i.id, i.uuid::text, i.name, COUNT(fd.id) AS appearance_count, \
-         MIN(fd.timestamp_secs) AS first_time, MAX(fd.timestamp_secs) AS last_time \
-         FROM {} i JOIN {} fd ON fd.identity_id = i.id \
-         WHERE fd.file_uuid = '{}'",
-        id_table,
-        fd_table,
-        file_uuid.replace('\'', "''")
-    );
 
+    // Query matching identities from PostgreSQL
+    let mut id_sql = format!(
+        "SELECT id, uuid::text, name FROM {} WHERE name IS NOT NULL",
+        id_table
+    );
     if let Some(q) = query {
         let safe = q.replace('\'', "''");
-        sql.push_str(&format!(" AND i.name ILIKE '%{}%'", safe));
+        id_sql.push_str(&format!(" AND name ILIKE '%{}%'", safe));
+    }
+    id_sql.push_str(" ORDER BY name ASC");
+
+    let identities: Vec<(i32, String, Option<String>)> =
+        sqlx::query_as(&id_sql).fetch_all(db.pool()).await?;
+
+    if identities.is_empty() {
+        return Ok(Vec::new());
     }
 
-    sql.push_str(" GROUP BY i.id, i.uuid, i.name");
+    // For each identity, scroll _faces points from Qdrant and aggregate
+    let qdrant = QdrantDb::new();
+    let mut results: Vec<PersonResult> = Vec::new();
 
-    if let Some(min) = min_appearances {
-        sql.push_str(&format!(" HAVING COUNT(fd.id) >= {}", min));
+    for (id, uuid, name) in &identities {
+        let scroll_filter = serde_json::json!({
+            "must": [
+                {"key": "identity_id", "match": {"value": id}},
+                {"key": "file_uuid", "match": {"value": file_uuid}}
+            ]
+        });
+
+        let points = match qdrant
+            .scroll_all_points("_faces", scroll_filter, 1000)
+            .await
+        {
+            Ok(p) => p,
+            Err(e) => {
+                tracing::warn!("Qdrant scroll failed for identity {}: {}", id, e);
+                continue;
+            }
+        };
+
+        if points.is_empty() {
+            continue;
+        }
+
+        let count = points.len() as i64;
+        if let Some(min) = min_appearances {
+            if (count as i32) < min {
+                continue;
+            }
+        }
+
+        let min_frame = points
+            .iter()
+            .filter_map(|p| p["payload"]["frame"].as_i64())
+            .min()
+            .unwrap_or(0);
+        let max_frame = points
+            .iter()
+            .filter_map(|p| p["payload"]["frame"].as_i64())
+            .max()
+            .unwrap_or(0);
+
+        // Look up FPS for this file
+        let video_table = crate::core::db::schema::table_name("videos");
+        let fps: f64 = sqlx::query_scalar(&format!(
+            "SELECT COALESCE(fps, 30.0) FROM {} WHERE file_uuid = $1",
+            video_table
+        ))
+        .bind(file_uuid)
+        .fetch_optional(db.pool())
+        .await?
+        .unwrap_or(30.0);
+
+        let first_time = if fps > 0.0 {
+            Some(min_frame as f64 / fps)
+        } else {
+            None
+        };
+        let last_time = if fps > 0.0 {
+            Some(max_frame as f64 / fps)
+        } else {
+            None
+        };
+
+        results.push(PersonResult {
+            identity_id: *id,
+            identity_uuid: uuid.clone(),
+            name: name.clone(),
+            appearance_count: count as i32,
+            first_appearance_time: first_time,
+            last_appearance_time: last_time,
+        });
     }
 
-    sql.push_str(" ORDER BY appearance_count DESC");
-    sql.push_str(&format!(" LIMIT {}", limit));
-
-    let rows: Vec<(i32, String, Option<String>, i64, Option<f64>, Option<f64>)> =
-        sqlx::query_as(&sql).fetch_all(db.pool()).await?;
-
-    let results: Vec<PersonResult> = rows
-        .into_iter()
-        .map(
-            |(identity_id, identity_uuid, name, appearance_count, first_time, last_time)| {
-                PersonResult {
-                    identity_id,
-                    identity_uuid,
-                    name,
-                    appearance_count: appearance_count as i32,
-                    first_appearance_time: first_time,
-                    last_appearance_time: last_time,
-                }
-            },
-        )
-        .collect();
+    // Sort by appearance_count descending, then limit
+    results.sort_by(|a, b| b.appearance_count.cmp(&a.appearance_count));
+    results.truncate(limit);
 
     Ok(results)
 }