fix: ASRX duplication, TKG edges, trace ingest, and add pipeline progress publishing

- ASRX handler no longer stores duplicate 'asr' pre_chunks
- Pre_chunks storage made idempotent (delete-before-insert)
- Rule 1 + trace_ingest changed to query 'asrx' not 'asr'
- Trace chunks removed (dynamic from TKG/Qdrant)
- TKG scroll_face_points fixed: trace_id >= 1 (not == 1)
- TKG AsrxSegmentEntry: start/end -> start_time/end_time (match ASRX JSON)
- Unregister error handling: log instead of silent discard
- Add publish_pipeline_progress calls at each pipeline stage
  (processors, rule1, face_trace, identity_agent, TKG, rule2, completion)

src/core/identity/storage.rs

@@ -186,8 +186,11 @@ pub fn rebuild_index() -> Result<usize> {
 }
 
 pub async fn save_identity_file_by_pool(pool: &sqlx::PgPool, uuid: &str) -> Result<()> {
+    use crate::core::db::QdrantDb;
+    use serde_json::json;
+    use std::collections::{HashMap, HashSet};
+
     let identity_table = crate::core::db::schema::table_name("identities");
-    let fd_table = crate::core::db::schema::table_name("face_detections");
 
     let clean = uuid.replace('-', "");
 
@@ -195,7 +198,7 @@ pub async fn save_identity_file_by_pool(pool: &sqlx::PgPool, uuid: &str) -> Resu
         &format!(
             "SELECT id::bigint, uuid::text, name, identity_type, source, status, metadata, COALESCE(reference_data, '{{}}'::jsonb) as reference_data, \
               NULL::real[] as voice_embedding, NULL::real[] as identity_embedding, \
-              face_embedding::real[] as face_embedding, \
+              NULL::real[] as face_embedding, \
               tmdb_id, tmdb_profile, created_at::timestamptz as created_at, NULL::timestamptz as updated_at \
               FROM {} WHERE REPLACE(uuid::text, '-', '') = $1",
             identity_table
@@ -207,24 +210,45 @@ pub async fn save_identity_file_by_pool(pool: &sqlx::PgPool, uuid: &str) -> Resu
     .with_context(|| format!("Identity not found in DB: {}", uuid))?;
 
     let identity_uuid = record.uuid.clone();
+    let identity_id = record.id;
 
-    let binding_rows = sqlx::query_as::<_, (String, Vec<i32>, i64)>(
-        &format!(
-            "SELECT fd.file_uuid, COALESCE(array_agg(DISTINCT fd.trace_id) FILTER (WHERE fd.trace_id IS NOT NULL), '{{}}'::int[]), COUNT(*)::bigint \
-             FROM {} fd WHERE fd.identity_id = $1 GROUP BY fd.file_uuid ORDER BY fd.file_uuid",
-            fd_table
-        )
-    )
-    .bind(record.id)
-    .fetch_all(pool)
-    .await?;
+    // Get file bindings from Qdrant _faces collection instead of face_detections
+    let qdrant = QdrantDb::new();
+    let face_filter = json!({
+        "must": [
+            {"key": "identity_id", "match": {"value": identity_id}}
+        ]
+    });
+    let face_points = qdrant
+        .scroll_all_points("_faces", face_filter, 500)
+        .await
+        .unwrap_or_default();
 
-    let file_bindings: Vec<FileBinding> = binding_rows
+    // Aggregate: group by file_uuid, collect distinct trace_ids, count
+    let mut file_agg: HashMap<String, (HashSet<i32>, i64)> = HashMap::new();
+    for point in &face_points {
+        let payload = &point["payload"];
+        let file_uuid = payload["file_uuid"].as_str().unwrap_or("").to_string();
+        let trace_id = payload["trace_id"].as_i64().unwrap_or(0) as i32;
+        if file_uuid.is_empty() {
+            continue;
+        }
+        let entry = file_agg.entry(file_uuid).or_default();
+        if trace_id > 0 {
+            entry.0.insert(trace_id);
+        }
+        entry.1 += 1;
+    }
+
+    let file_bindings: Vec<FileBinding> = file_agg
         .into_iter()
-        .map(|(fu, tids, cnt)| FileBinding {
-            file_uuid: fu,
-            trace_ids: tids,
-            face_count: cnt,
+        .map(|(fu, (tids, cnt))| {
+            let trace_ids: Vec<i32> = tids.into_iter().collect();
+            FileBinding {
+                file_uuid: fu,
+                trace_ids,
+                face_count: cnt,
+            }
         })
         .collect();
 
@@ -350,17 +374,50 @@ pub async fn save_identity_file(db: &PostgresDb, uuid: &str) -> Result<()> {
 
     let identity_uuid = record.uuid.clone();
 
-    let binding_rows = sqlx::query_as::<_, (String, Vec<i32>, i64)>(
-        "SELECT fd.file_uuid, COALESCE(array_agg(DISTINCT fd.trace_id) FILTER (WHERE fd.trace_id IS NOT NULL), '{}'::int[]), COUNT(*)::bigint \
-         FROM face_detections fd \
-         WHERE fd.identity_id = $1 \
-         GROUP BY fd.file_uuid \
-         ORDER BY fd.file_uuid"
-    )
-    .bind(record.id)
-    .fetch_all(db.pool())
-    .await
-    .with_context(|| format!("Failed to query bindings for identity: {}", identity_uuid))?;
+    // Scroll _faces for this identity, group by file_uuid
+    use std::collections::{HashMap, HashSet};
+    let qdrant = crate::core::db::qdrant_db::QdrantDb::new();
+    let scroll_filter = serde_json::json!({
+        "must": [
+            {"key": "identity_id", "match": {"value": record.id}}
+        ]
+    });
+    let face_points = qdrant
+        .scroll_all_points("_faces", scroll_filter, 1000)
+        .await
+        .with_context(|| format!("Failed to scroll _faces for identity: {}", identity_uuid))?;
+
+    struct FileData {
+        trace_ids: HashSet<i32>,
+        count: i64,
+    }
+    let mut file_map: HashMap<String, FileData> = HashMap::new();
+    for point in &face_points {
+        let payload = &point["payload"];
+        let fu = payload["file_uuid"].as_str().unwrap_or("").to_string();
+        if fu.is_empty() {
+            continue;
+        }
+        let trace_id = payload["trace_id"].as_i64().unwrap_or(0) as i32;
+        let entry = file_map.entry(fu).or_insert(FileData {
+            trace_ids: HashSet::new(),
+            count: 0,
+        });
+        if trace_id > 0 {
+            entry.trace_ids.insert(trace_id);
+        }
+        entry.count += 1;
+    }
+
+    let mut binding_rows: Vec<(String, Vec<i32>, i64)> = file_map
+        .into_iter()
+        .map(|(fu, fd)| {
+            let mut tids: Vec<i32> = fd.trace_ids.into_iter().collect();
+            tids.sort();
+            (fu, tids, fd.count)
+        })
+        .collect();
+    binding_rows.sort_by(|a, b| a.0.cmp(&b.0));
 
     let file_bindings: Vec<FileBinding> = binding_rows
         .into_iter()