fix: ASRX duplication, TKG edges, trace ingest, and add pipeline progress publishing
- ASRX handler no longer stores duplicate 'asr' pre_chunks - Pre_chunks storage made idempotent (delete-before-insert) - Rule 1 + trace_ingest changed to query 'asrx' not 'asr' - Trace chunks removed (dynamic from TKG/Qdrant) - TKG scroll_face_points fixed: trace_id >= 1 (not == 1) - TKG AsrxSegmentEntry: start/end -> start_time/end_time (match ASRX JSON) - Unregister error handling: log instead of silent discard - Add publish_pipeline_progress calls at each pipeline stage (processors, rule1, face_trace, identity_agent, TKG, rule2, completion)
This commit is contained in:
Accusys
+158
-5
@@ -82,6 +82,10 @@ struct ProcessorOutput {
|
||||
total_frames: i32,
|
||||
retry_count: i32,
|
||||
pid: i32,
|
||||
asr_status: Option<crate::core::processor::AsrStatus>,
|
||||
segment_count: usize,
|
||||
face_status: Option<crate::core::processor::FaceStatus>,
|
||||
total_faces: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -316,13 +320,16 @@ impl ProcessorPool {
|
||||
}
|
||||
|
||||
// Subscribe to Redis progress pub/sub and update processor hash in real-time
|
||||
let sub_db = db.clone();
|
||||
let sub_redis = redis.clone();
|
||||
let sub_uuid = job.uuid.clone();
|
||||
let sub_processor = processor_name.clone();
|
||||
let progress_handle = tokio::spawn(async move {
|
||||
let cb_db = sub_db.clone();
|
||||
let cb_redis = sub_redis.clone();
|
||||
let cb_uuid = sub_uuid.clone();
|
||||
let cb_processor = sub_processor.clone();
|
||||
let last_update = std::cell::Cell::new(0i64);
|
||||
if let Err(e) = sub_redis
|
||||
.subscribe_and_callback(&sub_uuid, move |msg| {
|
||||
tracing::info!(
|
||||
@@ -338,6 +345,7 @@ impl ProcessorPool {
|
||||
let r = cb_redis.clone();
|
||||
let u = cb_uuid.clone();
|
||||
let p = cb_processor.clone();
|
||||
let p2 = p.clone();
|
||||
tokio::spawn(async move {
|
||||
match r
|
||||
.update_worker_processor_status(
|
||||
@@ -354,6 +362,46 @@ impl ProcessorPool {
|
||||
Err(e) => tracing::error!("[Subscriber] FAILED {}: {}", p, e),
|
||||
}
|
||||
});
|
||||
// Sync progress to PostgreSQL every 5 seconds
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs() as i64;
|
||||
let elapsed = now - last_update.get();
|
||||
if elapsed >= 5 {
|
||||
tracing::info!(
|
||||
"[Subscriber] PG sync {}: cur={} tot={} (elapsed={})",
|
||||
p2,
|
||||
cur,
|
||||
tot,
|
||||
elapsed
|
||||
);
|
||||
last_update.set(now);
|
||||
let db_client = cb_db.clone();
|
||||
let u = cb_uuid.clone();
|
||||
let p = cb_processor.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = db_client
|
||||
.update_processor_progress(
|
||||
&u, &p, cur as u64, tot as u64, "running",
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::error!(
|
||||
"[Subscriber] PG progress update FAILED {}: {}",
|
||||
p,
|
||||
e
|
||||
);
|
||||
} else {
|
||||
tracing::info!(
|
||||
"[Subscriber] PG progress updated {}: cur={} tot={}",
|
||||
p,
|
||||
cur,
|
||||
tot
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
@@ -400,6 +448,32 @@ impl ProcessorPool {
|
||||
error!("Failed to update processor result to completed: {}", e);
|
||||
}
|
||||
|
||||
if let Some(ref asr_status) = output.asr_status {
|
||||
if let Err(e) = db
|
||||
.update_asr_status(
|
||||
processor_result_id,
|
||||
asr_status,
|
||||
output.segment_count,
|
||||
)
|
||||
.await
|
||||
{
|
||||
error!("Failed to update ASR status: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(ref face_status) = output.face_status {
|
||||
if let Err(e) = db
|
||||
.update_face_status(
|
||||
processor_result_id,
|
||||
face_status,
|
||||
output.total_faces,
|
||||
)
|
||||
.await
|
||||
{
|
||||
error!("Failed to update FACE status: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(e) = redis
|
||||
.update_worker_processor_status(
|
||||
&job.uuid,
|
||||
@@ -416,6 +490,20 @@ impl ProcessorPool {
|
||||
{
|
||||
error!("Failed to update Redis processor status: {}", e);
|
||||
}
|
||||
|
||||
// Also update PostgreSQL processing_status JSON
|
||||
if let Err(e) = db
|
||||
.update_processor_progress(
|
||||
&job.uuid,
|
||||
&processor_name,
|
||||
output.frames_processed as u64,
|
||||
output.total_frames as u64,
|
||||
"completed",
|
||||
)
|
||||
.await
|
||||
{
|
||||
error!("Failed to update PostgreSQL processor status: {}", e);
|
||||
}
|
||||
} else {
|
||||
error!(
|
||||
"Processor {} output failed verification for job {}: {:?}",
|
||||
@@ -569,6 +657,10 @@ impl ProcessorPool {
|
||||
total_frames,
|
||||
retry_count: 0,
|
||||
pid: 0,
|
||||
asr_status: None,
|
||||
segment_count: 0,
|
||||
face_status: None,
|
||||
total_faces: 0,
|
||||
})
|
||||
}
|
||||
ProcessorType::Yolo => {
|
||||
@@ -612,6 +704,10 @@ impl ProcessorPool {
|
||||
total_frames,
|
||||
retry_count: 0,
|
||||
pid: 0,
|
||||
asr_status: None,
|
||||
segment_count: 0,
|
||||
face_status: None,
|
||||
total_faces: 0,
|
||||
})
|
||||
}
|
||||
ProcessorType::Ocr => {
|
||||
@@ -655,6 +751,10 @@ impl ProcessorPool {
|
||||
total_frames,
|
||||
retry_count: 0,
|
||||
pid: 0,
|
||||
asr_status: None,
|
||||
segment_count: 0,
|
||||
face_status: None,
|
||||
total_faces: 0,
|
||||
})
|
||||
}
|
||||
ProcessorType::Face => {
|
||||
@@ -666,9 +766,16 @@ impl ProcessorPool {
|
||||
)
|
||||
.await?;
|
||||
let chunks_produced = result.frames.len() as i32;
|
||||
let face_status = result.status.clone();
|
||||
let total_faces = result.total_faces;
|
||||
tracing::info!(
|
||||
"FACE completed, storing {} frames for {}",
|
||||
"FACE completed, status={}, {} frames, {} total faces for {}",
|
||||
face_status
|
||||
.as_ref()
|
||||
.map(|s| s.to_string())
|
||||
.unwrap_or_default(),
|
||||
chunks_produced,
|
||||
total_faces,
|
||||
job.uuid
|
||||
);
|
||||
if let Err(e) = Self::store_face_chunks(db, &job.uuid, &result).await {
|
||||
@@ -720,6 +827,10 @@ impl ProcessorPool {
|
||||
total_frames,
|
||||
retry_count: 0,
|
||||
pid: 0,
|
||||
asr_status: None,
|
||||
segment_count: 0,
|
||||
face_status,
|
||||
total_faces,
|
||||
})
|
||||
}
|
||||
ProcessorType::FaceCluster => {
|
||||
@@ -741,6 +852,10 @@ impl ProcessorPool {
|
||||
total_frames: 0,
|
||||
retry_count: 0,
|
||||
pid: 0,
|
||||
asr_status: None,
|
||||
segment_count: 0,
|
||||
face_status: None,
|
||||
total_faces: 0,
|
||||
})
|
||||
}
|
||||
ProcessorType::Pose => {
|
||||
@@ -784,6 +899,10 @@ impl ProcessorPool {
|
||||
total_frames,
|
||||
retry_count: 0,
|
||||
pid: 0,
|
||||
asr_status: None,
|
||||
segment_count: 0,
|
||||
face_status: None,
|
||||
total_faces: 0,
|
||||
})
|
||||
}
|
||||
ProcessorType::Hand => {
|
||||
@@ -824,6 +943,10 @@ impl ProcessorPool {
|
||||
total_frames,
|
||||
retry_count: 0,
|
||||
pid: 0,
|
||||
asr_status: None,
|
||||
segment_count: 0,
|
||||
face_status: None,
|
||||
total_faces: 0,
|
||||
})
|
||||
}
|
||||
ProcessorType::Appearance => {
|
||||
@@ -851,14 +974,24 @@ impl ProcessorPool {
|
||||
total_frames,
|
||||
retry_count: 0,
|
||||
pid: 0,
|
||||
asr_status: None,
|
||||
segment_count: 0,
|
||||
face_status: None,
|
||||
total_faces: 0,
|
||||
})
|
||||
}
|
||||
ProcessorType::Asr => {
|
||||
let result =
|
||||
processor::process_asr(video_path, output_path.to_str().unwrap(), uuid).await?;
|
||||
let chunks_produced = result.segments.len() as i32;
|
||||
let asr_status = result.status.clone();
|
||||
let segment_count = result.segment_count;
|
||||
tracing::info!(
|
||||
"ASR completed, storing {} segments for {}",
|
||||
"ASR completed, status={}, {} segments for {}",
|
||||
asr_status
|
||||
.as_ref()
|
||||
.map(|s| s.to_string())
|
||||
.unwrap_or_default(),
|
||||
chunks_produced,
|
||||
job.uuid
|
||||
);
|
||||
@@ -892,6 +1025,10 @@ impl ProcessorPool {
|
||||
total_frames,
|
||||
retry_count: 0,
|
||||
pid: 0,
|
||||
asr_status,
|
||||
segment_count,
|
||||
face_status: None,
|
||||
total_faces: 0,
|
||||
})
|
||||
}
|
||||
ProcessorType::Asrx => {
|
||||
@@ -899,8 +1036,14 @@ impl ProcessorPool {
|
||||
processor::process_asrx(video_path, output_path.to_str().unwrap(), uuid)
|
||||
.await?;
|
||||
let chunks_produced = result.segments.len() as i32;
|
||||
let asr_status = result.status.clone();
|
||||
let segment_count = result.segment_count;
|
||||
tracing::info!(
|
||||
"ASRX completed, storing {} segments for {}",
|
||||
"ASRX completed, status={}, {} segments for {}",
|
||||
asr_status
|
||||
.as_ref()
|
||||
.map(|s| s.to_string())
|
||||
.unwrap_or_default(),
|
||||
chunks_produced,
|
||||
job.uuid
|
||||
);
|
||||
@@ -959,6 +1102,10 @@ impl ProcessorPool {
|
||||
total_frames,
|
||||
retry_count: 0,
|
||||
pid: 0,
|
||||
asr_status,
|
||||
segment_count,
|
||||
face_status: None,
|
||||
total_faces: 0,
|
||||
})
|
||||
}
|
||||
ProcessorType::Scene => {
|
||||
@@ -977,6 +1124,10 @@ impl ProcessorPool {
|
||||
total_frames,
|
||||
retry_count: 0,
|
||||
pid: 0,
|
||||
asr_status: None,
|
||||
segment_count: 0,
|
||||
face_status: None,
|
||||
total_faces: 0,
|
||||
});
|
||||
} else if scene_path.exists() {
|
||||
tracing::info!("Scene JSON exists for {}, loading from file", job.uuid);
|
||||
@@ -1025,6 +1176,10 @@ impl ProcessorPool {
|
||||
total_frames,
|
||||
retry_count: 0,
|
||||
pid: 0,
|
||||
asr_status: None,
|
||||
segment_count: 0,
|
||||
face_status: None,
|
||||
total_faces: 0,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1363,8 +1518,6 @@ impl ProcessorPool {
|
||||
|
||||
db.store_raw_pre_chunks_batch(uuid, "asrx", &pre_chunks_to_store)
|
||||
.await?;
|
||||
db.store_raw_pre_chunks_batch(uuid, "asr", &pre_chunks_to_store)
|
||||
.await?;
|
||||
db.store_speaker_detections_batch(uuid, &speaker_detections)
|
||||
.await?;
|
||||
Ok(())
|
||||
|
||||
Reference in New Issue
Block a user