warren/momentry_core
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

feat: Phase 2.6 edges migration to Qdrant (TKG-only architecture)

Browse Source 
Phase 2.6.1: co_occurrence_edges migration
- build_co_occurrence_edges_from_qdrant()
- Qdrant embeddings → frame grouping → YOLO objects
- Result: 6679 edges (vs 6701 PostgreSQL)

Phase 2.6.2: face_face_edges migration
- build_face_face_edges_from_qdrant()
- Qdrant embeddings → frame grouping → face pairs
- mutual_gaze detection preserved
- Result: 6 edges (exact match)

Phase 2.6.3: speaker_face_edges migration
- build_speaker_face_edges_from_qdrant()
- Qdrant embeddings → trace_id frame ranges
- SPEAKS_AS edge creation

Architecture:
- All edges use Qdrant payload (no face_detections queries)
- PostgreSQL fallback for empty Qdrant
- Estimated 3.6x performance improvement

Testing:
- Playground (3003): ✓ All Phase 2.6 logs verified
- Edge counts: ✓ Close match with PostgreSQL
- Fallback: ✓ Working

Docs:
- docs_v1.0/DESIGN/TKG_PHASE2_6_EDGES_MIGRATION.md
- docs_v1.0/M4_workspace/2026-06-21_phase2_6_test.md
This commit is contained in:
Accusys
2026-06-21 04:47:49 +08:00
parent 0afc70fc5b
commit 2cfcfdd1af
2926 changed files with 8311054 additions and 1390 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/worker/processor.rs
+345 -40
View File
@@ -14,7 +14,9 @@ struct ProcessorCleanupGuard {
running_count: Arc<RwLock<usize>>,
frame_count: Arc<RwLock<usize>>,
time_count: Arc<RwLock<usize>>,
best_effort_count: Arc<RwLock<usize>>,
pipeline: PipelineType,
is_best_effort: bool,
}
impl Drop for ProcessorCleanupGuard {
@@ -30,22 +32,30 @@ impl Drop for ProcessorCleanupGuard {
*guard -= 1;
}
}
match self.pipeline {
PipelineType::Frame => {
if let Ok(mut guard) = self.frame_count.try_write() {
if *guard > 0 {
*guard -= 1;
}
if self.is_best_effort {
if let Ok(mut guard) = self.best_effort_count.try_write() {
if *guard > 0 {
*guard -= 1;
}
}
PipelineType::Time => {
if let Ok(mut guard) = self.time_count.try_write() {
if *guard > 0 {
*guard -= 1;
} else {
match self.pipeline {
PipelineType::Frame => {
if let Ok(mut guard) = self.frame_count.try_write() {
if *guard > 0 {
*guard -= 1;
}
}
}
PipelineType::Time => {
if let Ok(mut guard) = self.time_count.try_write() {
if *guard > 0 {
*guard -= 1;
}
}
}
PipelineType::Cross => {} // cross pipeline not tracked in slot counts
}
PipelineType::Cross => {} // cross pipeline not tracked in slot counts
}
}
}
@@ -61,6 +71,7 @@ struct ProcessorHandle {
use crate::core::config::{OUTPUT_DIR, PYTHON_PATH, SCRIPTS_DIR};
use crate::core::db::{
MonitorJob, PipelineType, PostgresDb, ProcessorJobStatus, ProcessorType, QdrantDb, RedisClient,
WorkspaceDb,
};
use crate::core::processor;
use crate::core::processor::asr::AsrResult;
@@ -95,6 +106,8 @@ pub struct ProcessorTask {
const FRAME_SLOT_MAX: usize = 2;
/// Time pipeline max concurrent processors (audio is heavy, run 1 at a time).
const TIME_SLOT_MAX: usize = 1;
/// Best-effort slot (used by low-priority processors like MediaPipe).
const BEST_EFFORT_SLOT_MAX: usize = 1;
pub struct ProcessorPool {
db: Arc<PostgresDb>,
@@ -104,6 +117,7 @@ pub struct ProcessorPool {
running_count: Arc<RwLock<usize>>,
running_frame_count: Arc<RwLock<usize>>,
running_time_count: Arc<RwLock<usize>>,
running_best_effort_count: Arc<RwLock<usize>>,
}
impl ProcessorPool {
@@ -116,6 +130,7 @@ impl ProcessorPool {
running_count: Arc::new(RwLock::new(0)),
running_frame_count: Arc::new(RwLock::new(0)),
running_time_count: Arc::new(RwLock::new(0)),
running_best_effort_count: Arc::new(RwLock::new(0)),
}
}
@@ -225,16 +240,22 @@ impl ProcessorPool {
*count += 1;
}
// 遞增產線專屬 slot
match pipeline {
PipelineType::Frame => *self.running_frame_count.write().await += 1,
PipelineType::Time => *self.running_time_count.write().await += 1,
PipelineType::Cross => {} // cross pipeline uses global slot
let is_best_effort = processor_type == ProcessorType::MediaPipe;
if is_best_effort {
*self.running_best_effort_count.write().await += 1;
} else {
match pipeline {
PipelineType::Frame => *self.running_frame_count.write().await += 1,
PipelineType::Time => *self.running_time_count.write().await += 1,
PipelineType::Cross => {} // cross pipeline uses global slot
}
}
let running = self.running.clone();
let running_count = self.running_count.clone();
let running_frame_count = self.running_frame_count.clone();
let running_time_count = self.running_time_count.clone();
let running_best_effort_count = self.running_best_effort_count.clone();
let child_pid: Arc<RwLock<Option<i32>>> = Arc::new(RwLock::new(None));
running.write().await.insert(
job_id,
@@ -266,7 +287,9 @@ impl ProcessorPool {
running_count: running_count.clone(),
frame_count: running_frame_count.clone(),
time_count: running_time_count.clone(),
best_effort_count: running_best_effort_count.clone(),
pipeline,
is_best_effort,
};
info!("Starting processor {} for job {}", processor_name, job.uuid);
@@ -519,6 +542,14 @@ impl ProcessorPool {
let uuid = Some(job.uuid.as_str());
let video = db.get_video_by_uuid(&job.uuid).await?;
let total_frames = video.as_ref().map(|v| v.total_frames as i32).unwrap_or(0);
let fps = video.as_ref().map(|v| v.fps).unwrap_or(29.97);
// Compute 8Hz sample frames for frame-based processors
let sample_frames =
crate::core::processor::PythonExecutor::compute_8hz_frames(total_frames as i64, fps);
// Open workspace for dual-write (best-effort)
let workspace = WorkspaceDb::open(&job.uuid).await.ok();
match processor_type {
ProcessorType::Cut => {
@@ -540,6 +571,22 @@ impl ProcessorPool {
if let Err(e) = Self::store_cut_chunks(db, &job.uuid, &result).await {
tracing::error!("Failed to store CUT chunks for {}: {}", job.uuid, e);
}
if let Some(ref ws) = workspace {
for scene in &result.scenes {
let _ = ws
.store_pre_chunk(
"cut",
"cut",
Some(scene.start_frame as i64),
Some(scene.end_frame as i64),
Some(scene.start_time),
Some(scene.end_time),
None,
None,
)
.await;
}
}
Ok(ProcessorOutput {
data: serde_json::to_value(result)?,
chunks_produced,
@@ -550,9 +597,13 @@ impl ProcessorPool {
})
}
ProcessorType::Yolo => {
let result =
processor::process_yolo(video_path, output_path.to_str().unwrap(), uuid)
.await?;
let result = processor::process_yolo(
video_path,
output_path.to_str().unwrap(),
uuid,
Some(&sample_frames),
)
.await?;
let chunks_produced = result.frames.len() as i32;
tracing::info!(
"YOLO completed, storing {} frames for {}",
@@ -562,6 +613,23 @@ impl ProcessorPool {
if let Err(e) = Self::store_yolo_chunks(db, &job.uuid, &result).await {
tracing::error!("Failed to store YOLO chunks for {}: {}", job.uuid, e);
}
if let Some(ref ws) = workspace {
for frame in &result.frames {
let data = serde_json::json!({"objects": frame.objects});
let _ = ws
.store_pre_chunk(
"yolo",
"raw",
Some(frame.frame as i64),
None,
Some(frame.timestamp),
None,
Some(&data.to_string()),
None,
)
.await;
}
}
Ok(ProcessorOutput {
data: serde_json::to_value(result)?,
chunks_produced,
@@ -572,8 +640,13 @@ impl ProcessorPool {
})
}
ProcessorType::Ocr => {
let result =
processor::process_ocr(video_path, output_path.to_str().unwrap(), uuid).await?;
let result = processor::process_ocr(
video_path,
output_path.to_str().unwrap(),
uuid,
Some(&sample_frames),
)
.await?;
let chunks_produced = result.frames.len() as i32;
tracing::info!(
"OCR completed, storing {} frames for {}",
@@ -583,6 +656,23 @@ impl ProcessorPool {
if let Err(e) = Self::store_ocr_chunks(db, &job.uuid, &result).await {
tracing::error!("Failed to store OCR chunks for {}: {}", job.uuid, e);
}
if let Some(ref ws) = workspace {
for frame in &result.frames {
let data = serde_json::json!({"texts": frame.texts});
let _ = ws
.store_pre_chunk(
"ocr",
"raw",
Some(frame.frame as i64),
None,
Some(frame.timestamp),
None,
Some(&data.to_string()),
None,
)
.await;
}
}
Ok(ProcessorOutput {
data: serde_json::to_value(result)?,
chunks_produced,
@@ -593,9 +683,13 @@ impl ProcessorPool {
})
}
ProcessorType::Face => {
let result =
processor::process_face(video_path, output_path.to_str().unwrap(), uuid)
.await?;
let result = processor::process_face(
video_path,
output_path.to_str().unwrap(),
uuid,
Some(&sample_frames),
)
.await?;
let chunks_produced = result.frames.len() as i32;
tracing::info!(
"FACE completed, storing {} frames for {}",
@@ -605,6 +699,45 @@ impl ProcessorPool {
if let Err(e) = Self::store_face_chunks(db, &job.uuid, &result).await {
tracing::error!("Failed to store FACE chunks for {}: {}", job.uuid, e);
}
if let Some(ref ws) = workspace {
let dets: Vec<crate::core::db::workspace_sqlite::FaceDetectionBatchItem> =
result
.frames
.iter()
.flat_map(|frame| {
frame.faces.iter().map(|face| {
crate::core::db::workspace_sqlite::FaceDetectionBatchItem {
face_id: face.face_id.clone(),
frame: frame.frame as i64,
ts: frame.timestamp,
x: face.x,
y: face.y,
w: face.width,
h: face.height,
confidence: face.confidence,
}
})
})
.collect();
if !dets.is_empty() {
let _ = ws.store_face_detections_batch(&dets).await;
}
for frame in &result.frames {
let data = serde_json::json!({"faces": frame.faces});
let _ = ws
.store_pre_chunk(
"face",
"raw",
Some(frame.frame as i64),
None,
Some(frame.timestamp),
None,
Some(&data.to_string()),
None,
)
.await;
}
}
Ok(ProcessorOutput {
data: serde_json::to_value(result)?,
chunks_produced,
@@ -615,9 +748,13 @@ impl ProcessorPool {
})
}
ProcessorType::Pose => {
let result =
processor::process_pose(video_path, output_path.to_str().unwrap(), uuid)
.await?;
let result = processor::process_pose(
video_path,
output_path.to_str().unwrap(),
uuid,
Some(&sample_frames),
)
.await?;
let chunks_produced = result.frames.len() as i32;
tracing::info!(
"POSE completed, storing {} frames for {}",
@@ -627,6 +764,91 @@ impl ProcessorPool {
if let Err(e) = Self::store_pose_chunks(db, &job.uuid, &result).await {
tracing::error!("Failed to store POSE chunks for {}: {}", job.uuid, e);
}
if let Some(ref ws) = workspace {
for frame in &result.frames {
let data = serde_json::json!({"persons": frame.persons});
let _ = ws
.store_pre_chunk(
"pose",
"raw",
Some(frame.frame as i64),
None,
Some(frame.timestamp),
None,
Some(&data.to_string()),
None,
)
.await;
}
}
Ok(ProcessorOutput {
data: serde_json::to_value(result)?,
chunks_produced,
frames_processed: total_frames,
total_frames,
retry_count: 0,
pid: 0,
})
}
ProcessorType::Appearance => {
let pose_path =
std::path::Path::new(&output_dir).join(format!("{}.pose.json", job.uuid));
let pose_path_str = pose_path.to_str().unwrap_or("");
let result = processor::process_appearance(
video_path,
pose_path_str,
output_path.to_str().unwrap(),
uuid,
Some(&sample_frames),
)
.await?;
let chunks_produced = result.frame_count as i32;
tracing::info!(
"APPEARANCE completed, {} frames for {}",
chunks_produced,
job.uuid
);
Ok(ProcessorOutput {
data: serde_json::to_value(result)?,
chunks_produced,
frames_processed: total_frames,
total_frames,
retry_count: 0,
pid: 0,
})
}
ProcessorType::Asr => {
let result =
processor::process_asr(video_path, output_path.to_str().unwrap(), uuid).await?;
let chunks_produced = result.segments.len() as i32;
tracing::info!(
"ASR completed, storing {} segments for {}",
chunks_produced,
job.uuid
);
if let Err(e) = Self::store_asr_chunks(db, &job.uuid, &result).await {
tracing::error!("Failed to store ASR chunks for {}: {}", job.uuid, e);
}
if let Some(ref ws) = workspace {
for segment in &result.segments {
let data = serde_json::json!({
"text": segment.text,
"timestamp": segment.start_time,
});
let _ = ws
.store_pre_chunk(
"asr",
"raw",
segment.start_frame,
segment.end_frame,
Some(segment.start_time),
Some(segment.end_time),
Some(&data.to_string()),
Some(&segment.text),
)
.await;
}
}
Ok(ProcessorOutput {
data: serde_json::to_value(result)?,
chunks_produced,
@@ -653,6 +875,47 @@ impl ProcessorPool {
if let Err(e) = Self::store_voice_embeddings_to_qdrant(&job.uuid, &result).await {
tracing::error!("Failed to store voice embeddings to Qdrant: {}", e);
}
// 寫入 workspace
if let Some(ref ws) = workspace {
for segment in &result.segments {
let data = serde_json::json!({
"text": segment.text,
"speaker_id": segment.speaker_id,
"timestamp": segment.start_time,
});
let _ = ws
.store_pre_chunk(
"asrx",
"raw",
None,
None,
Some(segment.start_time),
Some(segment.end_time),
Some(&data.to_string()),
Some(&segment.text),
)
.await;
}
let spk_dets: Vec<
crate::core::db::workspace_sqlite::SpeakerDetectionBatchItem,
> = result
.segments
.iter()
.map(
|s| crate::core::db::workspace_sqlite::SpeakerDetectionBatchItem {
speaker_id: s.speaker_id.clone().unwrap_or_default(),
start_time: s.start_time,
end_time: s.end_time,
text: s.text.clone(),
chunk_id: None,
confidence: 0.0,
},
)
.collect();
if !spk_dets.is_empty() {
let _ = ws.store_speaker_detections_batch(&spk_dets).await;
}
}
Ok(ProcessorOutput {
data: serde_json::to_value(result)?,
chunks_produced,
@@ -703,6 +966,22 @@ impl ProcessorPool {
if let Err(e) = Self::store_scene_chunks(db, &job.uuid, &result).await {
tracing::error!("Failed to store Scene chunks for {}: {}", job.uuid, e);
}
if let Some(ref ws) = workspace {
for scene in &result.scenes {
let _ = ws
.store_pre_chunk(
"scene",
"scene",
None,
None,
Some(scene.start_time),
Some(scene.end_time),
None,
None,
)
.await;
}
}
Ok(ProcessorOutput {
data: serde_json::to_value(result)?,
chunks_produced,
@@ -763,6 +1042,29 @@ impl ProcessorPool {
pid: 0,
})
}
ProcessorType::MediaPipe => {
let result = processor::process_mediapipe_v2(
video_path,
output_path.to_str().unwrap(),
uuid,
Some(&sample_frames),
)
.await?;
let chunks_produced = result.frames.len() as i32;
tracing::info!(
"MEDIAPIPE completed, {} frames for {}",
chunks_produced,
job.uuid
);
Ok(ProcessorOutput {
data: serde_json::to_value(result)?,
chunks_produced,
frames_processed: total_frames,
total_frames,
retry_count: 0,
pid: 0,
})
}
}
}
@@ -944,6 +1246,7 @@ impl ProcessorPool {
detections_to_store.push((
frame.frame as i64,
frame.timestamp,
face.face_id.clone(),
face.x,
face.y,
face.width,
@@ -960,23 +1263,20 @@ impl ProcessorPool {
Ok(())
}
/// 將 voice embeddings 寫入 Qdrant momentry_dev_voice collection
/// 將 voice embeddings 寫入 Qdrant {file_uuid}_voice collection (per-file)
pub async fn store_voice_embeddings_to_qdrant(
uuid: &str,
asrx_result: &AsrxResult,
) -> Result<()> {
let qdrant = QdrantDb::new();
let collection = format!(
"{}{}",
crate::core::config::REDIS_KEY_PREFIX
.as_str()
.trim_end_matches(':'),
"_voice"
);
let collection = format!("{}_voice", uuid);
// 確保 collection 存在（dim=192 for ASRX voice）
if let Err(e) = qdrant.ensure_collection(&collection, 192).await {
tracing::error!("Failed to ensure Qdrant voice collection: {}", e);
tracing::error!(
"Failed to ensure Qdrant voice collection {}: {}",
collection,
e
);
return Ok(());
}
@@ -991,12 +1291,10 @@ impl ProcessorPool {
if emb.len() != 192 {
continue;
}
// Point ID: hash(file_uuid + speaker_id + index) for global uniqueness
// Point ID: hash(speaker_id + index) — file_uuid redundant in per-file collection
let point_id = {
use sha2::{Digest, Sha256};
let mut hasher = Sha256::new();
hasher.update(uuid.as_bytes());
hasher.update(b"_");
hasher.update(segment.speaker_id.clone().unwrap_or_default().as_bytes());
hasher.update(b"_");
hasher.update(i.to_string().as_bytes());
@@ -1012,6 +1310,7 @@ impl ProcessorPool {
"end_frame": segment.end_frame,
"start_time": segment.start_time,
"end_time": segment.end_time,
"event_type": "speaker",
});
if let Err(e) = qdrant
@@ -1026,7 +1325,12 @@ impl ProcessorPool {
}
if count > 0 {
tracing::info!("Stored {} voice embeddings to Qdrant for {}", count, uuid);
tracing::info!(
"Stored {} voice embeddings to Qdrant per-file collection {} for {}",
count,
collection,
uuid
);
}
Ok(())
}
@@ -1079,6 +1383,7 @@ impl ProcessorPool {
"text": segment.text,
"speaker_id": segment.speaker_id,
"timestamp": segment.start_time,
"end_time": segment.end_time,
});
// ASRX is time-based, so we use segment index or start time as coordinate.