warren/momentry_core
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

feat: Phase 2.6 edges migration to Qdrant (TKG-only architecture)

Browse Source 
Phase 2.6.1: co_occurrence_edges migration
- build_co_occurrence_edges_from_qdrant()
- Qdrant embeddings → frame grouping → YOLO objects
- Result: 6679 edges (vs 6701 PostgreSQL)

Phase 2.6.2: face_face_edges migration
- build_face_face_edges_from_qdrant()
- Qdrant embeddings → frame grouping → face pairs
- mutual_gaze detection preserved
- Result: 6 edges (exact match)

Phase 2.6.3: speaker_face_edges migration
- build_speaker_face_edges_from_qdrant()
- Qdrant embeddings → trace_id frame ranges
- SPEAKS_AS edge creation

Architecture:
- All edges use Qdrant payload (no face_detections queries)
- PostgreSQL fallback for empty Qdrant
- Estimated 3.6x performance improvement

Testing:
- Playground (3003): ✓ All Phase 2.6 logs verified
- Edge counts: ✓ Close match with PostgreSQL
- Fallback: ✓ Working

Docs:
- docs_v1.0/DESIGN/TKG_PHASE2_6_EDGES_MIGRATION.md
- docs_v1.0/M4_workspace/2026-06-21_phase2_6_test.md
This commit is contained in:
Accusys
2026-06-21 04:47:49 +08:00
parent 0afc70fc5b
commit 2cfcfdd1af
2926 changed files with 8311054 additions and 1390 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/processor/appearance.rs
+100
View File
@@ -0,0 +1,100 @@
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use std::time::Duration;
use super::executor::PythonExecutor;
const APPEARANCE_TIMEOUT: Duration = Duration::from_secs(7200);
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct AppearanceResult {
pub frame_count: u64,
pub fps: f64,
pub frames: Vec<AppearanceFrame>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct AppearanceFrame {
pub frame: u64,
pub timestamp: f64,
pub persons: Vec<AppearancePerson>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct AppearancePerson {
pub person_id: u64,
pub bbox: BBox,
pub hsv_histogram: Vec<Vec<f64>>,
pub dominant_colors: Vec<Vec<f64>>,
pub upper_body: Option<Vec<Vec<f64>>>,
pub lower_body: Option<Vec<Vec<f64>>>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct BBox {
pub x: i32,
pub y: i32,
pub width: i32,
pub height: i32,
}
pub async fn process_appearance(
video_path: &str,
pose_path: &str,
output_path: &str,
uuid: Option<&str>,
frames: Option<&[i64]>,
) -> Result<AppearanceResult> {
if std::path::Path::new(output_path).exists() {
let json_str =
std::fs::read_to_string(output_path).context("Failed to read APPEARANCE output")?;
let result: AppearanceResult =
serde_json::from_str(&json_str).context("Failed to parse APPEARANCE output")?;
tracing::info!(
"[APPEARANCE] Skipping (already exists): {} frames",
result.frame_count
);
return Ok(result);
}
let executor = PythonExecutor::new()?;
let script_name = "appearance_processor.py";
let script_path = executor.script_path(script_name);
tracing::info!(
"[APPEARANCE] Starting appearance extraction: {}",
video_path
);
if !script_path.exists() {
tracing::warn!("[APPEARANCE] Script not found, returning empty result");
return Ok(AppearanceResult {
frame_count: 0,
fps: 0.0,
frames: vec![],
});
}
executor
.run_with_output_idx_and_frames(
script_name,
&[video_path, pose_path, output_path],
uuid,
"APPEARANCE",
Some(APPEARANCE_TIMEOUT),
2,
frames,
)
.await
.with_context(|| format!("Failed to run {:?}", script_path))?;
let json_str =
std::fs::read_to_string(output_path).context("Failed to read APPEARANCE output")?;
let result: AppearanceResult =
serde_json::from_str(&json_str).context("Failed to parse APPEARANCE output")?;
tracing::info!("[APPEARANCE] Result: {} frames", result.frame_count);
Ok(result)
}
src/core/processor/asrx.rs
+57 -3
View File
@@ -1,4 +1,5 @@
use anyhow::{Context, Result};
use libc;
use serde::{Deserialize, Serialize};
use std::time::Duration;
use tokio::process::Command;
@@ -51,6 +52,26 @@ pub async fn process_asrx(
anyhow::bail!("asrx_processor.py not found");
}
// Verify script integrity via SHA256 checksum
executor
.verify_script_integrity("asrx_processor.py")
.context("Pre-execution integrity check failed for asrx_processor.py")?;
// Pre-flight: check ffprobe and ffmpeg availability
for tool in &["ffprobe", "ffmpeg"] {
let check = Command::new("which").arg(tool).output().await?;
if !check.status.success() {
anyhow::bail!("{} not found on PATH — required by ASRX", tool);
}
}
// Stage existing output: .json → .tmp (will be renamed back on success)
let output_path_obj = std::path::Path::new(output_path);
let tmp_path = output_path_obj.with_extension("json.tmp");
if output_path_obj.exists() {
let _ = std::fs::rename(output_path_obj, &tmp_path);
}
tracing::info!(
"[ASRX] Running: {} {} {} {}",
executor.python_path().display(),
@@ -68,14 +89,30 @@ pub async fn process_asrx(
}
cmd.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped());
.stderr(std::process::Stdio::piped())
.kill_on_drop(true)
.process_group(0);
let child = cmd.spawn().context("Failed to run ASRX processor")?;
let child_pid = child.id();
let output = match timeout(ASRX_TIMEOUT, child.wait_with_output()).await {
Ok(Ok(output)) => output,
Ok(Err(e)) => return Err(e).context("Failed to run ASRX processor"),
Err(_) => anyhow::bail!("ASRX processing timed out after {:?}", ASRX_TIMEOUT),
Ok(Err(e)) => {
let _ = std::fs::rename(&tmp_path, output_path_obj.with_extension("json.err"));
return Err(e).context("Failed to run ASRX processor");
}
Err(_) => {
// Timeout: kill process group, rename .tmp → .err
if let Some(pid) = child_pid {
let pgid = pid as i32;
unsafe {
libc::killpg(pgid, libc::SIGKILL);
}
}
let _ = std::fs::rename(&tmp_path, output_path_obj.with_extension("json.err"));
anyhow::bail!("ASRX processing timed out after {:?}", ASRX_TIMEOUT);
}
};
let stderr = String::from_utf8_lossy(&output.stderr);
@@ -98,9 +135,26 @@ pub async fn process_asrx(
tracing::info!("[ASRX] stderr output:\n{}", stderr);
if !output.status.success() {
// On failure: rename .tmp → .err (partial output preserved if valid JSON)
if tmp_path.exists() {
let is_valid = std::fs::read_to_string(&tmp_path)
.ok()
.and_then(|c| serde_json::from_str::<serde_json::Value>(&c).ok())
.is_some();
if is_valid {
let _ = std::fs::rename(&tmp_path, output_path_obj.with_extension("json.partial"));
} else {
let _ = std::fs::rename(&tmp_path, output_path_obj.with_extension("json.err"));
}
}
anyhow::bail!("ASRX failed: {}", stderr);
}
// Success: rename .tmp back to .json (if .tmp exists, otherwise use the direct output)
if tmp_path.exists() {
let _ = std::fs::rename(&tmp_path, output_path_obj);
}
let json_str = std::fs::read_to_string(output_path).context("Failed to read ASRX output")?;
let result: AsrxResult =
src/core/processor/cascade_vision.rs
+66 -60
View File
@@ -4,7 +4,7 @@ use std::path::Path;
use std::time::Duration;
use tracing::{debug, info, warn};
use crate::core::processor::clip::{ClipPrediction, detect_objects};
use crate::core::processor::clip::{detect_objects, ClipPrediction};
use crate::core::vision::qwen_vl_manager::QwenVLManager;
const DEFAULT_CLIP_THRESHOLD: f32 = 0.7;
@@ -39,9 +39,13 @@ impl CascadeVisionProcessor {
}
}
pub async fn detect_objects(&self, image_path: &Path, objects: &[&str]) -> Result<CascadeDetectionResult> {
pub async fn detect_objects(
&self,
image_path: &Path,
objects: &[&str],
) -> Result<CascadeDetectionResult> {
let start_time = std::time::Instant::now();
info!(
"[Cascade] Starting detection for {:?} with {} object classes (threshold: {:.2})",
image_path,
@@ -50,7 +54,7 @@ impl CascadeVisionProcessor {
);
let clip_result = self.run_clip_detection(image_path, objects).await?;
let max_clip_confidence = clip_result
.iter()
.map(|p| p.confidence)
@@ -58,21 +62,19 @@ impl CascadeVisionProcessor {
debug!(
"[Cascade] CLIP max confidence: {:.3} (threshold: {:.2})",
max_clip_confidence,
self.clip_threshold
max_clip_confidence, self.clip_threshold
);
if max_clip_confidence > self.clip_threshold {
info!(
"[Cascade] High confidence ({:.3} > {:.2}) → triggering Qwen3-VL",
max_clip_confidence,
self.clip_threshold
max_clip_confidence, self.clip_threshold
);
let qwenvl_result = self.run_qwenvl_detection(image_path, objects).await?;
let processing_time = start_time.elapsed().as_millis() as u64;
return Ok(CascadeDetectionResult {
detections: qwenvl_result,
model_used: "qwen3vl".to_string(),
@@ -84,12 +86,11 @@ impl CascadeVisionProcessor {
info!(
"[Cascade] Low confidence ({:.3} <= {:.2}) → using CLIP results only",
max_clip_confidence,
self.clip_threshold
max_clip_confidence, self.clip_threshold
);
let processing_time = start_time.elapsed().as_millis() as u64;
Ok(CascadeDetectionResult {
detections: clip_result,
model_used: "clip".to_string(),
@@ -99,35 +100,43 @@ impl CascadeVisionProcessor {
})
}
async fn run_clip_detection(&self, image_path: &Path, objects: &[&str]) -> Result<Vec<ClipPrediction>> {
async fn run_clip_detection(
&self,
image_path: &Path,
objects: &[&str],
) -> Result<Vec<ClipPrediction>> {
let image_path_str = image_path.display().to_string();
debug!("[Cascade] Running CLIP detection for {:?}", image_path);
let predictions = detect_objects(&image_path_str, objects, None, None)
.await
.context("CLIP detection failed")?;
debug!(
"[Cascade] CLIP detected {} objects",
predictions.len()
);
debug!("[Cascade] CLIP detected {} objects", predictions.len());
Ok(predictions)
}
async fn run_qwenvl_detection(&self, image_path: &Path, objects: &[&str]) -> Result<Vec<ClipPrediction>> {
async fn run_qwenvl_detection(
&self,
image_path: &Path,
objects: &[&str],
) -> Result<Vec<ClipPrediction>> {
let image_path_str = image_path.display().to_string();
debug!("[Cascade] Running Qwen3-VL detection for {:?}", image_path);
self.qwen_vl_manager.ensure_running().await?;
let prompt = self.build_detection_prompt(objects);
let client = reqwest::Client::new();
let url = format!("http://localhost:{}/v1/chat/completions", self.qwen_vl_manager.get_port());
let url = format!(
"http://localhost:{}/v1/chat/completions",
self.qwen_vl_manager.get_port()
);
let request_body = serde_json::json!({
"model": "Qwen3VL-8B-Instruct-Q8_0",
"messages": [
@@ -150,7 +159,7 @@ impl CascadeVisionProcessor {
"max_tokens": 500,
"temperature": 0.1
});
let response = client
.post(&url)
.json(&request_body)
@@ -158,17 +167,17 @@ impl CascadeVisionProcessor {
.send()
.await
.context("Qwen3-VL API request failed")?;
if !response.status().is_success() {
warn!("[Cascade] Qwen3-VL API error: {}", response.status());
anyhow::bail!("Qwen3-VL API returned error: {}", response.status());
}
let response_json: serde_json::Value = response
.json()
.await
.context("Failed to parse Qwen3-VL response")?;
let content = response_json
.get("choices")
.and_then(|choices| choices.get(0))
@@ -176,24 +185,21 @@ impl CascadeVisionProcessor {
.and_then(|message| message.get("content"))
.and_then(|content| content.as_str())
.unwrap_or("");
debug!("[Cascade] Qwen3-VL response: {}", content);
let detections = self.parse_qwenvl_response(content, objects);
self.qwen_vl_manager.update_last_request_time().await;
info!(
"[Cascade] Qwen3-VL detected {} objects",
detections.len()
);
info!("[Cascade] Qwen3-VL detected {} objects", detections.len());
Ok(detections)
}
fn build_detection_prompt(&self, objects: &[&str]) -> String {
let object_list = objects.join(", ");
format!(
"Analyze this image and detect the following objects: {}.\n\
For each detected object, provide:\n\
@@ -214,29 +220,29 @@ impl CascadeVisionProcessor {
fn parse_qwenvl_response(&self, content: &str, _objects: &[&str]) -> Vec<ClipPrediction> {
let json_start = content.find('{');
let json_end = content.rfind('}');
if json_start.is_none() || json_end.is_none() {
debug!("[Cascade] No JSON found in Qwen3-VL response");
return Vec::new();
}
let json_str = &content[json_start.unwrap()..=json_end.unwrap()];
let parsed: serde_json::Value = serde_json::from_str(json_str)
.unwrap_or(serde_json::json!({"detections": []}));
let parsed: serde_json::Value =
serde_json::from_str(json_str).unwrap_or(serde_json::json!({"detections": []}));
let detections = parsed
.get("detections")
.and_then(|d| d.as_array())
.map(|arr| arr.clone())
.unwrap_or_else(|| Vec::new());
detections
.iter()
.filter_map(|d| {
let label = d.get("label").and_then(|l| l.as_str()).unwrap_or("");
let confidence = d.get("confidence").and_then(|c| c.as_f64()).unwrap_or(0.0) as f32;
if !label.is_empty() && confidence > 0.0 {
Some(ClipPrediction {
label: label.to_string(),
@@ -265,7 +271,7 @@ mod tests {
let processor = CascadeVisionProcessor::new();
let objects = vec!["gun", "weapon", "person"];
let prompt = processor.build_detection_prompt(&objects);
assert!(prompt.contains("gun, weapon, person"));
assert!(prompt.contains("confidence score"));
assert!(prompt.contains("JSON"));
@@ -276,9 +282,9 @@ mod tests {
let processor = CascadeVisionProcessor::new();
let response = "{\"detections\": [{\"label\": \"gun\", \"confidence\": 0.95, \"description\": \"a handgun\"}]}";
let objects = vec!["gun"];
let detections = processor.parse_qwenvl_response(response, &objects);
assert_eq!(detections.len(), 1);
assert_eq!(detections[0].label, "gun");
assert!((detections[0].confidence - 0.95).abs() < 0.001);
@@ -289,9 +295,9 @@ mod tests {
let processor = CascadeVisionProcessor::new();
let response = "{\"detections\": []}";
let objects = vec!["gun"];
let detections = processor.parse_qwenvl_response(response, &objects);
assert_eq!(detections.len(), 0);
}
@@ -300,9 +306,9 @@ mod tests {
let processor = CascadeVisionProcessor::new();
let response = "This is not JSON";
let objects = vec!["gun"];
let detections = processor.parse_qwenvl_response(response, &objects);
assert_eq!(detections.len(), 0);
}
}
}
src/core/processor/clip.rs
+24 -26
View File
@@ -75,21 +75,19 @@ pub async fn classify_image(
.await
.context("Failed to run CLIP classifier")?;
let json_str = std::fs::read_to_string(&output_path)
.context("Failed to read CLIP output")?;
let json_str = std::fs::read_to_string(&output_path).context("Failed to read CLIP output")?;
let results: std::collections::HashMap<String, Vec<ClipPrediction>> =
serde_json::from_str(&json_str)
.context("Failed to parse CLIP output")?;
serde_json::from_str(&json_str).context("Failed to parse CLIP output")?;
let predictions = results
.get(image_path)
.cloned()
.unwrap_or_default();
let predictions = results.get(image_path).cloned().unwrap_or_default();
tracing::info!(
"[CLIP] Top prediction: {} ({:.3})",
predictions.first().map(|p| p.label.as_str()).unwrap_or("none"),
predictions
.first()
.map(|p| p.label.as_str())
.unwrap_or("none"),
predictions.first().map(|p| p.confidence).unwrap_or(0.0)
);
@@ -145,26 +143,28 @@ pub async fn detect_objects(
.await
.context("Failed to run CLIP object detection")?;
let json_str = std::fs::read_to_string(&output_path)
.context("Failed to read CLIP output")?;
let json_str = std::fs::read_to_string(&output_path).context("Failed to read CLIP output")?;
let results: std::collections::HashMap<String, Vec<ClipPrediction>> =
serde_json::from_str(&json_str)
.context("Failed to parse CLIP output")?;
serde_json::from_str(&json_str).context("Failed to parse CLIP output")?;
let detected = results
.get(image_path)
.cloned()
.unwrap_or_default();
let detected = results.get(image_path).cloned().unwrap_or_default();
if !detected.is_empty() {
tracing::info!(
"[CLIP] Detected {} objects: {}",
detected.len(),
detected.iter().map(|p| p.label.as_str()).collect::<Vec<_>>().join(", ")
detected
.iter()
.map(|p| p.label.as_str())
.collect::<Vec<_>>()
.join(", ")
);
} else {
tracing::info!("[CLIP] No objects detected above threshold {:.2}", threshold);
tracing::info!(
"[CLIP] No objects detected above threshold {:.2}",
threshold
);
}
Ok(detected)
@@ -189,8 +189,7 @@ pub async fn classify_images(
// Create temp file with image paths
let temp_file = format!("/tmp/clip_batch_{}.txt", uuid::Uuid::new_v4());
std::fs::write(&temp_file, image_paths.join("\n"))
.context("Failed to write batch file")?;
std::fs::write(&temp_file, image_paths.join("\n")).context("Failed to write batch file")?;
let mut args = vec![
temp_file.clone(),
@@ -224,12 +223,11 @@ pub async fn classify_images(
.await
.context("Failed to run batch CLIP classification")?;
let json_str = std::fs::read_to_string(&output_path)
.context("Failed to read CLIP batch output")?;
let json_str =
std::fs::read_to_string(&output_path).context("Failed to read CLIP batch output")?;
let results_map: std::collections::HashMap<String, Vec<ClipPrediction>> =
serde_json::from_str(&json_str)
.context("Failed to parse CLIP batch output")?;
serde_json::from_str(&json_str).context("Failed to parse CLIP batch output")?;
let results: Vec<ClipImageResult> = image_paths
.iter()
@@ -287,4 +285,4 @@ mod tests {
assert_eq!(result.predictions.len(), 2);
assert_eq!(result.predictions[0].label, "indoor");
}
}
}
src/core/processor/executor.rs
+352 -6
View File
@@ -164,8 +164,34 @@ impl PythonExecutor {
})
}
/// Compute 8Hz sample frames from total frames and FPS.
/// Returns frames at approximately 8 samples per second.
pub fn compute_8hz_frames(total_frames: i64, fps: f64) -> Vec<i64> {
let interval = (fps / 8.0).round() as i64;
let interval = interval.max(1);
(0..total_frames).step_by(interval as usize).collect()
}
/// Merge base frames with refinement frames (for adaptive sampling).
pub fn merge_refine_frames(base: &[i64], refine: &std::collections::HashSet<i64>) -> Vec<i64> {
let mut combined: std::collections::HashSet<i64> = base.iter().cloned().collect();
combined.extend(refine.iter().cloned());
let mut sorted: Vec<i64> = combined.into_iter().collect();
sorted.sort();
sorted
}
/// Format frame list as comma-separated string for --frames argument.
pub fn format_frames_arg(frames: &[i64]) -> String {
frames
.iter()
.map(|f| f.to_string())
.collect::<Vec<_>>()
.join(",")
}
/// Verify a script's SHA256 against the checksums manifest before execution.
fn verify_script_integrity(&self, script_name: &str) -> Result<()> {
pub fn verify_script_integrity(&self, script_name: &str) -> Result<()> {
let script_path = self.scripts_dir.join(script_name);
let rel_path = format!("./{}", script_name);
@@ -226,6 +252,147 @@ impl PythonExecutor {
uuid: Option<&str>,
log_prefix: &str,
timeout_duration: Option<Duration>,
) -> Result<()> {
self.run_with_output_idx(script_name, args, uuid, log_prefix, timeout_duration, 1)
.await
}
/// Run a script with optional frame list for 8Hz sampling.
/// If frames is provided, passes --frames=0,4,8,... to the script.
pub async fn run_with_frames(
&self,
script_name: &str,
args: &[&str],
uuid: Option<&str>,
log_prefix: &str,
timeout_duration: Option<Duration>,
frames: Option<&[i64]>,
) -> Result<()> {
let script_path = self.scripts_dir.join(script_name);
if !script_path.exists() {
anyhow::bail!("Script not found: {:?}", script_path);
}
self.verify_script_integrity(script_name).context(
"Pre-execution integrity check failed — possible version mismatch or corruption",
)?;
let output_idx = 1;
let output_path = args.get(output_idx).map(|p| std::path::PathBuf::from(p));
let tmp_path = output_path.as_ref().map(|p| {
let mut tmp = p.to_path_buf();
tmp.set_extension("json.tmp");
tmp
});
if let (Some(src), Some(dst)) = (&output_path, &tmp_path) {
if src.exists() {
let _ = std::fs::rename(src, dst);
}
}
let mut cmd = Command::new(&self.python_path);
cmd.arg(&script_path);
for arg in args {
cmd.arg(arg);
}
if let Some(u) = uuid {
cmd.arg("--uuid").arg(u);
}
// Pass frame list for 8Hz sampling
if let Some(frames) = frames {
let frames_str = Self::format_frames_arg(frames);
cmd.arg("--frames").arg(&frames_str);
tracing::info!("[{}] 8Hz sampling: {} frames", log_prefix, frames.len());
}
cmd.stdout(Stdio::piped());
cmd.stderr(Stdio::piped());
cmd.kill_on_drop(true);
cmd.process_group(0);
tracing::info!("[{}] Starting: {:?}", log_prefix, script_name);
let mut child = cmd
.spawn()
.with_context(|| format!("Failed to run {}", script_name))?;
let child_pid = child.id();
let stdout = child.stdout.take().context("Failed to capture stdout")?;
let stderr = child.stderr.take().context("Failed to capture stderr")?;
let mut stdout_reader = BufReader::new(stdout).lines();
let mut stderr_reader = BufReader::new(stderr).lines();
let run_future = async {
let mut stdout_done = false;
let mut stderr_done = false;
loop {
if !stdout_done {
match stdout_reader
.next_line()
.await
.context("Failed to read stdout")?
{
Some(line) => tracing::info!("[{}] {}", log_prefix, line),
None => stdout_done = true,
}
}
if !stderr_done {
match stderr_reader
.next_line()
.await
.context("Failed to read stderr")?
{
Some(line) => tracing::warn!("[{}] {}", log_prefix, line),
None => stderr_done = true,
}
}
if stdout_done && stderr_done {
break;
}
}
let status = child.wait().await?;
Ok::<_, anyhow::Error>(status)
};
let timeout_duration = timeout_duration.unwrap_or(Duration::from_secs(3600));
let status = timeout(timeout_duration, run_future)
.await
.map_err(|e| anyhow::anyhow!("Timeout after {}s: {}", timeout_duration.as_secs(), e))?
.with_context(|| format!("Failed to run {}", script_name))?;
// Kill entire process group on failure
if !status.success() {
if let Some(pid) = child_pid {
unsafe {
libc::kill(-(pid as i32), libc::SIGTERM);
}
}
anyhow::bail!("Script {} failed with exit status: {}", script_name, status);
}
// Rename .json.tmp back to .json on success
if let (Some(src), Some(dst)) = (&tmp_path, &output_path) {
if src.exists() {
let _ = std::fs::rename(src, dst);
}
}
Ok(())
}
pub async fn run_with_output_idx(
&self,
script_name: &str,
args: &[&str],
uuid: Option<&str>,
log_prefix: &str,
timeout_duration: Option<Duration>,
output_idx: usize,
) -> Result<()> {
let script_path = self.scripts_dir.join(script_name);
@@ -239,7 +406,7 @@ impl PythonExecutor {
)?;
// 標記輸出檔為處理中（add .tmp suffix）
let output_path = args.get(1).map(|p| std::path::PathBuf::from(p));
let output_path = args.get(output_idx).map(|p| std::path::PathBuf::from(p));
let tmp_path = output_path.as_ref().map(|p| {
let mut tmp = p.to_path_buf();
tmp.set_extension("json.tmp");
@@ -282,27 +449,29 @@ impl PythonExecutor {
let mut stderr_reader = BufReader::new(stderr).lines();
let run_future = async {
let mut stdout_done = false;
let mut stderr_done = false;
loop {
tokio::select! {
line = stdout_reader.next_line() => {
line = stdout_reader.next_line(), if !stdout_done => {
match line {
Ok(Some(line)) => {
if line.starts_with(&format!("{}_", log_prefix)) {
tracing::info!("[{}] {}", log_prefix, line);
}
}
Ok(None) => break,
Ok(None) => stdout_done = true,
Err(e) => tracing::warn!("[{}] stdout error: {}", log_prefix, e),
}
}
line = stderr_reader.next_line() => {
line = stderr_reader.next_line(), if !stderr_done => {
match line {
Ok(Some(line)) => {
if line.starts_with(&format!("{}_", log_prefix)) {
tracing::info!("[{}] {}", log_prefix, line);
}
}
Ok(None) => {}
Ok(None) => stderr_done = true,
Err(e) => tracing::warn!("[{}] stderr error: {}", log_prefix, e),
}
}
@@ -390,6 +559,183 @@ impl PythonExecutor {
Ok(())
}
/// Run with output_idx and optional frame list for 8Hz sampling.
pub async fn run_with_output_idx_and_frames(
&self,
script_name: &str,
args: &[&str],
uuid: Option<&str>,
log_prefix: &str,
timeout_duration: Option<Duration>,
output_idx: usize,
frames: Option<&[i64]>,
) -> Result<()> {
let script_path = self.scripts_dir.join(script_name);
if !script_path.exists() {
anyhow::bail!("Script not found: {:?}", script_path);
}
self.verify_script_integrity(script_name).context(
"Pre-execution integrity check failed — possible version mismatch or corruption",
)?;
let output_path = args.get(output_idx).map(|p| std::path::PathBuf::from(p));
let tmp_path = output_path.as_ref().map(|p| {
let mut tmp = p.to_path_buf();
tmp.set_extension("json.tmp");
tmp
});
if let (Some(src), Some(dst)) = (&output_path, &tmp_path) {
if src.exists() {
let _ = std::fs::rename(src, dst);
}
}
let mut cmd = Command::new(&self.python_path);
cmd.arg(&script_path);
for arg in args {
cmd.arg(arg);
}
if let Some(u) = uuid {
cmd.arg("--uuid").arg(u);
}
// Pass frame list for 8Hz sampling
if let Some(frames) = frames {
let frames_str = Self::format_frames_arg(frames);
cmd.arg("--frames").arg(&frames_str);
tracing::info!("[{}] 8Hz sampling: {} frames", log_prefix, frames.len());
}
cmd.stdout(Stdio::piped());
cmd.stderr(Stdio::piped());
cmd.kill_on_drop(true);
cmd.process_group(0);
tracing::info!("[{}] Starting: {:?}", log_prefix, script_name);
let mut child = cmd
.spawn()
.with_context(|| format!("Failed to run {}", script_name))?;
let child_pid = child.id();
let stdout = child.stdout.take().context("Failed to capture stdout")?;
let stderr = child.stderr.take().context("Failed to capture stderr")?;
let mut stdout_reader = BufReader::new(stdout).lines();
let mut stderr_reader = BufReader::new(stderr).lines();
let run_future = async {
let mut stdout_done = false;
let mut stderr_done = false;
loop {
tokio::select! {
line = stdout_reader.next_line(), if !stdout_done => {
match line {
Ok(Some(line)) => {
if line.starts_with(&format!("{}_", log_prefix)) {
tracing::info!("[{}] {}", log_prefix, line);
}
}
Ok(None) => stdout_done = true,
Err(e) => tracing::warn!("[{}] stdout error: {}", log_prefix, e),
}
}
line = stderr_reader.next_line(), if !stderr_done => {
match line {
Ok(Some(line)) => {
if line.starts_with(&format!("{}_", log_prefix)) {
tracing::info!("[{}] {}", log_prefix, line);
}
}
Ok(None) => stderr_done = true,
Err(e) => tracing::warn!("[{}] stderr error: {}", log_prefix, e),
}
}
status = child.wait() => {
match status {
Ok(status) => {
if !status.success() {
tracing::error!("[{}] Process failed: {}", log_prefix, status);
return Err(anyhow::anyhow!("{} exited with: {}", script_name, status));
}
tracing::info!("[{}] Completed successfully", log_prefix);
}
Err(e) => tracing::error!("[{}] wait error: {}", log_prefix, e),
}
break;
}
}
}
Ok(())
};
let mark_failed = || {
if let Some(tmp) = &tmp_path {
if tmp.exists() {
if let Some(out) = &output_path {
let is_valid = std::fs::read_to_string(tmp)
.ok()
.and_then(|c| serde_json::from_str::<serde_json::Value>(&c).ok())
.is_some();
if is_valid {
let mut partial_path = out.to_path_buf();
partial_path.set_extension("json.partial");
let _ = std::fs::rename(tmp, &partial_path);
tracing::warn!(
"[Executor] Partial output preserved: {:?}",
partial_path
);
} else {
let mut err_path = out.to_path_buf();
err_path.set_extension("json.err");
let _ = std::fs::rename(tmp, &err_path);
}
}
}
}
};
if let Some(duration) = timeout_duration {
match timeout(duration, run_future).await {
Ok(Ok(())) => {}
Ok(Err(e)) => {
mark_failed();
return Err(e);
}
Err(_) => {
mark_failed();
if let Some(pid) = child_pid {
let pgid = pid as i32;
unsafe {
libc::killpg(pgid, libc::SIGKILL);
}
}
child.kill().await.context("Failed to kill process")?;
anyhow::bail!("{} timed out after {:?}", script_name, duration);
}
}
} else {
if let Err(e) = run_future.await {
mark_failed();
return Err(e);
}
}
if let Some(tmp) = &tmp_path {
if tmp.exists() {
if let Some(out) = &output_path {
let _ = std::fs::rename(tmp, out);
}
}
}
Ok(())
}
pub async fn run_with_output(
&self,
script_name: &str,
src/core/processor/face.rs
+5 -3
View File
@@ -29,7 +29,7 @@ pub struct Face {
pub height: i32,
pub confidence: f32,
pub embedding: Option<Vec<f32>>,
pub landmarks: Option<Vec<Vec<f32>>>,
pub landmarks: Option<serde_json::Value>,
pub attributes: Option<FaceAttributes>,
}
@@ -43,6 +43,7 @@ pub async fn process_face(
video_path: &str,
output_path: &str,
uuid: Option<&str>,
frames: Option<&[i64]>,
) -> Result<FaceResult> {
let executor = PythonExecutor::new()?;
let script_path = executor.script_path("face_processor.py");
@@ -59,12 +60,13 @@ pub async fn process_face(
}
executor
.run(
.run_with_frames(
"face_processor.py",
&[video_path, output_path],
uuid,
"FACE",
Some(FACE_TIMEOUT),
frames,
)
.await
.with_context(|| format!("Failed to run {:?}", script_path))?;
@@ -99,7 +101,7 @@ mod tests {
height: 60,
confidence: 0.95,
embedding: Some(vec![0.1, 0.2, 0.3]),
landmarks: Some(vec![vec![10.0, 20.0], vec![30.0, 40.0]]),
landmarks: Some(serde_json::json!([[10.0, 20.0], [30.0, 40.0]])),
attributes: Some(FaceAttributes {
age: Some(30),
gender: Some("male".to_string()),
src/core/processor/mediapipe.rs
+96
View File
@@ -0,0 +1,96 @@
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use std::time::Duration;
use super::executor::PythonExecutor;
const MEDIAPIPE_TIMEOUT: Duration = Duration::from_secs(7200);
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipeResult {
pub frame_count: u64,
pub fps: f64,
pub frames: Vec<MediaPipeFrame>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipeFrame {
pub frame: u64,
pub timestamp: f64,
pub persons: Vec<MediaPipePerson>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipePerson {
pub person_id: u64,
pub pose: Option<MediaPipePose>,
pub left_hand: Option<MediaPipeHand>,
pub right_hand: Option<MediaPipeHand>,
pub face_mesh: Option<MediaPipeFaceMesh>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipePose {
pub landmarks: Vec<Vec<f64>>,
pub keypoints_33: Option<Vec<String>>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipeHand {
pub landmarks: Vec<Vec<f64>>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipeFaceMesh {
pub landmarks: Vec<Vec<f64>>,
}
pub async fn process_mediapipe(
video_path: &str,
output_path: &str,
uuid: Option<&str>,
) -> Result<MediaPipeResult> {
// If mediapipe.json already exists (written by face_processor), skip
if std::path::Path::new(output_path).exists() {
let json_str = std::fs::read_to_string(output_path).context("Failed to read MEDIAPIPE output")?;
let result: MediaPipeResult =
serde_json::from_str(&json_str).context("Failed to parse MEDIAPIPE output")?;
tracing::info!("[MEDIAPIPE] Skipping (already exists): {} frames", result.frames.len());
return Ok(result);
}
let executor = PythonExecutor::new()?;
let script_name = "mediapipe_processor_v1.11.py";
let script_path = executor.script_path(script_name);
tracing::info!("[MEDIAPIPE] Starting MediaPipe Holistic: {}", video_path);
if !script_path.exists() {
tracing::warn!("[MEDIAPIPE] Script not found, returning empty result");
return Ok(MediaPipeResult {
frame_count: 0,
fps: 0.0,
frames: vec![],
});
}
executor
.run(
script_name,
&[video_path, output_path],
uuid,
"MEDIAPIPE",
Some(MEDIAPIPE_TIMEOUT),
)
.await
.with_context(|| format!("Failed to run {:?}", script_path))?;
let json_str =
std::fs::read_to_string(output_path).context("Failed to read MEDIAPIPE output")?;
let result: MediaPipeResult =
serde_json::from_str(&json_str).context("Failed to parse MEDIAPIPE output")?;
tracing::info!("[MEDIAPIPE] Result: {} frames", result.frames.len());
Ok(result)
}
src/core/processor/mediapipe_v2.rs
+203
View File
@@ -0,0 +1,203 @@
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::time::Duration;
use tokio::process::Command;
use tokio::time::timeout;
use super::executor::PythonExecutor;
const MEDIAPIPE_TIMEOUT: Duration = Duration::from_secs(7200);
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipeResult {
pub metadata: MediaPipeMetadata,
pub frames: HashMap<String, MediaPipeDictEntry>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipeMetadata {
pub fps: f64,
pub total_frames: i64,
pub processed_frames: i64,
pub sample_interval: i64,
pub width: i64,
pub height: i64,
pub processor: String,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipeDictEntry {
pub frame_number: i64,
pub timestamp: f64,
pub persons: Vec<MediaPipePerson>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipePerson {
pub person_id: i64,
#[serde(default)]
pub bbox: Option<MediaPipeBBox>,
pub face_mesh: Option<serde_json::Value>,
pub pose: Option<serde_json::Value>,
pub hands: MediaPipeHands,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipeBBox {
pub x: i64,
pub y: i64,
pub width: i64,
pub height: i64,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipeHands {
pub left: Option<serde_json::Value>,
pub right: Option<serde_json::Value>,
}
pub async fn process_mediapipe_v2(
video_path: &str,
output_path: &str,
uuid: Option<&str>,
frames: Option<&[i64]>,
) -> Result<MediaPipeResult> {
let executor = PythonExecutor::new()?;
let script_path = executor.script_path("mediapipe_holistic_processor.py");
tracing::info!("[MEDIAPIPE] Starting MediaPipe Holistic: {}", video_path);
if !script_path.exists() {
anyhow::bail!("mediapipe_holistic_processor.py not found");
}
let mut cmd = Command::new(executor.python_path());
cmd.arg(&script_path).arg(video_path).arg(output_path);
// Use explicit frame list if provided, otherwise calculate sample_interval for ~8Hz
if let Some(frames) = frames {
let frames_str = frames
.iter()
.map(|f| f.to_string())
.collect::<Vec<_>>()
.join(",");
cmd.arg("--frames").arg(&frames_str);
tracing::info!("[MEDIAPIPE] 8Hz sampling: {} frames", frames.len());
} else {
let sample_interval = calculate_sample_interval(video_path).await;
cmd.arg("--sample-interval")
.arg(sample_interval.to_string());
}
if let Some(u) = uuid {
cmd.arg("--uuid").arg(u);
}
cmd.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped());
let child = cmd.spawn().context("Failed to run MEDIAPIPE processor")?;
let output = match timeout(MEDIAPIPE_TIMEOUT, child.wait_with_output()).await {
Ok(Ok(output)) => output,
Ok(Err(e)) => return Err(e).context("Failed to run MEDIAPIPE processor"),
Err(_) => anyhow::bail!(
"MEDIAPIPE processing timed out after {:?}",
MEDIAPIPE_TIMEOUT
),
};
let stderr = String::from_utf8_lossy(&output.stderr);
for line in stderr.lines() {
let trimmed = line.trim();
if trimmed.starts_with("MEDIAPIPE_START") {
tracing::info!("[MEDIAPIPE] Loading model...");
} else if trimmed.starts_with("MEDIAPIPE_FRAME:") {
let count = trimmed.trim_start_matches("MEDIAPIPE_FRAME:");
tracing::info!("[MEDIAPIPE] Processed {} frames...", count);
} else if trimmed.starts_with("MEDIAPIPE_COMPLETE:") {
let count = trimmed.trim_start_matches("MEDIAPIPE_COMPLETE:");
tracing::info!("[MEDIAPIPE] Completed! Total: {} frames", count);
} else if trimmed.starts_with("MEDIAPIPE_INFO:") {
let info = trimmed.trim_start_matches("MEDIAPIPE_INFO:");
tracing::info!("[MEDIAPIPE] {}", info);
} else if trimmed.starts_with("MEDIAPIPE_ERROR:") {
let err = trimmed.trim_start_matches("MEDIAPIPE_ERROR:");
tracing::error!("[MEDIAPIPE] {}", err);
}
}
tracing::info!("[MEDIAPIPE] stderr output:\n{}", stderr);
if !output.status.success() {
anyhow::bail!("MEDIAPIPE failed: {}", stderr);
}
let json_str =
std::fs::read_to_string(output_path).context("Failed to read MEDIAPIPE output")?;
let result: MediaPipeResult =
serde_json::from_str(&json_str).context("Failed to parse MEDIAPIPE output")?;
tracing::info!("[MEDIAPIPE] Result: {} frames", result.frames.len());
Ok(result)
}
async fn calculate_sample_interval(video_path: &str) -> i64 {
// Try ffprobe to get FPS, calculate sample_interval for ~8Hz
let probe_cmd = Command::new("ffprobe")
.args([
"-v",
"quiet",
"-print_format",
"json",
"-show_streams",
video_path,
])
.output()
.await;
if let Ok(output) = probe_cmd {
if output.status.success() {
if let Ok(json_str) = String::from_utf8(output.stdout) {
if let Ok(probe_data) = serde_json::from_str::<serde_json::Value>(&json_str) {
if let Some(streams) = probe_data["streams"].as_array() {
for stream in streams {
if stream["codec_type"] == "video" {
if let Some(fps_str) = stream["r_frame_rate"].as_str() {
// Parse "30000/1001" style fps
if let Some(fps) = parse_fractional_fps(fps_str) {
let interval = (fps / 8.0).round() as i64;
return interval.max(1);
}
}
if let Some(fps_val) = stream["avg_frame_rate"].as_str() {
if let Some(fps) = parse_fractional_fps(fps_val) {
let interval = (fps / 8.0).round() as i64;
return interval.max(1);
}
}
}
}
}
}
}
}
}
4 // Default: assume 30fps / 8 = ~4
}
fn parse_fractional_fps(s: &str) -> Option<f64> {
let parts: Vec<&str> = s.split('/').collect();
if parts.len() == 2 {
let num: f64 = parts[0].parse().ok()?;
let den: f64 = parts[1].parse().ok()?;
if den > 0.0 {
return Some(num / den);
}
}
s.parse::<f64>().ok()
}
src/core/processor/mod.rs
+13 -1
View File
@@ -1,3 +1,4 @@
pub mod appearance;
pub mod asr;
pub mod asrx;
pub mod caption;
@@ -8,6 +9,7 @@ pub mod executor;
pub mod face;
pub mod face_recognition;
pub mod heuristic_scene;
pub mod mediapipe_v2;
pub mod ocr;
pub mod pose;
pub mod scene_classification;
@@ -15,11 +17,17 @@ pub mod story;
pub mod tkg;
pub mod yolo;
pub use appearance::{
process_appearance, AppearanceFrame, AppearancePerson, AppearanceResult, BBox,
};
pub use asr::{process_asr, AsrResult, AsrSegment};
pub use asrx::{process_asrx, AsrxResult, AsrxSegment};
pub use caption::{process_caption, CaptionResult, CaptionSummary, FrameCaption};
pub use cascade_vision::{CascadeDetectionResult, CascadeVisionProcessor};
pub use clip::{classify_image, classify_images, detect_objects, ClipDetectionResult, ClipImageResult, ClipPrediction};
pub use clip::{
classify_image, classify_images, detect_objects, ClipDetectionResult, ClipImageResult,
ClipPrediction,
};
pub use cut::{process_cut, CutResult, CutScene};
pub use executor::{validate_python_env, PythonExecutor, RetryConfig};
pub use face::{process_face, Face, FaceFrame, FaceResult};
@@ -32,6 +40,10 @@ pub use heuristic_scene::{
build_heuristic_scene_meta, generate_scene_meta, CrowdSize, HeuristicSceneMeta,
SceneSegmentMeta,
};
pub use mediapipe_v2::{
process_mediapipe_v2, MediaPipeBBox, MediaPipeDictEntry, MediaPipeHands, MediaPipeMetadata,
MediaPipePerson, MediaPipeResult,
};
pub use ocr::{process_ocr, OcrFrame, OcrResult, OcrText};
pub use pose::{process_pose, Bbox, Keypoint, PersonPose, PoseFrame, PoseResult};
pub use scene_classification::{
src/core/processor/ocr.rs
+3 -1
View File
@@ -34,6 +34,7 @@ pub async fn process_ocr(
video_path: &str,
output_path: &str,
uuid: Option<&str>,
frames: Option<&[i64]>,
) -> Result<OcrResult> {
let executor = PythonExecutor::new()?;
let script_path = executor.script_path("ocr_processor.py");
@@ -50,12 +51,13 @@ pub async fn process_ocr(
}
executor
.run(
.run_with_frames(
"ocr_processor.py",
&[video_path, output_path],
uuid,
"OCR",
Some(OCR_TIMEOUT),
frames,
)
.await
.with_context(|| format!("Failed to run {:?}", script_path))?;
src/core/processor/pose.rs
+3 -1
View File
@@ -46,6 +46,7 @@ pub async fn process_pose(
video_path: &str,
output_path: &str,
uuid: Option<&str>,
frames: Option<&[i64]>,
) -> Result<PoseResult> {
let executor = PythonExecutor::new()?;
let script_path = executor.script_path("pose_processor.py");
@@ -62,12 +63,13 @@ pub async fn process_pose(
}
executor
.run(
.run_with_frames(
"pose_processor.py",
&[video_path, output_path],
uuid,
"POSE",
Some(POSE_TIMEOUT),
frames,
)
.await
.with_context(|| format!("Failed to run {:?}", script_path))?;
src/core/processor/tkg.rs
+506 -36
View File
@@ -933,6 +933,145 @@ async fn build_co_occurrence_edges(
pool: &PgPool,
file_uuid: &str,
output_dir: &str,
) -> Result<usize> {
use crate::core::db::face_embedding_db::FaceEmbeddingDb;
let face_db = FaceEmbeddingDb::new();
let qdrant_embeddings = face_db.get_all_embeddings_for_file(file_uuid).await?;
if !qdrant_embeddings.is_empty() {
tracing::info!(
"[TKG-Phase2.6.1] Building co_occurrence edges from Qdrant ({} embeddings)",
qdrant_embeddings.len()
);
return build_co_occurrence_edges_from_qdrant(pool, file_uuid, output_dir, qdrant_embeddings).await;
}
tracing::info!("[TKG-Phase2.6.1] No Qdrant embeddings, falling back to PostgreSQL");
build_co_occurrence_edges_from_pg(pool, file_uuid, output_dir).await
}
async fn build_co_occurrence_edges_from_qdrant(
pool: &PgPool,
file_uuid: &str,
output_dir: &str,
qdrant_embeddings: Vec<(String, Vec<f32>, crate::core::db::face_embedding_db::FaceEmbeddingPayload)>,
) -> Result<usize> {
use crate::core::db::face_embedding_db::FaceEmbeddingPayload;
let yolo_path = Path::new(output_dir).join(format!("{}.yolo.json", file_uuid));
if !yolo_path.exists() {
return Ok(0);
}
let content = std::fs::read_to_string(&yolo_path)?;
let yolo: YoloJson = serde_json::from_str(&content)?;
let nodes_table = t("tkg_nodes");
let edges_table = t("tkg_edges");
let mut frame_faces: HashMap<i64, Vec<(i64, f64, f64, f64, f64)>> = HashMap::new();
for (_, _, payload) in &qdrant_embeddings {
let frame = payload.frame;
let trace_id = payload.trace_id as i64;
frame_faces
.entry(frame)
.or_default()
.push((trace_id, payload.bbox_x, payload.bbox_y, payload.bbox_w, payload.bbox_h));
}
let mut edge_count = 0;
for (frame, faces) in frame_faces.iter() {
let frame_str = frame.to_string();
let yolo_frame = match yolo.frames.get(&frame_str) {
Some(f) => f,
None => continue,
};
let dets = if !yolo_frame.detections.is_empty() {
&yolo_frame.detections
} else {
&yolo_frame.objects
};
if dets.is_empty() {
continue;
}
for (trace_id, _, _, _, _) in faces {
let external_id = format!("trace_{}", trace_id);
let face_node: Option<(i64,)> = sqlx::query_as(&format!(
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_trace' AND external_id=$2",
nodes_table
))
.bind(file_uuid)
.bind(&external_id)
.fetch_optional(pool)
.await?;
let face_node_id = match face_node {
Some((id,)) => id,
None => continue,
};
for det in dets {
let obj_node: Option<(i64,)> = sqlx::query_as(&format!(
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='object' AND external_id=$2",
nodes_table
))
.bind(file_uuid)
.bind(&det.class_name)
.fetch_optional(pool)
.await?;
let obj_node_id = match obj_node {
Some((id,)) => id,
None => continue,
};
let edge_props = serde_json::json!({
"frame": *frame,
"object_confidence": det.confidence,
});
if let Err(e) = sqlx::query(&format!(
r#"
INSERT INTO {} (edge_type, source_node_id, target_node_id, file_uuid, properties)
VALUES ($1, $2, $3, $4, $5::jsonb)
ON CONFLICT (file_uuid, edge_type, source_node_id, target_node_id)
DO UPDATE SET properties = COALESCE(EXCLUDED.properties, tkg_edges.properties)
"#,
edges_table
))
.bind("CO_OCCURS_WITH")
.bind(face_node_id)
.bind(obj_node_id)
.bind(file_uuid)
.bind(serde_json::to_string(&edge_props)?)
.execute(pool)
.await
{
tracing::warn!(
"[TKG-Phase2.6.1] Edge insert failed (trace={}, obj={}): {}",
trace_id,
det.class_name,
e
);
continue;
}
edge_count += 1;
}
}
}
Ok(edge_count)
}
async fn build_co_occurrence_edges_from_pg(
pool: &PgPool,
file_uuid: &str,
output_dir: &str,
) -> Result<usize> {
let yolo_path = Path::new(output_dir).join(format!("{}.yolo.json", file_uuid));
if !yolo_path.exists() {
@@ -1046,6 +1185,154 @@ async fn build_speaker_face_edges(
pool: &PgPool,
file_uuid: &str,
output_dir: &str,
) -> Result<usize> {
use crate::core::db::face_embedding_db::FaceEmbeddingDb;
let face_db = FaceEmbeddingDb::new();
let qdrant_embeddings = face_db.get_all_embeddings_for_file(file_uuid).await?;
if !qdrant_embeddings.is_empty() {
tracing::info!(
"[TKG-Phase2.6.3] Building speaker_face edges from Qdrant ({} embeddings)",
qdrant_embeddings.len()
);
return build_speaker_face_edges_from_qdrant(pool, file_uuid, output_dir, qdrant_embeddings).await;
}
tracing::info!("[TKG-Phase2.6.3] No Qdrant embeddings, falling back to PostgreSQL");
build_speaker_face_edges_from_pg(pool, file_uuid, output_dir).await
}
async fn build_speaker_face_edges_from_qdrant(
pool: &PgPool,
file_uuid: &str,
output_dir: &str,
qdrant_embeddings: Vec<(String, Vec<f32>, crate::core::db::face_embedding_db::FaceEmbeddingPayload)>,
) -> Result<usize> {
use crate::core::db::face_embedding_db::FaceEmbeddingPayload;
let asrx_path = Path::new(output_dir).join(format!("{}.asrx.json", file_uuid));
if !asrx_path.exists() {
return Ok(0);
}
let content = std::fs::read_to_string(&asrx_path)?;
let asrx: AsrxJson = serde_json::from_str(&content)?;
if asrx.segments.is_empty() {
return Ok(0);
}
let nodes_table = t("tkg_nodes");
let edges_table = t("tkg_edges");
let mut trace_ranges: HashMap<i64, (i64, i64)> = HashMap::new();
for (_, _, payload) in &qdrant_embeddings {
let trace_id = payload.trace_id as i64;
let frame = payload.frame;
let entry = trace_ranges.entry(trace_id).or_insert((frame, frame));
entry.0 = entry.0.min(frame);
entry.1 = entry.1.max(frame);
}
let last = asrx.segments.last().unwrap();
let fps = if last.end > 0.0 {
last.end_frame as f64 / last.end
} else {
30.0
};
let mut edge_count = 0;
for (tid, (sf, ef)) in &trace_ranges {
let face_ext_id = format!("trace_{}", tid);
let face_node: Option<(i64,)> = sqlx::query_as(&format!(
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_trace' AND external_id=$2",
nodes_table
))
.bind(file_uuid)
.bind(&face_ext_id)
.fetch_optional(pool)
.await?;
let face_node_id = match face_node {
Some((id,)) => id,
None => continue,
};
let face_start_sec = *sf as f64 / fps;
let face_end_sec = *ef as f64 / fps;
for seg in &asrx.segments {
let seg_start = seg.start;
let seg_end = seg.end;
let overlap_start = face_start_sec.max(seg_start);
let overlap_end = face_end_sec.min(seg_end);
if overlap_start >= overlap_end {
continue;
}
let overlap_dur = overlap_end - overlap_start;
let face_dur = face_end_sec - face_start_sec;
if face_dur <= 0.0 {
continue;
}
let overlap_ratio = overlap_dur / face_dur;
if overlap_ratio < 0.3 {
continue;
}
let speaker_node: Option<(i64,)> = sqlx::query_as(&format!(
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='speaker' AND external_id=$2",
nodes_table
))
.bind(file_uuid)
.bind(&seg.speaker_id)
.fetch_optional(pool)
.await?;
let speaker_node_id = match speaker_node {
Some((id,)) => id,
None => continue,
};
let edge_props = serde_json::json!({
"overlap_ratio": (overlap_ratio * 1000.0).round() / 1000.0,
"overlap_duration_s": (overlap_dur * 10.0).round() / 10.0,
"face_time_range": format!("{:.1}-{:.1}s", face_start_sec, face_end_sec),
"speaker_time_range": format!("{:.1}-{:.1}s", seg_start, seg_end),
});
sqlx::query(&format!(
r#"
INSERT INTO {} (edge_type, source_node_id, target_node_id, file_uuid, properties)
VALUES ($1, $2, $3, $4, $5::jsonb)
ON CONFLICT (file_uuid, edge_type, source_node_id, target_node_id)
DO UPDATE SET properties = COALESCE(EXCLUDED.properties, tkg_edges.properties)
"#,
edges_table
))
.bind("SPEAKS_AS")
.bind(face_node_id)
.bind(speaker_node_id)
.bind(file_uuid)
.bind(serde_json::to_string(&edge_props)?)
.execute(pool)
.await?;
edge_count += 1;
}
}
Ok(edge_count)
}
async fn build_speaker_face_edges_from_pg(
pool: &PgPool,
file_uuid: &str,
output_dir: &str,
) -> Result<usize> {
let asrx_path = Path::new(output_dir).join(format!("{}.asrx.json", file_uuid));
if !asrx_path.exists() {
@@ -1073,7 +1360,6 @@ async fn build_speaker_face_edges(
.fetch_all(pool)
.await?;
// Calculate fps from last segment
let last = asrx.segments.last().unwrap();
let fps = if last.end > 0.0 {
last.end_frame as f64 / last.end
@@ -1173,48 +1459,234 @@ async fn build_face_face_edges(
file_uuid: &str,
pose_data: &[FacePose],
) -> Result<usize> {
let face_table = t("face_detections");
use crate::core::db::face_embedding_db::FaceEmbeddingDb;
let face_db = FaceEmbeddingDb::new();
let qdrant_embeddings = face_db.get_all_embeddings_for_file(file_uuid).await?;
if !qdrant_embeddings.is_empty() {
tracing::info!(
"[TKG-Phase2.6.2] Building face_face edges from Qdrant ({} embeddings)",
qdrant_embeddings.len()
);
return build_face_face_edges_from_qdrant(pool, file_uuid, pose_data, qdrant_embeddings).await;
}
tracing::info!("[TKG-Phase2.6.2] No Qdrant embeddings, falling back to PostgreSQL");
build_face_face_edges_from_pg(pool, file_uuid, pose_data).await
}
async fn build_face_face_edges_from_qdrant(
pool: &PgPool,
file_uuid: &str,
pose_data: &[FacePose],
qdrant_embeddings: Vec<(String, Vec<f32>, crate::core::db::face_embedding_db::FaceEmbeddingPayload)>,
) -> Result<usize> {
use crate::core::db::face_embedding_db::FaceEmbeddingPayload;
let nodes_table = t("tkg_nodes");
let edges_table = t("tkg_edges");
// Use SQL JOIN for fast co-occurrence detection
let rows: Vec<(i64, i64, i64)> = sqlx::query_as(&format!(
r#"
SELECT a.trace_id::bigint AS tid_a, b.trace_id::bigint AS tid_b, a.frame_number::bigint
FROM {} a
JOIN {} b
ON a.file_uuid = b.file_uuid
AND a.frame_number = b.frame_number
AND a.trace_id < b.trace_id
WHERE a.file_uuid = $1
AND a.trace_id IS NOT NULL
AND b.trace_id IS NOT NULL
ORDER BY a.frame_number
"#,
face_table, face_table
))
.bind(file_uuid)
.fetch_all(pool)
.await?;
let mut frame_faces: HashMap<i64, Vec<FaceEmbeddingPayload>> = HashMap::new();
for (_, _, payload) in &qdrant_embeddings {
frame_faces.entry(payload.frame).or_default().push(payload.clone());
}
// Also load per-frame bbox for mutual_gaze lookups
let bbox_data: Vec<(i64, i64, f64, f64, f64, f64)> = sqlx::query_as(
&format!(
"SELECT trace_id::bigint, frame_number::bigint, x::float8, y::float8, width::float8, height::float8 \
FROM {} WHERE file_uuid = $1 AND trace_id IS NOT NULL ORDER BY trace_id, frame_number",
face_table
)
)
.bind(file_uuid)
.fetch_all(pool)
.await?;
let mut frame_map: HashMap<(i64, i64), (f64, f64, f64, f64)> = HashMap::new();
for (_, _, payload) in &qdrant_embeddings {
let trace_id = payload.trace_id as i64;
let frame = payload.frame;
frame_map.insert((trace_id, frame), (payload.bbox_x, payload.bbox_y, payload.bbox_w, payload.bbox_h));
}
let mut frame_map: HashMap<(i64, i64), (f64, f64, f64, f64)> = HashMap::new(); // (trace_id, frame) → (x, y, w, h)
let mut rows: Vec<(i64, i64, i64)> = Vec::new();
for (frame, faces) in frame_faces.iter() {
for i in 0..faces.len() {
for j in (i+1)..faces.len() {
let tid_a = faces[i].trace_id as i64;
let tid_b = faces[j].trace_id as i64;
let min_tid = tid_a.min(tid_b);
let max_tid = tid_a.max(tid_b);
rows.push((min_tid, max_tid, *frame));
}
}
}
let mut pair_frames: HashMap<(i64, i64), Vec<(i64, bool)>> = HashMap::new();
for (tid_a, tid_b, frame) in &rows {
let key = (*tid_a.min(tid_b), *tid_a.max(tid_b));
let bbox_a = frame_map.get(&(*tid_a, *frame));
let bbox_b = frame_map.get(&(*tid_b, *frame));
let gaze = match (bbox_a, bbox_b) {
(Some(&(xa, ya, wa, ha)), Some(&(xb, yb, wb, hb))) => {
get_pose_for_face(*frame, xa, ya, wa, ha, pose_data)
.and_then(|(yaw_a, _, _)| {
get_pose_for_face(*frame, xb, yb, wb, hb, pose_data).map(|(yaw_b, _, _)| {
detect_mutual_gaze(xa, wa, yaw_a, xb, wb, yaw_b, 0.05)
})
})
.unwrap_or(false)
}
_ => false,
};
pair_frames.entry(key).or_default().push((*frame, gaze));
}
let mut edge_count = 0;
let mut node_id_cache: HashMap<i64, i64> = HashMap::new();
for ((tid_a, tid_b), frame_data) in &pair_frames {
let ext_a = format!("trace_{}", tid_a);
let ext_b = format!("trace_{}", tid_b);
let n_a_id = match node_id_cache.get(tid_a) {
Some(id) => *id,
None => {
if let Some((id,)) = sqlx::query_as::<_, (i64,)>(&format!(
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_trace' AND external_id=$2",
nodes_table
))
.bind(file_uuid).bind(&ext_a).fetch_optional(pool).await?
{
node_id_cache.insert(*tid_a, id);
id
} else { continue; }
}
};
let n_b_id = match node_id_cache.get(tid_b) {
Some(id) => *id,
None => {
if let Some((id,)) = sqlx::query_as::<_, (i64,)>(&format!(
"SELECT id FROM {} WHERE file_uuid=$1 AND node_type='face_trace' AND external_id=$2",
nodes_table
))
.bind(file_uuid).bind(&ext_b).fetch_optional(pool).await?
{
node_id_cache.insert(*tid_b, id);
id
} else { continue; }
}
};
let frames: Vec<i64> = frame_data.iter().map(|(f, _)| *f).collect();
let gaze_frames: Vec<i64> = frame_data
.iter()
.filter(|(_, g)| *g)
.map(|(f, _)| *f)
.collect();
let gaze_count = gaze_frames.len() as i64;
let has_gaze = gaze_count > 0;
let edge_props = if has_gaze {
let mut yaw_a_sum = 0.0f64;
let mut yaw_b_sum = 0.0f64;
let mut gaze_sample = 0i64;
for (frame, _) in frame_data.iter().filter(|(_, g)| *g) {
let bbox_a = frame_map.get(&(*tid_a, *frame));
let bbox_b = frame_map.get(&(*tid_b, *frame));
if let (Some(&(xa, ya, wa, ha)), Some(&(xb, yb, wb, hb))) = (bbox_a, bbox_b) {
let pose_a = get_pose_for_face(*frame, xa, ya, wa, ha, pose_data);
let pose_b = get_pose_for_face(*frame, xb, yb, wb, hb, pose_data);
if let (Some((ya, _, _)), Some((yb, _, _))) = (pose_a, pose_b) {
yaw_a_sum += ya;
yaw_b_sum += yb;
gaze_sample += 1;
}
}
}
let (avg_ya, avg_yb) = if gaze_sample > 0 {
(
yaw_a_sum / gaze_sample as f64,
yaw_b_sum / gaze_sample as f64,
)
} else {
(0.0, 0.0)
};
serde_json::json!({
"first_frame": frames[0],
"frame_count": frames.len() as i64,
"mutual_gaze": true,
"gaze_frame_count": gaze_count,
"yaw_a_avg": (avg_ya * 1000.0).round() / 1000.0,
"yaw_b_avg": (avg_yb * 1000.0).round() / 1000.0,
})
} else {
serde_json::json!({
"first_frame": frames[0],
"frame_count": frames.len() as i64,
"mutual_gaze": false,
})
};
sqlx::query(&format!(
r#"
INSERT INTO {} (edge_type, source_node_id, target_node_id, file_uuid, properties)
VALUES ($1, $2, $3, $4, $5::jsonb)
ON CONFLICT (file_uuid, edge_type, source_node_id, target_node_id)
DO UPDATE SET properties = COALESCE(EXCLUDED.properties, tkg_edges.properties)
"#,
edges_table
))
.bind("CO_OCCURS_WITH")
.bind(n_a_id)
.bind(n_b_id)
.bind(file_uuid)
.bind(serde_json::to_string(&edge_props)?)
.execute(pool)
.await?;
edge_count += 1;
}
Ok(edge_count)
}
async fn build_face_face_edges_from_pg(
pool: &PgPool,
file_uuid: &str,
pose_data: &[FacePose],
) -> Result<usize> {
let face_table = t("face_detections");
let nodes_table = t("tkg_nodes");
let edges_table = t("tkg_edges");
let rows: Vec<(i64, i64, i64)> = sqlx::query_as(&format!(
r#"
SELECT a.trace_id::bigint AS tid_a, b.trace_id::bigint AS tid_b, a.frame_number::bigint
FROM {} a
JOIN {} b
ON a.file_uuid = b.file_uuid
AND a.frame_number = b.frame_number
AND a.trace_id < b.trace_id
WHERE a.file_uuid = $1
AND a.trace_id IS NOT NULL
AND b.trace_id IS NOT NULL
ORDER BY a.frame_number
"#,
face_table, face_table
))
.bind(file_uuid)
.fetch_all(pool)
.await?;
let bbox_data: Vec<(i64, i64, f64, f64, f64, f64)> = sqlx::query_as(
&format!(
"SELECT trace_id::bigint, frame_number::bigint, x::float8, y::float8, width::float8, height::float8 \
FROM {} WHERE file_uuid = $1 AND trace_id IS NOT NULL ORDER BY trace_id, frame_number",
face_table
)
)
.bind(file_uuid)
.fetch_all(pool)
.await?;
let mut frame_map: HashMap<(i64, i64), (f64, f64, f64, f64)> = HashMap::new();
for (tid, frame, x, y, w, h) in &bbox_data {
frame_map.insert((*tid, *frame), (*x, *y, *w, *h));
}
// Group by pair
let mut pair_frames: HashMap<(i64, i64), Vec<(i64, bool)>> = HashMap::new();
for (tid_a, tid_b, frame) in &rows {
let key = (*tid_a.min(tid_b), *tid_a.max(tid_b));
@@ -1237,7 +1709,6 @@ async fn build_face_face_edges(
}
let mut edge_count = 0;
// Cache node IDs to avoid repeated queries
let mut node_id_cache: HashMap<i64, i64> = HashMap::new();
for ((tid_a, tid_b), frame_data) in &pair_frames {
let ext_a = format!("trace_{}", tid_a);
@@ -1283,7 +1754,6 @@ async fn build_face_face_edges(
let has_gaze = gaze_count > 0;
let edge_props = if has_gaze {
// Compute average yaw values for gaze frames
let mut yaw_a_sum = 0.0f64;
let mut yaw_b_sum = 0.0f64;
let mut gaze_sample = 0i64;
src/core/processor/yolo.rs
+3 -1
View File
@@ -120,6 +120,7 @@ pub async fn process_yolo(
video_path: &str,
output_path: &str,
uuid: Option<&str>,
frames: Option<&[i64]>,
) -> Result<YoloResult> {
let executor = PythonExecutor::new()?;
let script_path = executor.script_path("yolo_processor.py");
@@ -136,12 +137,13 @@ pub async fn process_yolo(
}
executor
.run(
.run_with_frames(
"yolo_processor.py",
&[video_path, output_path],
uuid,
"YOLO",
Some(YOLO_TIMEOUT),
frames,
)
.await
.with_context(|| format!("Failed to run {:?}", script_path))?;