warren/momentry_core
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

feat: ASRX hybrid pipeline, identity history, worker fixes, checkpoint system

Browse Source
This commit is contained in:
Accusys
2026-06-02 07:13:23 +08:00
parent e3066c3f49
commit e1572907ae
198 changed files with 43705 additions and 8910 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/chunk/rule3_ingest.rs
+1 -15
View File
@@ -1,6 +1,5 @@
use crate::core::config::OUTPUT_DIR;
use crate::core::db::schema;
use crate::core::llm::client::generate_5w1h_summary;
use anyhow::{Context, Result};
use serde::Deserialize;
use sqlx::PgPool;
@@ -115,19 +114,6 @@ pub async fn ingest_rule3(pool: &PgPool, file_uuid: &str) -> Result<usize> {
let aggregated_text = texts.join(" ");
// 3. Call LLM for Summary
let summary = if !aggregated_text.is_empty() {
match generate_5w1h_summary(&aggregated_text).await {
Ok(s) => s,
Err(e) => {
warn!("LLM Summary failed for scene {}: {}", scene.scene_number, e);
"LLM Error".to_string()
}
}
} else {
"No Audio".to_string()
};
info!(
"Scene {}: {} -> {} ({} sentences)",
scene.scene_number,
@@ -168,7 +154,7 @@ pub async fn ingest_rule3(pool: &PgPool, file_uuid: &str) -> Result<usize> {
.bind(scene.end_frame as i64)
.bind(&metadata)
.bind(&aggregated_text)
.bind(&summary)
.bind(&String::new())
.bind(&metadata)
.bind(&child_ids)
.execute(&mut *tx)
src/core/chunk/types.rs
+4 -345
View File
@@ -1,7 +1,6 @@
use crate::core::time::FrameTime;
use serde::{Deserialize, Serialize};
// ==================== ChunkType ====================
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "snake_case")]
pub enum ChunkType {
@@ -10,7 +9,6 @@ pub enum ChunkType {
Cut,
Trace,
Story,
Visual, // 視覺分片 (Phase 2.1)
}
impl ChunkType {
@@ -21,17 +19,15 @@ impl ChunkType {
ChunkType::Cut => "cut",
ChunkType::Trace => "trace",
ChunkType::Story => "story",
ChunkType::Visual => "visual",
}
}
}
// ==================== ChunkRule ====================
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "snake_case")]
pub enum ChunkRule {
Rule1, // 直接轉換
Rule2, // 集合內容
Rule1,
Rule2,
}
impl ChunkRule {
@@ -43,73 +39,6 @@ impl ChunkRule {
}
}
// ==================== 視覺分片相關結構 (Phase 2.1) ====================
/// 邊界框
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BoundingBox {
pub x: i32,
pub y: i32,
pub width: i32,
pub height: i32,
}
/// 檢測到的物件
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DetectedObject {
/// 物件類別名稱
pub class_name: String,
/// 物件類別 ID
pub class_id: u32,
/// 信心值 (0.0-1.0)
pub confidence: f32,
/// 邊界框
pub bbox: Option<BoundingBox>,
/// 出現次數 (在分片內)
pub occurrence: u32,
}
/// 關鍵幀的物件列表
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct KeyframeObjects {
/// 關鍵幀時間 (秒) - 僅供參考，主要使用 frame_number
pub timestamp: f64,
/// 關鍵幀幀號 - 主要時間標示
pub frame_number: u64,
/// 檢測到的物件
pub objects: Vec<DetectedObject>,
}
/// 視覺元數據
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VisualMetadata {
/// 總物件數量
pub object_count: u32,
/// 唯一物件類別列表
pub unique_classes: Vec<String>,
/// 最高信心值
pub max_confidence: f32,
/// 平均信心值
pub avg_confidence: f32,
/// 空間密度（每幀平均物件數）
pub spatial_density: f32,
}
/// 視覺分片內容
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VisualChunkContent {
/// 關鍵幀物件列表，每個關鍵幀包含 frame_number
pub keyframe_objects: Vec<KeyframeObjects>,
/// 主要物件標籤（出現在大多數幀中的物件）
pub dominant_objects: Vec<String>,
/// 物件關係 (object1, relationship, object2) - 可選
pub object_relationships: Vec<(String, String, String)>,
/// 場景描述 - 可選
pub scene_description: Option<String>,
/// 視覺元數據
pub metadata: VisualMetadata,
}
// ==================== Chunk 主結構 ====================
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Chunk {
pub file_id: i32,
@@ -117,11 +46,8 @@ pub struct Chunk {
pub chunk_id: String,
pub chunk_type: ChunkType,
pub rule: ChunkRule,
/// Frames per second (can be fractional, e.g., 29.97, 23.976)
pub fps: f64,
/// Start frame (0-based) - 主要時間標示
pub start_frame: i64,
/// End frame (exclusive) - 主要時間標示
pub end_frame: i64,
pub text_content: Option<String>,
pub content: serde_json::Value,
@@ -129,13 +55,11 @@ pub struct Chunk {
pub vector_id: Option<String>,
pub frame_count: i32,
pub pre_chunk_ids: Vec<i32>,
pub parent_chunk_id: Option<String>, // For parent-child chunk hierarchy
pub child_chunk_ids: Vec<String>, // Child chunk IDs (for parent chunks)
pub visual_stats: Option<serde_json::Value>,
pub parent_chunk_id: Option<String>,
pub child_chunk_ids: Vec<String>,
}
impl Chunk {
/// 創建新分片
pub fn new(
file_id: i32,
uuid: String,
@@ -166,167 +90,17 @@ impl Chunk {
pre_chunk_ids: vec![],
parent_chunk_id: None,
child_chunk_ids: vec![],
visual_stats: None,
}
}
/// 創建視覺分片 (Phase 2.1)
pub fn new_visual(
file_id: i32,
uuid: String,
chunk_id: String,
start_frame: i64,
end_frame: i64,
fps: f64,
visual_content: VisualChunkContent,
) -> Self {
let content = serde_json::to_value(&visual_content)
.unwrap_or_else(|_| serde_json::json!({"error": "Failed to serialize visual content"}));
Self::new(
file_id,
uuid,
chunk_id,
ChunkType::Visual,
ChunkRule::Rule2,
start_frame,
end_frame,
fps,
content,
)
}
/// 從 YOLO 幀創建視覺分片 (Phase 2.1)
pub fn from_yolo_frames(
file_id: i32,
uuid: String,
chunk_id: String,
start_frame: i64,
end_frame: i64,
fps: f64,
yolo_frames: Vec<crate::core::processor::yolo::YoloFrame>,
) -> Self {
// 將 YOLO 幀轉換為關鍵幀物件
let keyframe_objects: Vec<KeyframeObjects> = yolo_frames
.iter()
.map(|frame| {
let objects: Vec<DetectedObject> = frame
.objects
.iter()
.map(|obj| DetectedObject {
class_name: obj.class_name.clone(),
class_id: obj.class_id,
confidence: obj.confidence,
bbox: Some(BoundingBox {
x: obj.x,
y: obj.y,
width: obj.width,
height: obj.height,
}),
occurrence: 1,
})
.collect();
KeyframeObjects {
timestamp: frame.timestamp,
frame_number: frame.frame,
objects,
}
})
.collect();
// 計算物件統計
let total_objects: u32 = yolo_frames.iter().map(|f| f.objects.len() as u32).sum();
// 收集所有物件類別
let all_classes: Vec<String> = yolo_frames
.iter()
.flat_map(|f| f.objects.iter().map(|o| o.class_name.clone()))
.collect();
// 獲取唯一類別
let unique_classes: Vec<String> = all_classes
.iter()
.cloned()
.collect::<std::collections::HashSet<_>>()
.into_iter()
.collect();
// 計算信心值統計
let confidences: Vec<f32> = yolo_frames
.iter()
.flat_map(|f| f.objects.iter().map(|o| o.confidence))
.collect();
let max_confidence = confidences.iter().copied().fold(0.0f32, f32::max);
let avg_confidence = if !confidences.is_empty() {
confidences.iter().sum::<f32>() / confidences.len() as f32
} else {
0.0
};
// 計算主要物件（出現在大多數幀中的物件）
let mut object_counts = std::collections::HashMap::new();
for frame in &yolo_frames {
let frame_classes: std::collections::HashSet<_> =
frame.objects.iter().map(|o| o.class_name.clone()).collect();
for class in frame_classes {
*object_counts.entry(class).or_insert(0) += 1;
}
}
let mut dominant_objects: Vec<String> = object_counts
.into_iter()
.filter(|(_, count)| *count as f32 / yolo_frames.len() as f32 > 0.5)
.map(|(class, _)| class)
.collect();
dominant_objects.sort();
// 創建視覺內容
let visual_content = VisualChunkContent {
keyframe_objects,
dominant_objects,
object_relationships: vec![], // 可選：後期添加關係檢測
scene_description: None, // 可選：後期添加 LLM 生成的場景描述
metadata: VisualMetadata {
object_count: total_objects,
unique_classes,
max_confidence,
avg_confidence,
spatial_density: if yolo_frames.len() > 0 {
total_objects as f32 / yolo_frames.len() as f32
} else {
0.0
},
},
};
Self::new_visual(
file_id,
uuid,
chunk_id,
start_frame,
end_frame,
fps,
visual_content,
)
}
/// 將分片轉換為幀時間
pub fn to_frame_time(&self) -> FrameTime {
// 使用第一個幀作為參考點
FrameTime::from_frames(self.start_frame, self.fps)
}
/// 檢查是否是父分片
pub fn is_parent(&self) -> bool {
self.parent_chunk_id.is_some()
}
/// 從秒數創建新分片（舊版轉換）
///
/// 這對於從存儲時間為秒的舊系統遷移很有用。
/// 幀數通過舍入 `seconds * fps` 計算。
#[allow(clippy::too_many_arguments)]
pub fn from_seconds(
file_id: i32,
@@ -354,197 +128,82 @@ impl Chunk {
)
}
/// 返回開始時間為 `FrameTime`
pub fn start_time(&self) -> FrameTime {
FrameTime::from_frames(self.start_frame, self.fps)
}
/// 返回結束時間為 `FrameTime`
pub fn end_time(&self) -> FrameTime {
FrameTime::from_frames(self.end_frame, self.fps)
}
/// 返回持續時間的幀數
pub fn duration_frames(&self) -> i64 {
self.end_frame - self.start_frame
}
/// 返回持續時間的秒數
pub fn duration_seconds(&self) -> f64 {
self.duration_frames() as f64 / self.fps
}
/// 將開始時間格式化為 "seconds.frame" (例如："123.04")
pub fn format_start_sec_frame(&self) -> String {
self.start_time().format_sec_frame()
}
/// 將結束時間格式化為 "seconds.frame" (例如："456.15")
pub fn format_end_sec_frame(&self) -> String {
self.end_time().format_sec_frame()
}
/// 將開始時間格式化為 "HH:MM:SS"
pub fn format_start_hms(&self) -> String {
self.start_time().format_hms()
}
/// 將結束時間格式化為 "HH:MM:SS"
pub fn format_end_hms(&self) -> String {
self.end_time().format_hms()
}
/// 將開始時間格式化為 "HH:MM:SS.FF"
pub fn format_start_hms_frame(&self) -> String {
self.start_time().format_hms_frame()
}
/// 將結束時間格式化為 "HH:MM:SS.FF"
pub fn format_end_hms_frame(&self) -> String {
self.end_time().format_hms_frame()
}
/// 返回 (start_seconds, end_seconds) 元組用於兼容性
///
/// 這在遷移期間提供向後兼容性。
/// 建議使用 `start_time()` 和 `end_time()` 方法。
pub fn time_range_seconds(&self) -> (f64, f64) {
(self.start_time().seconds(), self.end_time().seconds())
}
/// 添加元數據
pub fn with_metadata(mut self, metadata: serde_json::Value) -> Self {
self.metadata = Some(metadata);
self
}
/// 添加向量 ID
pub fn with_vector_id(mut self, vector_id: String) -> Self {
self.vector_id = Some(vector_id);
self
}
/// 添加文本內容
pub fn with_text_content(mut self, text: String) -> Self {
self.text_content = Some(text);
self
}
/// 設置幀數
pub fn with_frame_count(mut self, count: i32) -> Self {
self.frame_count = count;
self
}
/// 設置前一個分片 ID
pub fn with_pre_chunk_ids(mut self, ids: Vec<i32>) -> Self {
self.pre_chunk_ids = ids;
self
}
/// 設置父分片 ID
pub fn with_parent_chunk_id(mut self, parent_id: String) -> Self {
self.parent_chunk_id = Some(parent_id);
self
}
/// 設置子分片 ID
pub fn with_child_chunk_ids(mut self, child_ids: Vec<String>) -> Self {
self.child_chunk_ids = child_ids;
self
}
}
// ==================== VisualChunkContent 輔助方法 ====================
impl VisualChunkContent {
/// 計算兩個 YOLO 幀之間的相似度（基於物件組成）
pub fn frame_similarity(
frame1: &crate::core::processor::yolo::YoloFrame,
frame2: &crate::core::processor::yolo::YoloFrame,
) -> f32 {
if frame1.objects.is_empty() && frame2.objects.is_empty() {
return 1.0; // 兩個空幀完全相似
}
if frame1.objects.is_empty() || frame2.objects.is_empty() {
return 0.0; // 一個空一個非空，不相似
}
// 創建物件類別名稱集合
let set1: std::collections::HashSet<String> = frame1
.objects
.iter()
.map(|o| o.class_name.clone())
.collect();
let set2: std::collections::HashSet<String> = frame2
.objects
.iter()
.map(|o| o.class_name.clone())
.collect();
// 計算 Jaccard 相似度
let intersection: Vec<_> = set1.intersection(&set2).collect();
let union: Vec<_> = set1.union(&set2).collect();
if union.is_empty() {
0.0
} else {
intersection.len() as f32 / union.len() as f32
}
}
/// 獲取視覺分片的摘要（使用關鍵幀的 frame_number）
pub fn summary(&self, fps: f64) -> String {
if self.keyframe_objects.is_empty() {
return "Empty visual chunk".to_string();
}
let first_frame = self.keyframe_objects.first().unwrap().frame_number;
let last_frame = self.keyframe_objects.last().unwrap().frame_number;
// 計算時間（僅供參考）
let start_time = if fps > 0.0 {
first_frame as f64 / fps
} else {
0.0
};
let end_time = if fps > 0.0 {
last_frame as f64 / fps
} else {
0.0
};
let duration = end_time - start_time;
let frame_count = self.keyframe_objects.len();
format!(
"Visual chunk: frames {} to {} (duration: {:.1}s, {} frames). Objects: {} total, {} unique. Dominant: {}",
first_frame,
last_frame,
duration,
frame_count,
self.metadata.object_count,
self.metadata.unique_classes.len(),
if self.dominant_objects.is_empty() {
"none".to_string()
} else {
self.dominant_objects.join(", ")
}
)
}
/// 檢查是否包含特定物件類別
pub fn contains_object(&self, class_name: &str) -> bool {
self.keyframe_objects
.iter()
.any(|ko| ko.objects.iter().any(|obj| obj.class_name == class_name))
}
/// 獲取信心值高於閾值的所有物件
pub fn high_confidence_objects(&self, threshold: f32) -> Vec<&DetectedObject> {
self.keyframe_objects
.iter()
.flat_map(|ko| ko.objects.iter())
.filter(|obj| obj.confidence >= threshold)
.collect()
}
}
src/core/config.rs
+7 -5
View File
@@ -56,7 +56,7 @@ pub static REDIS_URL: Lazy<String> = Lazy::new(|| {
env::var("REDIS_URL").unwrap_or_else(|_| {
let password = env::var("REDIS_PASSWORD").unwrap_or_else(|_| "accusys".to_string());
// Format: redis://[:password]@host:port (use default user)
format!("redis://:{}@localhost:6379", password)
format!("redis://default:{}@localhost:6379", password)
})
});
@@ -277,12 +277,14 @@ pub mod llm {
}
/// Ollama embedding endpoint (vector embeddings for text sync).
pub static OLLAMA_URL: Lazy<String> =
Lazy::new(|| env::var("MOMENTRY_OLLAMA_URL").unwrap_or_else(|_| "http://127.0.0.1:11434".to_string()));
pub static OLLAMA_URL: Lazy<String> = Lazy::new(|| {
env::var("MOMENTRY_OLLAMA_URL").unwrap_or_else(|_| "http://127.0.0.1:11434".to_string())
});
/// Text embedding server (comic-embed or alternative).
pub static EMBED_URL: Lazy<String> =
Lazy::new(|| env::var("MOMENTRY_EMBED_URL").unwrap_or_else(|_| "http://127.0.0.1:11436".to_string()));
pub static EMBED_URL: Lazy<String> = Lazy::new(|| {
env::var("MOMENTRY_EMBED_URL").unwrap_or_else(|_| "http://127.0.0.1:11436".to_string())
});
/// LLM health endpoint.
pub static LLM_HEALTH_URL: Lazy<String> = Lazy::new(|| {
src/core/db/identity_merge_history.rs
+604
View File
@@ -0,0 +1,604 @@
use anyhow::{Context, Result};
use bson::{doc, oid::ObjectId, DateTime as BsonDateTime, Document};
use chrono::{DateTime, Utc};
use mongodb::{Client, Collection, Database, IndexModel};
use serde::{Deserialize, Serialize};
use serde_json::Value as JsonValue;
use uuid::Uuid;
const COLLECTION_NAME: &str = "identity_merge_history";
fn bson_doc_to_json(doc: &Document) -> JsonValue {
match bson::to_bson(doc) {
Ok(bson) => bson.into_relaxed_extjson(),
Err(_) => JsonValue::Null,
}
}
fn json_value_to_bson_doc(value: &JsonValue) -> Document {
bson::to_document(value).unwrap_or_default()
}
fn doc_field_to_json(doc: &Document, key: &str) -> JsonValue {
doc.get(key)
.map(|b| b.clone().into_relaxed_extjson())
.unwrap_or(JsonValue::Null)
}
fn json_to_bson(value: &JsonValue) -> bson::Bson {
bson::to_bson(value).unwrap_or(bson::Bson::Null)
}
#[derive(Debug, Clone)]
pub struct IdentityMergeHistory {
pub id: Option<ObjectId>,
pub merge_id: String,
pub source_identity: IdentitySnapshot,
pub target_identity: TargetIdentitySnapshot,
pub aliases_added_to_target: Vec<AliasEntry>,
pub metadata_fields_added: Vec<String>,
pub faces_transferred: FacesTransferred,
pub merge_params: MergeParams,
pub merged_at: DateTime<Utc>,
pub undo_deadline: DateTime<Utc>,
pub undone: bool,
pub undone_at: Option<DateTime<Utc>>,
pub undone_by: Option<String>,
pub undone_snapshot: Option<UndoneSnapshot>,
pub undo_expired: bool,
}
#[derive(Debug, Clone)]
pub struct IdentitySnapshot {
pub id: i64,
pub uuid: String,
pub name: String,
pub identity_type: Option<String>,
pub source: Option<String>,
pub status: String,
pub tmdb_id: Option<i64>,
pub tmdb_profile: Option<String>,
pub metadata: JsonValue,
pub created_at: Option<DateTime<Utc>>,
pub face_count: i64,
}
#[derive(Debug, Clone)]
pub struct TargetIdentitySnapshot {
pub id: i64,
pub uuid: String,
pub name: String,
pub metadata_before: JsonValue,
pub metadata_after: Option<JsonValue>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AliasEntry {
pub name: String,
pub locale: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub source: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FacesTransferred {
pub file_uuid: String,
pub face_ids: Vec<String>,
pub trace_ids: Vec<i32>,
pub count: i64,
}
#[derive(Debug, Clone)]
pub struct UndoneSnapshot {
pub source_identity_id: i64,
pub source_uuid: String,
pub source_name: String,
pub target_metadata_at_undo: JsonValue,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MergeParams {
pub keep_history: bool,
pub cleared_stranger_id: bool,
pub performed_by_user: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MergeHistoryQuery {
pub source_uuid: Option<String>,
pub target_uuid: Option<String>,
pub merge_id: Option<String>,
pub undone: Option<bool>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MergeHistoryEntry {
pub merge_id: String,
pub source_name: String,
pub target_name: String,
pub faces_transferred: i64,
pub merged_at: DateTime<Utc>,
pub undo_deadline: DateTime<Utc>,
pub undone: bool,
pub undo_expired: bool,
}
impl IdentityMergeHistory {
pub fn from_document(doc: &Document) -> Result<Self> {
let source = doc
.get_document("source_identity")
.context("Missing source_identity")?;
let target = doc
.get_document("target_identity")
.context("Missing target_identity")?;
let faces = doc
.get_document("faces_transferred")
.context("Missing faces_transferred")?;
let aliases = doc
.get_array("aliases_added_to_target")
.unwrap_or(&vec![])
.clone();
let fields = doc
.get_array("metadata_fields_added")
.unwrap_or(&vec![])
.clone();
let merge_params_doc = doc
.get_document("merge_params")
.unwrap_or(&Document::new())
.clone();
let mut parsed_aliases = Vec::new();
for a in aliases {
if let Some(d) = a.as_document() {
parsed_aliases.push(AliasEntry {
name: d.get_str("name").unwrap_or("").to_string(),
locale: d.get_str("locale").unwrap_or("en").to_string(),
source: d.get_str("source").ok().map(|s| s.to_string()),
});
}
}
let mut parsed_fields = Vec::new();
for f in fields {
if let Some(s) = f.as_str() {
parsed_fields.push(s.to_string());
}
}
let undone_snapshot = doc.get_document("undone_snapshot").ok().and_then(|d| {
let sid = d.get_i64("source_identity_id").unwrap_or(0);
let suuid = d.get_str("source_uuid").unwrap_or("").to_string();
let sname = d.get_str("source_name").unwrap_or("").to_string();
let meta = doc_field_to_json(d, "target_metadata_at_undo");
Some(UndoneSnapshot {
source_identity_id: sid,
source_uuid: suuid,
source_name: sname,
target_metadata_at_undo: meta,
})
});
Ok(IdentityMergeHistory {
id: doc.get_object_id("_id").ok(),
merge_id: doc.get_str("merge_id").unwrap_or("").to_string(),
source_identity: IdentitySnapshot {
id: source.get_i64("id").unwrap_or(0),
uuid: source.get_str("uuid").unwrap_or("").to_string(),
name: source.get_str("name").unwrap_or("").to_string(),
identity_type: source.get_str("identity_type").ok().map(|s| s.to_string()),
source: source.get_str("source").ok().map(|s| s.to_string()),
status: source.get_str("status").unwrap_or("").to_string(),
tmdb_id: source.get_i64("tmdb_id").ok(),
tmdb_profile: source.get_str("tmdb_profile").ok().map(|s| s.to_string()),
metadata: doc_field_to_json(source, "metadata"),
created_at: source
.get_datetime("created_at")
.map(|d| d.to_chrono())
.ok(),
face_count: source.get_i64("face_count").unwrap_or(0),
},
target_identity: TargetIdentitySnapshot {
id: target.get_i64("id").unwrap_or(0),
uuid: target.get_str("uuid").unwrap_or("").to_string(),
name: target.get_str("name").unwrap_or("").to_string(),
metadata_before: doc_field_to_json(target, "metadata_before"),
metadata_after: target
.get("metadata_after")
.map(|b| b.clone().into_relaxed_extjson()),
},
aliases_added_to_target: parsed_aliases,
metadata_fields_added: parsed_fields,
faces_transferred: FacesTransferred {
file_uuid: faces.get_str("file_uuid").unwrap_or("").to_string(),
face_ids: faces
.get_array("face_ids")
.map(|arr| {
arr.iter()
.filter_map(|b| b.as_str().map(|s| s.to_string()))
.collect()
})
.unwrap_or_default(),
trace_ids: faces
.get_array("trace_ids")
.map(|arr| arr.iter().filter_map(|b| b.as_i32()).collect())
.unwrap_or_default(),
count: faces.get_i64("count").unwrap_or(0),
},
merge_params: MergeParams {
keep_history: merge_params_doc.get_bool("keep_history").unwrap_or(true),
cleared_stranger_id: merge_params_doc
.get_bool("cleared_stranger_id")
.unwrap_or(true),
performed_by_user: merge_params_doc
.get_str("performed_by_user")
.ok()
.map(|s| s.to_string()),
},
merged_at: doc
.get_datetime("merged_at")
.map(|d| d.to_chrono())
.unwrap_or_default(),
undo_deadline: doc
.get_datetime("undo_deadline")
.map(|d| d.to_chrono())
.unwrap_or_default(),
undone: doc.get_bool("undone").unwrap_or(false),
undone_at: doc.get_datetime("undone_at").map(|d| d.to_chrono()).ok(),
undone_by: doc.get_str("undone_by").ok().map(|s| s.to_string()),
undone_snapshot,
undo_expired: doc.get_bool("undo_expired").unwrap_or(false),
})
}
pub fn to_document(&self) -> Document {
let mut doc = doc! {
"merge_id": &self.merge_id,
"source_identity": {
"id": self.source_identity.id as i64,
"uuid": &self.source_identity.uuid,
"name": &self.source_identity.name,
"identity_type": self.source_identity.identity_type.as_deref(),
"source": self.source_identity.source.as_deref(),
"status": &self.source_identity.status,
"tmdb_id": self.source_identity.tmdb_id,
"tmdb_profile": self.source_identity.tmdb_profile.as_deref(),
"metadata": json_to_bson(&self.source_identity.metadata),
"created_at": self.source_identity.created_at
.map(|dt| BsonDateTime::from_chrono(dt)),
"face_count": self.source_identity.face_count,
},
"target_identity": {
"id": self.target_identity.id as i64,
"uuid": &self.target_identity.uuid,
"name": &self.target_identity.name,
"metadata_before": json_to_bson(&self.target_identity.metadata_before),
"metadata_after": self.target_identity.metadata_after.as_ref().map(json_to_bson),
},
"aliases_added_to_target": self.aliases_added_to_target.iter().map(|a| {
doc! {
"name": &a.name,
"locale": &a.locale,
"source": a.source.as_deref(),
}
}).collect::<Vec<Document>>(),
"metadata_fields_added": &self.metadata_fields_added,
"faces_transferred": {
"file_uuid": &self.faces_transferred.file_uuid,
"face_ids": &self.faces_transferred.face_ids,
"trace_ids": &self.faces_transferred.trace_ids,
"count": self.faces_transferred.count,
},
"merge_params": {
"keep_history": self.merge_params.keep_history,
"cleared_stranger_id": self.merge_params.cleared_stranger_id,
"performed_by_user": self.merge_params.performed_by_user.as_deref(),
},
"merged_at": BsonDateTime::from_chrono(self.merged_at),
"undo_deadline": BsonDateTime::from_chrono(self.undo_deadline),
"undone": self.undone,
"undone_at": self.undone_at.map(|dt| BsonDateTime::from_chrono(dt)),
"undone_by": self.undone_by.as_deref(),
"undone_snapshot": self.undone_snapshot.as_ref().map(|s| {
doc! {
"source_identity_id": s.source_identity_id,
"source_uuid": &s.source_uuid,
"source_name": &s.source_name,
"target_metadata_at_undo": json_to_bson(&s.target_metadata_at_undo),
}
}),
"undo_expired": self.undo_expired,
};
if let Some(ref oid) = self.id {
doc.insert("_id", oid.clone());
}
doc
}
}
#[derive(Clone)]
pub struct IdentityMergeHistoryStore {
client: Client,
db: Database,
collection: Collection<Document>,
}
impl IdentityMergeHistoryStore {
pub async fn init() -> Result<Self> {
let uri = crate::core::config::MONGODB_URL.as_str();
let client = Client::with_uri_str(uri)
.await
.context("Failed to connect to MongoDB")?;
let db_name = crate::core::config::MONGODB_DATABASE.as_str();
let db = client.database(db_name);
let collection: Collection<Document> = db.collection(COLLECTION_NAME);
let store = Self {
client,
db,
collection,
};
store.ensure_indexes().await?;
Ok(store)
}
async fn ensure_indexes(&self) -> Result<()> {
let merge_id_index = IndexModel::builder()
.keys(doc! { "merge_id": 1 })
.options(
mongodb::options::IndexOptions::builder()
.unique(true)
.build(),
)
.build();
let merged_at_index = IndexModel::builder().keys(doc! { "merged_at": -1 }).build();
let source_uuid_index = IndexModel::builder()
.keys(doc! { "source_identity.uuid": 1 })
.build();
let target_uuid_index = IndexModel::builder()
.keys(doc! { "target_identity.uuid": 1 })
.build();
self.collection
.create_indexes(
[
merge_id_index,
merged_at_index,
source_uuid_index,
target_uuid_index,
],
None,
)
.await
.context("Failed to create identity_merge_history indexes")?;
tracing::info!("MongoDB identity_merge_history indexes ensured");
Ok(())
}
pub fn generate_merge_id() -> String {
Uuid::new_v4().to_string()
}
pub async fn store_merge_history(&self, history: &IdentityMergeHistory) -> Result<()> {
let doc = history.to_document();
self.collection
.insert_one(doc, None)
.await
.context("Failed to store merge history in MongoDB")?;
tracing::info!(
"Stored merge history: merge_id={}, source={}, target={}, faces={}",
history.merge_id,
history.source_identity.name,
history.target_identity.name,
history.faces_transferred.count
);
Ok(())
}
pub async fn get_merge_history(&self, merge_id: &str) -> Result<Option<IdentityMergeHistory>> {
let filter = doc! { "merge_id": merge_id };
let result = self
.collection
.find_one(filter, None)
.await
.context("Failed to get merge history from MongoDB")?;
match result {
Some(doc) => {
let history = IdentityMergeHistory::from_document(&doc)
.context("Failed to parse merge history from MongoDB")?;
Ok(Some(history))
}
None => Ok(None),
}
}
pub async fn query_merge_history(
&self,
query: MergeHistoryQuery,
page: u32,
page_size: u32,
) -> Result<(Vec<MergeHistoryEntry>, u64)> {
let mut filter = doc! {};
if let Some(source_uuid) = query.source_uuid {
filter.insert("source_identity.uuid", source_uuid);
}
if let Some(target_uuid) = query.target_uuid {
filter.insert("target_identity.uuid", target_uuid);
}
if let Some(merge_id) = query.merge_id {
filter.insert("merge_id", merge_id);
}
if let Some(undone) = query.undone {
filter.insert("undone", undone);
}
let skip = (page - 1) * page_size;
let limit = page_size;
let mut cursor = self
.collection
.find(filter.clone(), None)
.await
.context("Failed to query merge history")?;
let total = self
.collection
.count_documents(filter, None)
.await
.context("Failed to count merge history")?;
let mut results: Vec<MergeHistoryEntry> = Vec::new();
let mut count = 0;
while cursor.advance().await.context("Failed to advance cursor")? {
if count >= skip && results.len() < limit as usize {
let doc: Document = cursor
.deserialize_current()
.context("Failed to deserialize")?;
let merge_id = doc.get_str("merge_id").unwrap_or("").to_string();
let source_name = doc
.get_document("source_identity")
.map(|d| d.get_str("name").unwrap_or("").to_string())
.unwrap_or_default();
let target_name = doc
.get_document("target_identity")
.map(|d| d.get_str("name").unwrap_or("").to_string())
.unwrap_or_default();
let faces_count = doc
.get_document("faces_transferred")
.map(|d| d.get_i64("count").unwrap_or(0))
.unwrap_or(0);
let merged_at = doc
.get_datetime("merged_at")
.map(|d| d.to_chrono())
.unwrap_or_default();
let undo_deadline = doc
.get_datetime("undo_deadline")
.map(|d| d.to_chrono())
.unwrap_or_default();
let undone = doc.get_bool("undone").unwrap_or(false);
let undo_expired = doc.get_bool("undo_expired").unwrap_or(false);
results.push(MergeHistoryEntry {
merge_id,
source_name,
target_name,
faces_transferred: faces_count,
merged_at,
undo_deadline,
undone,
undo_expired,
});
}
count += 1;
}
Ok((results, total))
}
pub async fn mark_as_undone(
&self,
merge_id: &str,
undone_by: Option<&str>,
undone_snapshot: UndoneSnapshot,
) -> Result<()> {
let filter = doc! { "merge_id": merge_id };
let snapshot_doc = doc! {
"source_identity_id": undone_snapshot.source_identity_id,
"source_uuid": &undone_snapshot.source_uuid,
"source_name": &undone_snapshot.source_name,
"target_metadata_at_undo": json_to_bson(&undone_snapshot.target_metadata_at_undo),
};
let update = doc! {
"$set": {
"undone": true,
"undone_at": BsonDateTime::from_chrono(Utc::now()),
"undone_by": undone_by,
"undone_snapshot": snapshot_doc,
}
};
self.collection
.update_one(filter, update, None)
.await
.context("Failed to mark merge as undone")?;
tracing::info!("Marked merge {} as undone", merge_id);
Ok(())
}
pub async fn mark_as_redone(&self, merge_id: &str, redone_by: Option<&str>) -> Result<()> {
let now = Utc::now();
let new_deadline = now + chrono::Duration::hours(24);
let filter = doc! { "merge_id": merge_id };
let update = doc! {
"$set": {
"undone": false,
"undone_at": bson::Bson::Null,
"undone_by": redone_by,
"undone_snapshot": bson::Bson::Null,
"undo_deadline": BsonDateTime::from_chrono(new_deadline),
"undo_expired": false
}
};
self.collection
.update_one(filter, update, None)
.await
.context("Failed to mark merge as redone")?;
tracing::info!(
"Marked merge {} as redone (new deadline: {})",
merge_id,
new_deadline
);
Ok(())
}
pub async fn check_undo_deadline(&self, merge_id: &str) -> Result<bool> {
let history = self
.get_merge_history(merge_id)
.await?
.context("Merge history not found")?;
let now = Utc::now();
if now > history.undo_deadline {
return Ok(false);
}
Ok(true)
}
pub async fn mark_expired_merges(&self) -> Result<u64> {
let now = BsonDateTime::from_chrono(Utc::now());
let filter = doc! {
"undo_deadline": { "$lt": now },
"undone": false,
"undo_expired": false
};
let update = doc! { "$set": { "undo_expired": true } };
let result = self
.collection
.update_many(filter, update, None)
.await
.context("Failed to mark expired merges")?;
let count = result.modified_count;
if count > 0 {
tracing::info!("Marked {} expired merges", count);
}
Ok(count)
}
}
src/core/db/mod.rs
+8 -5
View File
@@ -32,17 +32,21 @@ pub trait VectorStore: Send + Sync {
async fn search(&self, query_vector: &[f32], limit: usize) -> Result<Vec<SearchResult>>;
}
pub mod identity_merge_history;
pub mod mongodb_db;
pub mod postgres_db;
pub mod qdrant_db;
pub mod redis_client;
pub mod redis_db;
pub mod sync_db;
pub use identity_merge_history::{
AliasEntry, FacesTransferred, IdentityMergeHistory, IdentityMergeHistoryStore,
IdentitySnapshot, MergeHistoryEntry, MergeHistoryQuery, MergeParams, TargetIdentitySnapshot,
UndoneSnapshot,
};
pub use mongodb_db::MongoDb;
pub use postgres_db::{
Bm25Result, CandidateRecord, CreateApiKeyConfig, FileIdentityRecord, FileRecord,
HybridSearchResult, IdentityChunkRecord, IdentityDetailRecord, IdentityFaceRecord,
Bm25Result, CandidateRecord, CreateApiKeyConfig, FileFaceRecord, FileIdentityRecord,
FileRecord, HybridSearchResult, IdentityChunkRecord, IdentityDetailRecord, IdentityFaceRecord,
IdentityFileRecord, MonitorJob, MonitorJobStats, MonitorJobStatus, PipelineType, PostgresDb,
ProcessorJobStatus, ProcessorResult, ProcessorType, ResourceRecord, VideoRecord, VideoStatus,
};
@@ -52,4 +56,3 @@ pub use redis_client::{
ProgressMessage, RedisClient,
};
pub use redis_db::RedisDb;
pub use sync_db::SyncDb;
src/core/db/mongodb_db.rs
-3
View File
@@ -131,7 +131,6 @@ impl MongoDb {
pre_chunk_ids: vec![],
parent_chunk_id: doc.parent_chunk_id,
child_chunk_ids: doc.child_chunk_ids,
visual_stats: None,
}
})
.collect();
@@ -190,7 +189,6 @@ impl MongoDb {
pre_chunk_ids: vec![],
parent_chunk_id: doc.parent_chunk_id,
child_chunk_ids: doc.child_chunk_ids,
visual_stats: None,
}
})
.collect();
@@ -246,7 +244,6 @@ impl MongoDb {
pre_chunk_ids: vec![],
parent_chunk_id: doc.parent_chunk_id,
child_chunk_ids: doc.child_chunk_ids,
visual_stats: None,
}
})
.collect();
src/core/db/qdrant_db.rs
+13 -47
View File
@@ -70,7 +70,7 @@ impl QdrantDb {
return Ok(());
}
let create_url = format!("{}/collections", self.base_url);
let create_url = format!("{}/collections/{}", self.base_url, self.collection_name);
let body = serde_json::json!({
"vectors": {
"size": vector_dim,
@@ -79,7 +79,7 @@ impl QdrantDb {
});
self.client
.post(&create_url)
.put(&create_url)
.header("api-key", &self.api_key)
.header("Content-Type", "application/json")
.json(&body)
@@ -867,50 +867,6 @@ impl VectorStore for QdrantDb {
}
}
/// Sync face embeddings from PostgreSQL to Qdrant for ANN search
pub async fn sync_face_embeddings(file_uuid: &str) -> Result<()> {
use crate::core::config::DATABASE_URL;
use sqlx::Row;
let pool = sqlx::PgPool::connect(&DATABASE_URL).await?;
let table = crate::core::db::schema::table_name("face_detections");
let qdrant: QdrantDb = QdrantDb::new();
let query = format!(
"SELECT id, trace_id, frame_number, embedding FROM {} \
WHERE file_uuid = $1 AND embedding IS NOT NULL \
AND ((metadata->>'qc_ok')::boolean IS NULL OR (metadata->>'qc_ok')::boolean = true)",
table
);
let rows = sqlx::query(&query).bind(file_uuid).fetch_all(&pool).await?;
let mut count = 0u64;
for row in &rows {
let id: i32 = row.get(0);
let trace_id: Option<i32> = row.get(1);
let frame_number: i64 = row.get(2);
let embedding: Option<Vec<f32>> = row.get(3);
if let (Some(emb), Some(tid)) = (embedding, trace_id) {
if let Err(e) = qdrant
.upsert_face_embedding(id as u64, &emb, file_uuid, tid, frame_number)
.await
{
tracing::warn!("Qdrant upsert failed for face {}: {}", id, e);
continue;
}
count += 1;
}
}
tracing::info!(
"Synced {} face embeddings to Qdrant for {}",
count,
file_uuid
);
Ok(())
}
pub async fn sync_trace_embeddings(file_uuid: &str) -> Result<()> {
use crate::core::config::DATABASE_URL;
use sqlx::Row;
@@ -984,12 +940,22 @@ pub async fn sync_trace_embeddings(file_uuid: &str) -> Result<()> {
}
// Push to Qdrant in batches
// Point ID: hash(file_uuid + trace_id) for global uniqueness
for chunk in trace_avgs.chunks(500) {
let batch: Vec<(u64, &[f32], Option<serde_json::Value>)> = chunk
.iter()
.map(|t| {
let point_id = {
use sha2::{Digest, Sha256};
let mut hasher = Sha256::new();
hasher.update(file_uuid.as_bytes());
hasher.update(b"_");
hasher.update(t.tid.to_string().as_bytes());
let hash = hasher.finalize();
u64::from_be_bytes(hash[0..8].try_into().unwrap())
};
(
t.tid as u64,
point_id,
t.avg_emb.as_slice(),
Some(serde_json::json!({
"trace_id": t.tid,
src/core/db/redis_client.rs
+3 -1
View File
@@ -319,7 +319,9 @@ impl RedisClient {
"timestamp": chrono::Utc::now().to_rfc3339(),
});
let _: usize = conn.publish(&channel, serde_json::to_string(&alert_json)?).await?;
let _: usize = conn
.publish(&channel, serde_json::to_string(&alert_json)?)
.await?;
tracing::warn!(
"Processor alert: {} | {} | {} | {}",
src/core/db/sync_db.rs → src/core/db/sync_db.rs.bak
+4 -1
View File
@@ -78,7 +78,10 @@ impl SyncDb {
pub async fn embed_text(&self, text: &str) -> Result<Vec<f32>> {
let client = reqwest::Client::new();
let response = client
.post(&format!("{}/api/embeddings", crate::core::config::OLLAMA_URL.as_str()))
.post(&format!(
"{}/api/embeddings",
crate::core::config::OLLAMA_URL.as_str()
))
.json(&serde_json::json!({
"model": "all-minilm",
"prompt": text,
src/core/frame_cache.rs
+13 -6
View File
@@ -78,12 +78,19 @@ impl FrameManager {
.and_then(|s| s.strip_suffix(".jpg"))
{
if let Ok(frame_num) = num_str.parse::<u64>() {
let timestamp = frame_num as f64 / fps;
frames.push(CachedFrame {
path: entry.path(),
frame_number: frame_num,
timestamp_secs: timestamp,
});
let frame_path = entry.path();
if let Ok(data) = std::fs::read(&frame_path) {
if crate::core::thumbnail::validator::is_valid_jpeg(&data) {
let timestamp = frame_num as f64 / fps;
frames.push(CachedFrame {
path: frame_path,
frame_number: frame_num,
timestamp_secs: timestamp,
});
} else {
info!("[FrameCache] Skipping invalid JPEG: {:?}", frame_path);
}
}
}
}
}
src/core/identity/storage.rs
+1 -1
View File
@@ -193,7 +193,7 @@ pub async fn save_identity_file_by_pool(pool: &sqlx::PgPool, uuid: &str) -> Resu
let record = sqlx::query_as::<_, crate::core::db::IdentityDetailRecord>(
&format!(
"SELECT id, uuid::text, name, identity_type, source, status, metadata, reference_data, \
"SELECT id::bigint, uuid::text, name, identity_type, source, status, metadata, COALESCE(reference_data, '{{}}'::jsonb) as reference_data, \
NULL::real[] as voice_embedding, NULL::real[] as identity_embedding, \
face_embedding::real[] as face_embedding, \
tmdb_id, tmdb_profile, created_at::timestamptz as created_at, NULL::timestamptz as updated_at \
src/core/llm/function_calling.rs
+87 -21
View File
@@ -97,6 +97,68 @@ pub fn llm_vision_model() -> String {
config::llm::VISION_MODEL.clone()
}
/// Call the vision LLM with text + base64 images. Returns the generated text.
pub async fn call_llm_vision(
system_prompt: &str,
user_text: &str,
base64_images: Vec<String>,
max_tokens: u32,
timeout_secs: u64,
) -> anyhow::Result<String> {
let mut content_parts: Vec<Value> = vec![json!({"type": "text", "text": user_text})];
for img in &base64_images {
content_parts.push(json!({
"type": "image_url",
"image_url": {"url": format!("data:image/jpeg;base64,{}", img)}
}));
}
let messages = json!([
{"role": "system", "content": system_prompt},
{"role": "user", "content": content_parts}
]);
let req = json!({
"model": llm_vision_model(),
"messages": messages,
"temperature": 0.1,
"max_tokens": max_tokens,
"stream": false,
});
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(timeout_secs))
.build()?;
let res = client.post(&llm_vision_url()).json(&req).send().await?;
if !res.status().is_success() {
let text = res.text().await.unwrap_or_default();
anyhow::bail!("Vision LLM API error: {}", text);
}
#[derive(Deserialize)]
struct VisionResponse {
choices: Vec<VisionChoice>,
}
#[derive(Deserialize)]
struct VisionChoice {
message: VisionMessage,
}
#[derive(Deserialize)]
struct VisionMessage {
content: Option<String>,
}
let vision_res: VisionResponse = res.json().await?;
let content = vision_res
.choices
.into_iter()
.next()
.and_then(|c| c.message.content)
.unwrap_or_default();
Ok(content.trim().to_string())
}
/// Build a tool definition JSON for function calling
pub fn make_tool(name: &str, description: &str, properties: Value, required: Vec<&str>) -> ToolDef {
ToolDef {
@@ -121,9 +183,11 @@ pub async fn call_llm(
timeout_secs: u64,
) -> anyhow::Result<LlmResponse> {
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(
if timeout_secs > 0 { timeout_secs } else { *config::llm::CHAT_TIMEOUT_SECS },
))
.timeout(std::time::Duration::from_secs(if timeout_secs > 0 {
timeout_secs
} else {
*config::llm::CHAT_TIMEOUT_SECS
}))
.build()?;
let req = ChatRequest {
@@ -135,11 +199,7 @@ pub async fn call_llm(
tools,
};
let res = client
.post(&llm_chat_url())
.json(&req)
.send()
.await?;
let res = client.post(&llm_chat_url()).json(&req).send().await?;
if !res.status().is_success() {
let text = res.text().await.unwrap_or_default();
@@ -147,13 +207,17 @@ pub async fn call_llm(
}
let chat_res: ChatResponse = res.json().await?;
let choice = chat_res.choices.into_iter().next()
let choice = chat_res
.choices
.into_iter()
.next()
.ok_or_else(|| anyhow::anyhow!("Empty LLM response"))?;
match choice.finish_reason.as_deref() {
Some("tool_calls") => {
let calls = choice.message.tool_calls
.ok_or_else(|| anyhow::anyhow!("finish_reason=tool_calls but no tool_calls in message"))?;
let calls = choice.message.tool_calls.ok_or_else(|| {
anyhow::anyhow!("finish_reason=tool_calls but no tool_calls in message")
})?;
Ok(LlmResponse::ToolCalls(calls))
}
_ => {
@@ -164,16 +228,18 @@ pub async fn call_llm(
}
/// Helper to build the system prompt + user messages
pub fn build_conversation(system_prompt: &str, user_query: &str, history: Vec<ChatMessage>) -> Vec<ChatMessage> {
let mut messages = vec![
ChatMessage {
role: "system".to_string(),
content: Some(system_prompt.to_string()),
tool_calls: None,
tool_call_id: None,
name: None,
},
];
pub fn build_conversation(
system_prompt: &str,
user_query: &str,
history: Vec<ChatMessage>,
) -> Vec<ChatMessage> {
let mut messages = vec![ChatMessage {
role: "system".to_string(),
content: Some(system_prompt.to_string()),
tool_calls: None,
tool_call_id: None,
name: None,
}];
// Add history (user + assistant exchanges)
messages.extend(history);
// Add current user query
src/core/processor/asrx.rs
+38 -12
View File
@@ -18,12 +18,22 @@ pub struct AsrxResult {
#[derive(Debug, Serialize, Deserialize)]
pub struct AsrxSegment {
#[serde(alias = "start")]
pub start_time: f64,
#[serde(alias = "end")]
pub end_time: f64,
#[serde(default)]
pub start_frame: u64,
#[serde(default)]
pub end_frame: u64,
pub text: String,
pub speaker_id: Option<String>,
#[serde(default)]
pub language: Option<String>,
#[serde(default)]
pub lang_prob: Option<f64>,
#[serde(default)]
pub quality: Option<f64>,
}
pub async fn process_asrx(
@@ -32,24 +42,16 @@ pub async fn process_asrx(
uuid: Option<&str>,
) -> Result<AsrxResult> {
let executor = PythonExecutor::new()?;
let script_path = executor.script_path("asrx_processor_custom.py");
let script_path = executor.script_path("asrx_processor.py");
tracing::info!(
"[ASRX] Starting speaker diarization (custom): {}",
"[ASRX] Starting hybrid speaker diarization: {}",
video_path
);
if !script_path.exists() {
tracing::warn!("[ASRX] Custom script not found, falling back to original");
let fallback_path = executor.script_path("asrx_processor.py");
if !fallback_path.exists() {
tracing::warn!("[ASRX] No script found, returning empty result");
return Ok(AsrxResult {
language: None,
segments: vec![],
embeddings: None,
});
}
tracing::error!("[ASRX] Script not found: {:?}", script_path);
anyhow::bail!("asrx_processor.py not found");
}
tracing::info!(
@@ -65,6 +67,7 @@ pub async fn process_asrx(
if let Some(u) = uuid {
cmd.arg("--uuid").arg(u);
cmd.arg("--file-uuid").arg(u);
}
cmd.stdout(std::process::Stdio::piped())
@@ -126,6 +129,9 @@ mod tests {
end_frame: 75,
text: "Hello".to_string(),
speaker_id: Some("SPEAKER_00".to_string()),
language: None,
lang_prob: None,
quality: None,
}],
embeddings: None,
};
@@ -173,7 +179,27 @@ mod tests {
end_frame: 150,
text: "Test".to_string(),
speaker_id: None,
language: None,
lang_prob: None,
quality: None,
};
assert!(segment.end_time > segment.start_time);
}
#[test]
fn test_asrx_backward_compat_old_format() {
let json = r#"{
"language": "en",
"segments": [
{"start": 10.0, "end": 12.5, "text": "Hello", "speaker_id": "SPEAKER_00"}
]
}"#;
let result: AsrxResult = serde_json::from_str(json).unwrap();
assert_eq!(result.segments.len(), 1);
assert_eq!(result.segments[0].start_time, 10.0);
assert_eq!(result.segments[0].end_time, 12.5);
assert_eq!(result.segments[0].text, "Hello");
assert_eq!(result.segments[0].start_frame, 0);
assert_eq!(result.segments[0].end_frame, 0);
}
}
src/core/processor/cut.rs
+24 -12
View File
@@ -43,11 +43,15 @@ pub async fn process_cut(
let script_path = executor.script_path("cut_processor.py");
if !script_path.exists() {
return Ok(CutResult {
let empty_result = CutResult {
frame_count: 0,
fps: 0.0,
scenes: vec![],
});
};
let json = serde_json::to_string_pretty(&empty_result)?;
std::fs::write(output_path, &json)
.with_context(|| format!("Failed to write {:?}", output_path))?;
return Ok(empty_result);
}
executor
@@ -127,18 +131,26 @@ fn try_native_cut(video_path: &str) -> Result<CutResult> {
.context("Failed to run ffmpeg scene detection")?;
let stderr_output = String::from_utf8_lossy(&scene_output.stderr);
let stdout_output = String::from_utf8_lossy(&scene_output.stdout);
let mut scene_times: Vec<f64> = Vec::new();
// Parse ffmpeg showinfo output for scene changes
// Format: [Parsed_showinfo...] pts:123.456 pts_time:123.456 ...
for line in stderr_output.lines() {
if line.contains("pts_time:") {
if let Some(pos) = line.find("pts_time:") {
let rest = &line[pos + 9..];
let time_str = rest.split_whitespace().next().unwrap_or("");
if let Ok(t) = time_str.parse::<f64>() {
scene_times.push(t);
}
// Parse ffprobe output for scene changes (check both stderr and stdout)
// Format: pts_time=123.456 or pts_time:123.456
for line in stderr_output.lines().chain(stdout_output.lines()) {
// Try pts_time= format (standard ffprobe output)
if let Some(pos) = line.find("pts_time=") {
let rest = &line[pos + 9..];
let time_str = rest.split_whitespace().next().unwrap_or("");
if let Ok(t) = time_str.parse::<f64>() {
scene_times.push(t);
}
}
// Try pts_time: format (showinfo filter output)
else if let Some(pos) = line.find("pts_time:") {
let rest = &line[pos + 9..];
let time_str = rest.split_whitespace().next().unwrap_or("");
if let Ok(t) = time_str.parse::<f64>() {
scene_times.push(t);
}
}
}
src/core/processor/mod.rs
-2
View File
@@ -11,7 +11,6 @@ pub mod pose;
pub mod scene_classification;
pub mod story;
pub mod tkg;
pub mod visual_chunk;
pub mod yolo;
pub use asr::{process_asr, AsrResult, AsrSegment};
@@ -40,5 +39,4 @@ pub use tkg::{
build_tkg, query_auto_representative_frame, FrameTraceInfo, MainIdentityInfo,
RepresentativeFrameResult, TkgResult,
};
pub use visual_chunk::{process_visual_chunk, process_visual_chunk_advanced, VisualChunkResult};
pub use yolo::{process_yolo, YoloFrame, YoloObject, YoloResult};
src/core/processor/tkg.rs
+153 -41
View File
@@ -38,7 +38,10 @@ fn load_face_pose_data(output_dir: &str, file_uuid: &str) -> Result<Vec<FacePose
let mut poses = Vec::new();
if let Some(frames) = json.get("frames").and_then(|v| v.as_array()) {
for frame_entry in frames {
let frame_num = frame_entry.get("frame").and_then(|v| v.as_i64()).unwrap_or(0);
let frame_num = frame_entry
.get("frame")
.and_then(|v| v.as_i64())
.unwrap_or(0);
if let Some(faces) = frame_entry.get("faces").and_then(|v| v.as_array()) {
for face in faces {
let bbox = match face.get("bbox") {
@@ -68,7 +71,14 @@ fn load_face_pose_data(output_dir: &str, file_uuid: &str) -> Result<Vec<FacePose
/// Match a face from face_detections (frame, x, y, w, h) to its pose in face.json
/// Uses bbox center distance to find the best match when multiple faces per frame.
fn get_pose_for_face(frame: i64, x: f64, y: f64, w: f64, h: f64, poses: &[FacePose]) -> Option<(f64, f64, f64)> {
fn get_pose_for_face(
frame: i64,
x: f64,
y: f64,
w: f64,
h: f64,
poses: &[FacePose],
) -> Option<(f64, f64, f64)> {
let cx = x + w / 2.0;
let cy = y + h / 2.0;
let mut best_dist = f64::MAX;
@@ -86,8 +96,12 @@ fn get_pose_for_face(frame: i64, x: f64, y: f64, w: f64, h: f64, poses: &[FacePo
}
fn detect_mutual_gaze(
bbox_a_x: f64, bbox_a_w: f64, yaw_a: f64,
bbox_b_x: f64, bbox_b_w: f64, yaw_b: f64,
bbox_a_x: f64,
bbox_a_w: f64,
yaw_a: f64,
bbox_b_x: f64,
bbox_b_w: f64,
yaw_b: f64,
threshold: f64,
) -> bool {
let cx_a = bbox_a_x + bbox_a_w / 2.0;
@@ -138,12 +152,16 @@ struct AsrxSegmentEntry {
#[serde(default)]
speaker_id: String,
#[serde(default)]
start_time: f64,
start: f64,
#[serde(default)]
end_time: f64,
end: f64,
#[serde(default)]
text: String,
#[allow(dead_code)]
#[serde(default)]
start_frame: i64,
#[allow(dead_code)]
#[serde(default)]
end_frame: i64,
}
@@ -195,7 +213,10 @@ pub struct TkgResult {
pub async fn build_tkg(db: &PostgresDb, file_uuid: &str, output_dir: &str) -> Result<TkgResult> {
let pool = db.pool();
let pose_data = load_face_pose_data(output_dir, file_uuid).unwrap_or_default();
tracing::info!("[TKG] Loaded {} pose entries from face.json", pose_data.len());
tracing::info!(
"[TKG] Loaded {} pose entries from face.json",
pose_data.len()
);
let n_face = build_face_trace_nodes(pool, file_uuid, &pose_data).await?;
let n_objects = build_yolo_object_nodes(pool, file_uuid, output_dir).await?;
@@ -217,7 +238,11 @@ pub async fn build_tkg(db: &PostgresDb, file_uuid: &str, output_dir: &str) -> Re
// ── Node builders ─────────────────────────────────────────────────
async fn build_face_trace_nodes(pool: &PgPool, file_uuid: &str, pose_data: &[FacePose]) -> Result<usize> {
async fn build_face_trace_nodes(
pool: &PgPool,
file_uuid: &str,
pose_data: &[FacePose],
) -> Result<usize> {
let face_table = t("face_detections");
let nodes_table = t("tkg_nodes");
@@ -257,7 +282,10 @@ async fn build_face_trace_nodes(pool: &PgPool, file_uuid: &str, pose_data: &[Fac
// Group by trace_id: trace_id → Vec<(frame, x, y, w, h)>
let mut trace_frames: HashMap<i64, Vec<(i64, f64, f64, f64, f64)>> = HashMap::new();
for (tid, frame, x, y, w, h) in &frame_rows {
trace_frames.entry(*tid).or_default().push((*frame, *x, *y, *w, *h));
trace_frames
.entry(*tid)
.or_default()
.push((*frame, *x, *y, *w, *h));
}
let mut count = 0;
@@ -274,7 +302,9 @@ async fn build_face_trace_nodes(pool: &PgPool, file_uuid: &str, pose_data: &[Fac
if let Some(frames) = trace_frames.get(&tid) {
for (frame, x, y, w, h) in frames {
if let Some((yaw, pitch, roll)) = get_pose_for_face(*frame, *x, *y, *w, *h, pose_data) {
if let Some((yaw, pitch, roll)) =
get_pose_for_face(*frame, *x, *y, *w, *h, pose_data)
{
yaw_sum += yaw;
pitch_sum += pitch;
roll_sum += roll;
@@ -284,7 +314,11 @@ async fn build_face_trace_nodes(pool: &PgPool, file_uuid: &str, pose_data: &[Fac
}
let (avg_yaw, avg_pitch, avg_roll) = if pose_count > 0 {
(yaw_sum / pose_count as f64, pitch_sum / pose_count as f64, roll_sum / pose_count as f64)
(
yaw_sum / pose_count as f64,
pitch_sum / pose_count as f64,
roll_sum / pose_count as f64,
)
} else {
(0.0, 0.0, 0.0)
};
@@ -401,8 +435,44 @@ async fn build_speaker_nodes(pool: &PgPool, file_uuid: &str, output_dir: &str) -
let nodes_table = t("tkg_nodes");
let mut count = 0;
// Group segments by speaker_id
let mut speaker_segments: HashMap<String, Vec<&AsrxSegmentEntry>> = HashMap::new();
for seg in &asrx.segments {
speaker_segments
.entry(seg.speaker_id.clone())
.or_default()
.push(seg);
}
for (sid, stat) in &stats {
let props = serde_json::json!({ "segment_count": stat.count });
let segs = speaker_segments.get(sid);
let (full_text, segments_json) = if let Some(seg_list) = segs {
let full: String = seg_list
.iter()
.map(|s| s.text.trim())
.filter(|t| !t.is_empty())
.collect::<Vec<_>>()
.join(" ");
let segments: Vec<serde_json::Value> = seg_list
.iter()
.map(|s| {
serde_json::json!({
"start": s.start,
"end": s.end,
"text": s.text,
})
})
.collect();
(full, serde_json::Value::Array(segments))
} else {
(String::new(), serde_json::Value::Array(vec![]))
};
let props = serde_json::json!({
"segment_count": stat.count,
"segments": segments_json,
"full_text": full_text,
});
sqlx::query(&format!(
r#"
@@ -576,8 +646,8 @@ async fn build_speaker_face_edges(
// Calculate fps from last segment
let last = asrx.segments.last().unwrap();
let fps = if last.end_time > 0.0 {
last.end_frame as f64 / last.end_time
let fps = if last.end > 0.0 {
last.end_frame as f64 / last.end
} else {
30.0
};
@@ -604,8 +674,8 @@ async fn build_speaker_face_edges(
let face_end_sec = *ef as f64 / fps;
for seg in &asrx.segments {
let seg_start = seg.start_time;
let seg_end = seg.end_time;
let seg_start = seg.start;
let seg_end = seg.end;
let overlap_start = face_start_sec.max(seg_start);
let overlap_end = face_end_sec.min(seg_end);
@@ -669,7 +739,11 @@ async fn build_speaker_face_edges(
Ok(edge_count)
}
async fn build_face_face_edges(pool: &PgPool, file_uuid: &str, pose_data: &[FacePose]) -> Result<usize> {
async fn build_face_face_edges(
pool: &PgPool,
file_uuid: &str,
pose_data: &[FacePose],
) -> Result<usize> {
let face_table = t("face_detections");
let nodes_table = t("tkg_nodes");
let edges_table = t("tkg_edges");
@@ -722,8 +796,9 @@ async fn build_face_face_edges(pool: &PgPool, file_uuid: &str, pose_data: &[Face
(Some(&(xa, ya, wa, ha)), Some(&(xb, yb, wb, hb))) => {
get_pose_for_face(*frame, xa, ya, wa, ha, pose_data)
.and_then(|(yaw_a, _, _)| {
get_pose_for_face(*frame, xb, yb, wb, hb, pose_data)
.map(|(yaw_b, _, _)| detect_mutual_gaze(xa, wa, yaw_a, xb, wb, yaw_b, 0.05))
get_pose_for_face(*frame, xb, yb, wb, hb, pose_data).map(|(yaw_b, _, _)| {
detect_mutual_gaze(xa, wa, yaw_a, xb, wb, yaw_b, 0.05)
})
})
.unwrap_or(false)
}
@@ -770,7 +845,11 @@ async fn build_face_face_edges(pool: &PgPool, file_uuid: &str, pose_data: &[Face
};
let frames: Vec<i64> = frame_data.iter().map(|(f, _)| *f).collect();
let gaze_frames: Vec<i64> = frame_data.iter().filter(|(_, g)| *g).map(|(f, _)| *f).collect();
let gaze_frames: Vec<i64> = frame_data
.iter()
.filter(|(_, g)| *g)
.map(|(f, _)| *f)
.collect();
let gaze_count = gaze_frames.len() as i64;
let has_gaze = gaze_count > 0;
@@ -793,8 +872,13 @@ async fn build_face_face_edges(pool: &PgPool, file_uuid: &str, pose_data: &[Face
}
}
let (avg_ya, avg_yb) = if gaze_sample > 0 {
(yaw_a_sum / gaze_sample as f64, yaw_b_sum / gaze_sample as f64)
} else { (0.0, 0.0) };
(
yaw_a_sum / gaze_sample as f64,
yaw_b_sum / gaze_sample as f64,
)
} else {
(0.0, 0.0)
};
serde_json::json!({
"first_frame": frames[0],
@@ -902,9 +986,14 @@ pub async fn query_auto_representative_frame(
.context("Failed to detect main identities")?;
let main_ids: Vec<(i32, String, String, i64)> = mains;
let main_idents: Vec<MainIdentityInfo> = main_ids.iter().map(|(_, u, n, c)|
MainIdentityInfo { identity_uuid: u.clone(), name: n.clone(), face_count: *c }
).collect();
let main_idents: Vec<MainIdentityInfo> = main_ids
.iter()
.map(|(_, u, n, c)| MainIdentityInfo {
identity_uuid: u.clone(),
name: n.clone(),
face_count: *c,
})
.collect();
let frame_number: Option<i64> = if main_ids.len() >= 2 {
let id_a = main_ids[0].0;
@@ -915,16 +1004,20 @@ pub async fn query_auto_representative_frame(
AND trace_id IS NOT NULL GROUP BY trace_id ORDER BY COUNT(*) DESC LIMIT 1",
fd_table
))
.bind(file_uuid).bind(id_a)
.fetch_optional(pool).await?;
.bind(file_uuid)
.bind(id_a)
.fetch_optional(pool)
.await?;
let trace_b: Option<(i32,)> = sqlx::query_as(&format!(
"SELECT trace_id FROM {} WHERE file_uuid = $1 AND identity_id = $2 \
AND trace_id IS NOT NULL GROUP BY trace_id ORDER BY COUNT(*) DESC LIMIT 1",
fd_table
))
.bind(file_uuid).bind(id_b)
.fetch_optional(pool).await?;
.bind(file_uuid)
.bind(id_b)
.fetch_optional(pool)
.await?;
match (trace_a, trace_b) {
(Some((ta,)), Some((tb,))) => {
@@ -940,11 +1033,18 @@ pub async fn query_auto_representative_frame(
LIMIT 1",
edges_table, nodes_table, nodes_table
))
.bind(file_uuid).bind(ta).bind(tb)
.fetch_optional(pool).await?;
.bind(file_uuid)
.bind(ta)
.bind(tb)
.fetch_optional(pool)
.await?;
if let Some((f,)) = tkg_frame {
if f <= half_frame { Some(f) } else { None }
if f <= half_frame {
Some(f)
} else {
None
}
} else {
sqlx::query_scalar::<_, i64>(&format!(
"SELECT MIN(fd_a.frame_number)::bigint \
@@ -954,8 +1054,12 @@ pub async fn query_auto_representative_frame(
AND fd_b.identity_id = $3 AND fd_a.frame_number <= $4",
fd_table, fd_table
))
.bind(file_uuid).bind(id_a).bind(id_b).bind(half_frame)
.fetch_optional(pool).await?
.bind(file_uuid)
.bind(id_a)
.bind(id_b)
.bind(half_frame)
.fetch_optional(pool)
.await?
}
}
_ => None,
@@ -976,8 +1080,11 @@ pub async fn query_auto_representative_frame(
LIMIT 1",
fd_table
))
.bind(file_uuid).bind(first_id).bind(half_frame)
.fetch_optional(pool).await?
.bind(file_uuid)
.bind(first_id)
.bind(half_frame)
.fetch_optional(pool)
.await?
} else {
None
}
@@ -995,20 +1102,25 @@ pub async fn query_auto_representative_frame(
LIMIT 1",
fd_table
))
.bind(file_uuid).bind(half_frame)
.fetch_optional(pool).await?
.bind(file_uuid)
.bind(half_frame)
.fetch_optional(pool)
.await?
}
};
let frame_number = frame_number.ok_or_else(|| anyhow::anyhow!("No faces found in this file"))?;
let frame_number =
frame_number.ok_or_else(|| anyhow::anyhow!("No faces found in this file"))?;
let face_quality: f64 = sqlx::query_scalar::<_, f64>(&format!(
"SELECT COALESCE(MAX((width::float8 * height::float8) * confidence::float8), 0) \
FROM {} WHERE file_uuid = $1 AND frame_number = $2",
fd_table
))
.bind(file_uuid).bind(frame_number)
.fetch_one(pool).await?;
.bind(file_uuid)
.bind(frame_number)
.fetch_one(pool)
.await?;
let traces: Vec<FrameTraceInfo> = sqlx::query_as::<_, (i32, Option<String>, Option<String>, i32, i32, i32, i32, f64)>(&format!(
"SELECT fd.trace_id, i.uuid::text, i.name, fd.x, fd.y, fd.width, fd.height, fd.confidence::float8 \
src/core/processor/visual_chunk.rs
-594
View File
@@ -1,594 +0,0 @@
//! 視覺分片處理器 (Phase 2.2)
//!
//! 從 YOLO 結果生成視覺分片
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use std::time::Duration;
use super::executor::PythonExecutor;
use super::yolo::{YoloFrame, YoloResult};
const VISUAL_CHUNK_TIMEOUT: Duration = Duration::from_secs(3600);
/// 視覺分片處理結果
#[derive(Debug, Serialize, Deserialize, Clone, Default)]
pub struct VisualChunkResult {
/// 生成的視覺分片數量
pub chunk_count: u32,
/// 處理的總幀數
pub total_frames: u32,
/// 檢測到的總物件數
pub total_objects: u32,
/// 唯一物件類別數
pub unique_classes: u32,
/// 生成的視覺分片
pub chunks: Vec<crate::core::chunk::Chunk>,
}
/// 從 YOLO 結果生成視覺分片
pub async fn process_visual_chunk(
file_id: i32,
uuid: String,
video_path: &str,
yolo_result: &YoloResult,
chunk_index_offset: u32,
fps: f64,
) -> Result<VisualChunkResult> {
tracing::info!(
"[VisualChunk] Starting visual chunk generation for video: {}, {} frames",
video_path,
yolo_result.frames.len()
);
if yolo_result.frames.is_empty() {
tracing::warn!("[VisualChunk] No YOLO frames to process");
return Ok(VisualChunkResult {
chunk_count: 0,
total_frames: 0,
total_objects: 0,
unique_classes: 0,
chunks: vec![],
});
}
// 策略 1: 固定幀數分片（每 N 幀一個分片）
let chunks = create_fixed_frame_chunks(file_id, &uuid, yolo_result, chunk_index_offset, fps);
// 統計信息
let total_objects: u32 = yolo_result
.frames
.iter()
.map(|f| f.objects.len() as u32)
.sum();
let all_classes: Vec<String> = yolo_result
.frames
.iter()
.flat_map(|f| f.objects.iter().map(|o| o.class_name.clone()))
.collect();
let unique_classes: u32 = all_classes
.iter()
.cloned()
.collect::<std::collections::HashSet<_>>()
.len() as u32;
tracing::info!(
"[VisualChunk] Generated {} visual chunks from {} frames, {} total objects, {} unique classes",
chunks.len(),
yolo_result.frames.len(),
total_objects,
unique_classes
);
Ok(VisualChunkResult {
chunk_count: chunks.len() as u32,
total_frames: yolo_result.frames.len() as u32,
total_objects,
unique_classes,
chunks,
})
}
/// 創建固定幀數分片（每 N 幀一個分片）
fn create_fixed_frame_chunks(
file_id: i32,
uuid: &str,
yolo_result: &YoloResult,
chunk_index_offset: u32,
fps: f64,
) -> Vec<crate::core::chunk::Chunk> {
let mut chunks = Vec::new();
// 配置：每 30 幀創建一個分片（約 1 秒，如果 fps=30）
let frames_per_chunk = 30;
let total_frames = yolo_result.frames.len();
if total_frames == 0 {
return chunks;
}
let mut chunk_index = chunk_index_offset;
let mut start_idx = 0;
while start_idx < total_frames {
let end_idx = std::cmp::min(start_idx + frames_per_chunk, total_frames);
// 獲取這個分片的幀
let chunk_frames: Vec<YoloFrame> = yolo_result.frames[start_idx..end_idx]
.iter()
.cloned()
.collect();
if chunk_frames.is_empty() {
break;
}
// 計算幀範圍
let start_frame = chunk_frames.first().unwrap().frame as i64;
let end_frame = chunk_frames.last().unwrap().frame as i64 + 1; // exclusive
// 創建視覺分片
let chunk = crate::core::chunk::Chunk::from_yolo_frames(
file_id,
uuid.to_string(),
format!("vis_{}", chunk_index),
start_frame,
end_frame,
fps,
chunk_frames,
);
chunks.push(chunk);
// 更新索引
start_idx = end_idx;
chunk_index += 1;
}
chunks
}
/// 基於物件相似度創建分片
fn create_similarity_based_chunks(
file_id: i32,
uuid: &str,
yolo_result: &YoloResult,
chunk_index_offset: u32,
fps: f64,
similarity_threshold: f32,
min_frames_per_chunk: usize,
) -> Vec<crate::core::chunk::Chunk> {
let mut chunks = Vec::new();
if yolo_result.frames.is_empty() {
return chunks;
}
let mut current_chunk_frames: Vec<YoloFrame> = Vec::new();
let mut chunk_index = chunk_index_offset;
let mut current_start_frame = 0;
for (i, frame) in yolo_result.frames.iter().enumerate() {
if current_chunk_frames.is_empty() {
current_chunk_frames.push(frame.clone());
current_start_frame = frame.frame as i64;
continue;
}
// 檢查相似度（簡化版本：檢查物件類別是否相同）
let last_frame = current_chunk_frames.last().unwrap();
let similarity = calculate_frame_similarity(last_frame, frame);
if similarity >= similarity_threshold {
// 相似度高，加入當前分片
current_chunk_frames.push(frame.clone());
} else {
// 相似度低，創建新分片
if current_chunk_frames.len() >= min_frames_per_chunk {
let end_frame = current_chunk_frames.last().unwrap().frame as i64 + 1;
let chunk = crate::core::chunk::Chunk::from_yolo_frames(
file_id,
uuid.to_string(),
format!("vis_{}", chunk_index),
current_start_frame,
end_frame,
fps,
current_chunk_frames.clone(),
);
chunks.push(chunk);
chunk_index += 1;
}
// 開始新的分片
current_chunk_frames = vec![frame.clone()];
current_start_frame = frame.frame as i64;
}
}
// 處理最後一個分片
if current_chunk_frames.len() >= min_frames_per_chunk {
let end_frame = current_chunk_frames.last().unwrap().frame as i64 + 1;
let chunk = crate::core::chunk::Chunk::from_yolo_frames(
file_id,
uuid.to_string(),
format!("vis_{}", chunk_index),
current_start_frame,
end_frame,
fps,
current_chunk_frames,
);
chunks.push(chunk);
}
chunks
}
/// 計算兩個幀之間的相似度（基於物件類別）
fn calculate_frame_similarity(frame1: &YoloFrame, frame2: &YoloFrame) -> f32 {
if frame1.objects.is_empty() && frame2.objects.is_empty() {
return 1.0;
}
if frame1.objects.is_empty() || frame2.objects.is_empty() {
return 0.0;
}
let set1: std::collections::HashSet<String> = frame1
.objects
.iter()
.map(|o| o.class_name.clone())
.collect();
let set2: std::collections::HashSet<String> = frame2
.objects
.iter()
.map(|o| o.class_name.clone())
.collect();
let intersection: Vec<_> = set1.intersection(&set2).collect();
let union: Vec<_> = set1.union(&set2).collect();
if union.is_empty() {
0.0
} else {
intersection.len() as f32 / union.len() as f32
}
}
/// 使用 Python 腳本生成視覺分片（進階版本）
pub async fn process_visual_chunk_advanced(
video_path: &str,
output_path: &str,
uuid: Option<&str>,
) -> Result<VisualChunkResult> {
let executor = PythonExecutor::new()?;
let script_path = executor.script_path("visual_chunk_processor.py");
tracing::info!(
"[VisualChunk] Starting advanced visual chunk generation: {}",
video_path
);
if !script_path.exists() {
tracing::warn!("[VisualChunk] Script not found, using basic generation");
// 這裡可以回退到基本生成方法
return Ok(VisualChunkResult {
chunk_count: 0,
total_frames: 0,
total_objects: 0,
unique_classes: 0,
chunks: vec![],
});
}
let yolo_path = uuid.map(|u| {
std::path::PathBuf::from(crate::core::config::OUTPUT_DIR.as_str())
.join(format!("{}.yolo.json", u))
.to_string_lossy()
.to_string()
});
let args: &[&str] = if let Some(ref yp) = yolo_path {
&[video_path, output_path, "--yolo-result", yp]
} else {
&[video_path, output_path]
};
let result = match executor
.run(
"visual_chunk_processor.py",
args,
uuid,
"VisualChunk",
Some(VISUAL_CHUNK_TIMEOUT),
)
.await
{
Ok(_) => match std::fs::read_to_string(output_path) {
Ok(json_str) => match serde_json::from_str::<VisualChunkResult>(&json_str) {
Ok(r) => r,
Err(e) => {
tracing::warn!(
"[VisualChunk] Failed to parse output ({}), returning empty",
e
);
VisualChunkResult::default()
}
},
Err(e) => {
tracing::warn!(
"[VisualChunk] Failed to read output ({}), returning empty",
e
);
VisualChunkResult::default()
}
},
Err(e) => {
tracing::warn!(
"[VisualChunk] Failed to run script ({}), returning empty",
e
);
VisualChunkResult::default()
}
};
tracing::info!(
"[VisualChunk] Advanced generation result: {} chunks, {} frames",
result.chunk_count,
result.total_frames
);
Ok(result)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_calculate_frame_similarity() {
use crate::core::processor::yolo::{YoloFrame, YoloObject};
let frame1 = YoloFrame {
frame: 0,
timestamp: 0.0,
objects: vec![
YoloObject {
class_name: "person".to_string(),
class_id: 0,
x: 100,
y: 200,
width: 50,
height: 100,
confidence: 0.95,
},
YoloObject {
class_name: "car".to_string(),
class_id: 2,
x: 300,
y: 150,
width: 80,
height: 60,
confidence: 0.87,
},
],
};
let frame2 = YoloFrame {
frame: 1,
timestamp: 0.033,
objects: vec![
YoloObject {
class_name: "person".to_string(),
class_id: 0,
x: 110,
y: 210,
width: 52,
height: 102,
confidence: 0.92,
},
YoloObject {
class_name: "car".to_string(),
class_id: 2,
x: 310,
y: 155,
width: 82,
height: 62,
confidence: 0.85,
},
],
};
let frame3 = YoloFrame {
frame: 2,
timestamp: 0.066,
objects: vec![YoloObject {
class_name: "dog".to_string(),
class_id: 16,
x: 150,
y: 250,
width: 40,
height: 60,
confidence: 0.78,
}],
};
// 相同物件的幀應該高度相似
let similarity_same = calculate_frame_similarity(&frame1, &frame2);
assert!((similarity_same - 1.0).abs() < 0.001);
// 不同物件的幀應該不相似
let similarity_diff = calculate_frame_similarity(&frame1, &frame3);
assert!((similarity_diff - 0.0).abs() < 0.001);
// 空幀應該完全相似
let empty_frame = YoloFrame {
frame: 3,
timestamp: 0.1,
objects: vec![],
};
let similarity_empty = calculate_frame_similarity(&empty_frame, &empty_frame);
assert!((similarity_empty - 1.0).abs() < 0.001);
}
#[tokio::test]
async fn test_create_fixed_frame_chunks() {
use crate::core::processor::yolo::{YoloFrame, YoloObject, YoloResult};
// 創建測試 YOLO 結果（60 幀，每幀都有物件）
let mut frames = Vec::new();
for i in 0..60 {
frames.push(YoloFrame {
frame: i as u64,
timestamp: i as f64 / 30.0, // 假設 fps=30
objects: vec![YoloObject {
class_name: "person".to_string(),
class_id: 0,
x: 100,
y: 200,
width: 50,
height: 100,
confidence: 0.9,
}],
});
}
let yolo_result = YoloResult {
frame_count: 60,
fps: 30.0,
frames,
};
let chunks = create_fixed_frame_chunks(1, "test-uuid", &yolo_result, 0, 30.0);
// 60 幀，每 30 幀一個分片，應該有 2 個分片
assert_eq!(chunks.len(), 2);
// 檢查第一個分片
let first_chunk = &chunks[0];
assert_eq!(
first_chunk.chunk_type,
crate::core::chunk::ChunkType::Visual
);
assert_eq!(first_chunk.start_frame, 0);
assert_eq!(first_chunk.end_frame, 30); // exclusive
assert_eq!(first_chunk.frame_count, 30);
// 檢查第二個分片
let second_chunk = &chunks[1];
assert_eq!(
second_chunk.chunk_type,
crate::core::chunk::ChunkType::Visual
);
assert_eq!(second_chunk.start_frame, 30);
assert_eq!(second_chunk.end_frame, 60); // exclusive
assert_eq!(second_chunk.frame_count, 30);
}
#[test]
fn test_create_similarity_based_chunks() {
use crate::core::processor::yolo::{YoloFrame, YoloObject, YoloResult};
// 創建測試 YOLO 結果
let frames = vec![
YoloFrame {
// 幀 0-4: 都有 person 和 car
frame: 0,
timestamp: 0.0,
objects: vec![
YoloObject {
class_name: "person".to_string(),
class_id: 0,
x: 100,
y: 200,
width: 50,
height: 100,
confidence: 0.9,
},
YoloObject {
class_name: "car".to_string(),
class_id: 2,
x: 300,
y: 150,
width: 80,
height: 60,
confidence: 0.8,
},
],
},
YoloFrame {
// 幀 1
frame: 1,
timestamp: 0.033,
objects: vec![
YoloObject {
class_name: "person".to_string(),
class_id: 0,
x: 110,
y: 210,
width: 52,
height: 102,
confidence: 0.88,
},
YoloObject {
class_name: "car".to_string(),
class_id: 2,
x: 310,
y: 155,
width: 82,
height: 62,
confidence: 0.78,
},
],
},
YoloFrame {
// 幀 5-9: 只有 dog
frame: 5,
timestamp: 0.166,
objects: vec![YoloObject {
class_name: "dog".to_string(),
class_id: 16,
x: 150,
y: 250,
width: 40,
height: 60,
confidence: 0.7,
}],
},
YoloFrame {
// 幀 6
frame: 6,
timestamp: 0.2,
objects: vec![YoloObject {
class_name: "dog".to_string(),
class_id: 16,
x: 155,
y: 255,
width: 42,
height: 62,
confidence: 0.68,
}],
},
];
let yolo_result = YoloResult {
frame_count: 7,
fps: 30.0,
frames,
};
let chunks = create_similarity_based_chunks(
1,
"test-uuid",
&yolo_result,
0,
30.0,
0.5, // similarity threshold
2, // min frames per chunk
);
// 應該有 2 個分片：一個是 person+car，一個是 dog
assert_eq!(chunks.len(), 2);
}
}
src/core/thumbnail/mod.rs
+2
View File
@@ -1,3 +1,5 @@
pub mod validator;
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use std::path::{Path, PathBuf};
src/core/thumbnail/validator.rs
+202
View File
@@ -0,0 +1,202 @@
use anyhow::{bail, Result};
pub const JPEG_MIN_SIZE: usize = 100;
pub const JPEG_SOI_MARKER: [u8; 3] = [0xFF, 0xD8, 0xFF];
pub const JPEG_EOI_MARKER: [u8; 2] = [0xFF, 0xD9];
pub fn validate_jpeg(data: &[u8]) -> Result<()> {
if data.len() < JPEG_MIN_SIZE {
bail!(
"JPEG too small: {} bytes (minimum {})",
data.len(),
JPEG_MIN_SIZE
);
}
if data[0..3] != JPEG_SOI_MARKER {
bail!(
"Invalid JPEG header: expected {:02X?}, got {:02X?}",
JPEG_SOI_MARKER,
&data[0..3]
);
}
if data[data.len() - 2..] != JPEG_EOI_MARKER {
bail!(
"Incomplete JPEG: missing EOI marker, got {:02X?}",
&data[data.len() - 2..]
);
}
Ok(())
}
pub fn is_valid_jpeg(data: &[u8]) -> bool {
validate_jpeg(data).is_ok()
}
pub fn jpeg_size_ok(data: &[u8]) -> bool {
data.len() >= JPEG_MIN_SIZE
}
pub fn jpeg_header_ok(data: &[u8]) -> bool {
data.len() >= 3 && data[0..3] == JPEG_SOI_MARKER
}
pub fn jpeg_footer_ok(data: &[u8]) -> bool {
data.len() >= 2 && data[data.len() - 2..] == JPEG_EOI_MARKER
}
pub fn validate_frame(frame: i64, total_frames: i64) -> Result<()> {
if frame < 0 {
bail!("Frame number cannot be negative: {}", frame);
}
if frame > total_frames {
bail!("Frame {} exceeds total frames {}", frame, total_frames);
}
Ok(())
}
pub fn validate_crop(
x: i32,
y: i32,
w: i32,
h: i32,
video_width: i32,
video_height: i32,
) -> Result<()> {
if x < 0 || y < 0 || w <= 0 || h <= 0 {
bail!(
"Invalid crop parameters: x={}, y={}, w={}, h={} (must be positive)",
x,
y,
w,
h
);
}
if x + w > video_width {
bail!(
"Crop width exceeds video: x+w={} > video_width={}",
x + w,
video_width
);
}
if y + h > video_height {
bail!(
"Crop height exceeds video: y+h={} > video_height={}",
y + h,
video_height
);
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_validate_jpeg_valid() {
let valid_jpeg = vec![
0xFF, 0xD8, 0xFF, // SOI marker
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D,
0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B,
0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29,
0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0x3E, 0x3F, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45,
0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53,
0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F, 0xFF,
0xD9, // EOI marker
];
assert!(validate_jpeg(&valid_jpeg).is_ok());
}
#[test]
fn test_validate_jpeg_too_small() {
let small_data = vec![0xFF, 0xD8, 0xFF, 0xFF, 0xD9];
assert!(validate_jpeg(&small_data).is_err());
}
#[test]
fn test_validate_jpeg_invalid_header() {
let invalid_header = vec![
0x00, 0x00, 0x00, // wrong header
0x00, 0x01, 0x02, 0x03, 0xFF, 0xD9,
];
assert!(validate_jpeg(&invalid_header).is_err());
}
#[test]
fn test_validate_jpeg_missing_footer() {
let missing_footer = vec![0xFF, 0xD8, 0xFF, 0x00, 0x01, 0x02, 0x03];
assert!(validate_jpeg(&missing_footer).is_err());
}
#[test]
fn test_validate_frame_valid() {
assert!(validate_frame(500, 1000).is_ok());
assert!(validate_frame(0, 1000).is_ok());
assert!(validate_frame(1000, 1000).is_ok());
}
#[test]
fn test_validate_frame_exceeds() {
assert!(validate_frame(1001, 1000).is_err());
assert!(validate_frame(-1, 1000).is_err());
}
#[test]
fn test_validate_crop_valid() {
assert!(validate_crop(100, 100, 200, 200, 1920, 1080).is_ok());
assert!(validate_crop(0, 0, 1920, 1080, 1920, 1080).is_ok());
}
#[test]
fn test_validate_crop_exceeds_width() {
assert!(validate_crop(1800, 100, 200, 200, 1920, 1080).is_err());
}
#[test]
fn test_validate_crop_exceeds_height() {
assert!(validate_crop(100, 900, 200, 200, 1920, 1080).is_err());
}
#[test]
fn test_validate_crop_negative() {
assert!(validate_crop(-1, 100, 200, 200, 1920, 1080).is_err());
assert!(validate_crop(100, -1, 200, 200, 1920, 1080).is_err());
}
#[test]
fn test_is_valid_jpeg() {
let valid_jpeg = vec![
0xFF, 0xD8, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A,
0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18,
0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26,
0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, 0x30, 0x31, 0x32, 0x33, 0x34,
0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0x3E, 0x3F, 0x40, 0x41, 0x42,
0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50,
0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E,
0x5F, 0xFF, 0xD9,
];
assert!(is_valid_jpeg(&valid_jpeg));
assert!(!is_valid_jpeg(&[0xFF, 0xD8, 0xFF, 0xFF, 0xD9])); // too small
}
#[test]
fn test_jpeg_helpers() {
let valid_jpeg = vec![
0xFF, 0xD8, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A,
0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18,
0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26,
0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, 0x30, 0x31, 0x32, 0x33, 0x34,
0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0x3E, 0x3F, 0x40, 0x41, 0x42,
0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50,
0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E,
0x5F, 0xFF, 0xD9,
];
assert!(jpeg_size_ok(&valid_jpeg));
assert!(jpeg_header_ok(&valid_jpeg));
assert!(jpeg_footer_ok(&valid_jpeg));
}
}
src/core/tmdb/probe.rs
+12 -13
View File
@@ -91,22 +91,21 @@ async fn upsert_identities_from_disk(
{
Ok(identity_file) => {
let identities_table = crate::core::db::schema::table_name("identities");
let uuid_clean = identity_file.identity_uuid.replace('-', "");
let result = sqlx::query(&format!(
"INSERT INTO {} (uuid, name, identity_type, source, status, tmdb_id, tmdb_profile, metadata) \
VALUES ($1::uuid, $2, 'people', 'tmdb', 'confirmed', $3, $4, $5::jsonb) \
VALUES (gen_random_uuid(), $1, 'people', 'tmdb', 'confirmed', $2, $3, $4::jsonb) \
ON CONFLICT (tmdb_id) WHERE tmdb_id IS NOT NULL DO UPDATE SET \
uuid = COALESCE({}.uuid, $1::uuid), \
tmdb_profile = COALESCE(EXCLUDED.tmdb_profile, {}.tmdb_profile), \
metadata = {}.metadata || $5::jsonb",
identities_table, identities_table, identities_table, identities_table
))
.bind(&identity_file.identity_uuid)
.bind(&identity_file.name)
.bind(identity_file.tmdb_id)
.bind(&identity_file.tmdb_profile)
.bind(&identity_file.metadata)
.execute(db.pool())
.await;
metadata = jsonb_deep_merge({}.metadata, $4::jsonb)",
identities_table, identities_table, identities_table
))
.bind(&identity_file.name)
.bind(identity_file.tmdb_id)
.bind(&identity_file.tmdb_profile)
.bind(&identity_file.metadata)
.execute(db.pool())
.await;
match result {
Ok(_) => {
@@ -226,7 +225,7 @@ pub async fn create_identities_from_data(
VALUES ($1, 'people', 'tmdb', 'confirmed', $2, $3, $4::jsonb) \
ON CONFLICT (tmdb_id) WHERE tmdb_id IS NOT NULL DO UPDATE SET \
tmdb_profile = COALESCE(EXCLUDED.tmdb_profile, {}.tmdb_profile), \
metadata = {}.metadata || $4::jsonb \
metadata = jsonb_deep_merge({}.metadata, $4::jsonb) \
RETURNING uuid",
identities_table, identities_table, identities_table
))