warren/momentry_core
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

refactor: remove face embedding architecture - single Qdrant _faces collection

Browse Source 
- Delete FaceEmbeddingDb module (face_embedding_db.rs)
- Stub match_faces_iterative, generate_seed_embeddings, tmdb_match_handler
- Remove sync_trace_embeddings, populate_face_embeddings_to_qdrant
- Remove embedding from face.json output (face_processor.py)
- Remove embedding from PG UPDATE (store_traced_faces.py)
- Remove workspace traces staging (checkin.rs, qdrant_workspace.rs)
- Fix tests: add pose_angle to Face, hand_nodes to TkgResult

Disabled functions (need reimplement with _faces):
- match_faces_iterative (identity agent)
- generate_seed_embeddings (TMDb seeds)
- tmdb_match_handler (TMDb matching)
- cluster_face_embeddings, search_similar_faces
- merge_traces_within_cuts
This commit is contained in:
Accusys
2026-06-24 22:27:09 +08:00
parent 360cb991e1
commit 074cdcdbed
60 changed files with 657 additions and 9454 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/checkin.rs
+1 -38
View File
@@ -145,42 +145,6 @@ pub async fn checkin(db: &PostgresDb, file_uuid: &str) -> Result<CheckinResult>
}
}
}
// Traces → production traces collection
let traces_coll = format!(
"{}_traces",
crate::core::config::REDIS_KEY_PREFIX
.as_str()
.trim_end_matches(':')
);
for point in &ws_data.traces {
if let Some(ref vector) = point.vector {
let payload_val: serde_json::Value =
serde_json::to_value(&point.payload).unwrap_or(serde_json::Value::Null);
let point_id: u64 = match point.id.parse::<u64>() {
Ok(id) => id,
Err(_) => {
use std::hash::{Hash, Hasher};
let mut hasher = std::collections::hash_map::DefaultHasher::new();
point.id.hash(&mut hasher);
hasher.finish()
}
};
if let Err(e) = qdrant
.upsert_vector_to_collection(
&traces_coll,
point_id,
vector,
Some(payload_val),
)
.await
{
warn!("Failed to checkin trace vector {}: {}", point.id, e);
} else {
vectors_moved += 1;
}
}
}
}
Err(e) => {
warn!("Failed to scroll Qdrant workspace for {}: {}", file_uuid, e);
@@ -297,10 +261,9 @@ pub async fn checkout(db: &PostgresDb, file_uuid: &str) -> Result<CheckoutResult
let prefix = crate::core::config::REDIS_KEY_PREFIX
.as_str()
.trim_end_matches(':');
let traces_coll = format!("{}_traces", prefix);
let voice_coll = format!("{}_voice", file_uuid);
for coll in &[traces_coll, voice_coll] {
for coll in &[voice_coll] {
if let Err(e) = QdrantDb::delete_by_uuid_from_collection(
&qdrant.client,
&qdrant.base_url,
src/core/db/face_embedding_db.rs
-950
View File
@@ -1,950 +0,0 @@
use anyhow::{Context, Result};
use reqwest::Client;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
pub struct FaceEmbeddingDb {
client: Client,
base_url: String,
api_key: String,
collection_name: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FaceEmbeddingPayload {
pub file_uuid: String,
pub trace_id: i32,
pub frame: i64,
pub bbox_x: f64,
pub bbox_y: f64,
pub bbox_w: f64,
pub bbox_h: f64,
pub confidence: f64,
pub yaw: f64,
pub pitch: f64,
pub roll: f64,
#[serde(skip_serializing_if = "Option::is_none")]
pub identity_uuid: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub identity_ref: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub stranger_ref: Option<String>,
#[serde(skip_serializing_if = "Option::is_none", rename = "type")]
pub r#type: Option<String>,
}
#[derive(Debug, Clone, Deserialize)]
pub struct FaceEmbeddingPoint {
pub id: String,
pub vector: Vec<f32>,
pub payload: FaceEmbeddingPayload,
pub score: f64,
}
impl FaceEmbeddingDb {
pub fn new() -> Self {
let schema = std::env::var("DATABASE_SCHEMA").unwrap_or_else(|_| "dev".to_string());
let collection_name = format!("{}_face_embeddings", schema);
let base_url =
std::env::var("QDRANT_URL").unwrap_or_else(|_| "http://localhost:6333".to_string());
let api_key = std::env::var("QDRANT_API_KEY")
.unwrap_or_else(|_| "Test3200Test3200Test3200".to_string());
Self {
client: Client::new(),
base_url,
api_key,
collection_name,
}
}
pub async fn init_collection(&self) -> Result<()> {
let url = format!("{}/collections/{}", self.base_url, self.collection_name);
let response = self
.client
.get(&url)
.header("api-key", &self.api_key)
.send()
.await?;
if response.status().is_success() {
tracing::info!(
"[FaceEmbedding] Collection {} already exists",
self.collection_name
);
return Ok(());
}
let create_url = format!("{}/collections/{}", self.base_url, self.collection_name);
let body = serde_json::json!({
"vectors": {
"size": 512,
"distance": "Cosine"
}
});
self.client
.put(&create_url)
.header("api-key", &self.api_key)
.header("Content-Type", "application/json")
.json(&body)
.send()
.await
.context("Failed to create face embeddings collection")?;
tracing::info!(
"[FaceEmbedding] Created collection {} (dim=512)",
self.collection_name
);
Ok(())
}
pub async fn upsert_embedding(
&self,
point_id: &str,
embedding: &[f32],
payload: &FaceEmbeddingPayload,
) -> Result<()> {
let url = format!(
"{}/collections/{}/points?wait=true",
self.base_url, self.collection_name
);
let body = serde_json::json!({
"points": [{
"id": point_id,
"vector": embedding,
"payload": payload
}]
});
let response = self
.client
.put(&url)
.header("api-key", &self.api_key)
.header("Content-Type", "application/json")
.json(&body)
.send()
.await
.context("Failed to upsert face embedding")?;
if !response.status().is_success() {
let text = response.text().await.unwrap_or_default();
anyhow::bail!("Qdrant upsert failed: {}", text);
}
Ok(())
}
pub async fn batch_upsert(
&self,
points: Vec<(String, Vec<f32>, FaceEmbeddingPayload)>,
) -> Result<usize> {
if points.is_empty() {
return Ok(0);
}
let url = format!(
"{}/collections/{}/points?wait=true",
self.base_url, self.collection_name
);
let body = serde_json::json!({
"points": points.iter().map(|(id, vec, payload)| {
// Parse id as u64 for Qdrant (requires integer or UUID)
let id_num: u64 = id.parse().unwrap_or(0);
serde_json::json!({
"id": id_num,
"vector": vec,
"payload": payload
})
}).collect::<Vec<_>>()
});
let response = self
.client
.put(&url)
.header("api-key", &self.api_key)
.header("Content-Type", "application/json")
.json(&body)
.send()
.await
.context("Failed to batch upsert face embeddings")?;
if !response.status().is_success() {
let status = response.status();
let text = response.text().await.unwrap_or_default();
anyhow::bail!("Qdrant batch upsert failed (HTTP {}): {}", status, text);
}
Ok(points.len())
}
pub async fn update_identity_by_trace(
&self,
file_uuid: &str,
trace_id: i32,
identity_uuid: &str,
) -> Result<usize> {
let url = format!(
"{}/collections/{}/points",
self.base_url, self.collection_name
);
let body = serde_json::json!({
"filter": {
"must": [
{
"key": "file_uuid",
"match": { "value": file_uuid }
},
{
"key": "trace_id",
"match": { "value": trace_id }
}
]
},
"payload": {
"identity_uuid": identity_uuid
}
});
let response = self
.client
.post(&url)
.header("api-key", &self.api_key)
.header("Content-Type", "application/json")
.json(&body)
.send()
.await
.context("Failed to update identity_uuid in Qdrant")?;
if !response.status().is_success() {
let text = response.text().await.unwrap_or_default();
anyhow::bail!("Qdrant identity update failed: {}", text);
}
tracing::info!(
"[FaceEmbedding] Updated identity_uuid={} for file={}, trace={}",
identity_uuid, file_uuid, trace_id
);
Ok(1)
}
pub async fn clear_identity_by_trace(
&self,
file_uuid: &str,
trace_id: i32,
) -> Result<usize> {
let url = format!(
"{}/collections/{}/points",
self.base_url, self.collection_name
);
let body = serde_json::json!({
"filter": {
"must": [
{
"key": "file_uuid",
"match": { "value": file_uuid }
},
{
"key": "trace_id",
"match": { "value": trace_id }
}
]
},
"payload": {
"identity_uuid": null
}
});
let response = self
.client
.post(&url)
.header("api-key", &self.api_key)
.header("Content-Type", "application/json")
.json(&body)
.send()
.await
.context("Failed to clear identity_uuid in Qdrant")?;
if !response.status().is_success() {
let text = response.text().await.unwrap_or_default();
anyhow::bail!("Qdrant identity clear failed: {}", text);
}
tracing::info!(
"[FaceEmbedding] Cleared identity_uuid for file={}, trace={}",
file_uuid, trace_id
);
Ok(1)
}
pub async fn search_similar(
&self,
query_embedding: &[f32],
file_uuid: Option<&str>,
limit: usize,
threshold: f64,
) -> Result<Vec<FaceEmbeddingPoint>> {
let url = format!(
"{}/collections/{}/points/search",
self.base_url, self.collection_name
);
let mut filter = serde_json::json!({});
if let Some(fu) = file_uuid {
filter = serde_json::json!({
"must": [{
"key": "file_uuid",
"match": { "value": fu }
}]
});
}
let body = serde_json::json!({
"vector": query_embedding,
"limit": limit,
"with_payload": true,
"with_vector": false,
"filter": filter
});
let response = self
.client
.post(&url)
.header("api-key", &self.api_key)
.header("Content-Type", "application/json")
.json(&body)
.send()
.await
.context("Failed to search face embeddings")?;
let status = response.status();
let text = response.text().await.unwrap_or_default();
if !status.is_success() {
anyhow::bail!("Qdrant search failed: {} - {}", status, text);
}
#[derive(Deserialize)]
struct SearchResult {
result: Vec<PointResult>,
}
#[derive(Deserialize)]
struct PointResult {
id: serde_json::Value,
score: f64,
payload: HashMap<String, serde_json::Value>,
}
let parsed: SearchResult =
serde_json::from_str(&text).context("Failed to parse Qdrant search response")?;
let results: Vec<FaceEmbeddingPoint> = parsed
.result
.into_iter()
.filter(|r| r.score >= threshold)
.map(|r| {
let id = match r.id {
serde_json::Value::String(s) => s,
serde_json::Value::Number(n) => n.to_string(),
_ => "unknown".to_string(),
};
let payload = FaceEmbeddingPayload {
file_uuid: r
.payload
.get("file_uuid")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
trace_id: r
.payload
.get("trace_id")
.and_then(|v| v.as_i64())
.unwrap_or(0) as i32,
frame: r.payload.get("frame").and_then(|v| v.as_i64()).unwrap_or(0),
bbox_x: r
.payload
.get("bbox_x")
.and_then(|v| v.as_f64())
.unwrap_or(0.0),
bbox_y: r
.payload
.get("bbox_y")
.and_then(|v| v.as_f64())
.unwrap_or(0.0),
bbox_w: r
.payload
.get("bbox_w")
.and_then(|v| v.as_f64())
.unwrap_or(0.0),
bbox_h: r
.payload
.get("bbox_h")
.and_then(|v| v.as_f64())
.unwrap_or(0.0),
confidence: r
.payload
.get("confidence")
.and_then(|v| v.as_f64())
.unwrap_or(0.0),
yaw: r.payload.get("yaw").and_then(|v| v.as_f64()).unwrap_or(0.0),
pitch: r
.payload
.get("pitch")
.and_then(|v| v.as_f64())
.unwrap_or(0.0),
roll: r
.payload
.get("roll")
.and_then(|v| v.as_f64())
.unwrap_or(0.0),
identity_uuid: r
.payload
.get("identity_uuid")
.and_then(|v| v.as_str())
.map(|s| s.to_string()),
identity_ref: r
.payload
.get("identity_ref")
.and_then(|v| v.as_str())
.map(|s| s.to_string()),
stranger_ref: r
.payload
.get("stranger_ref")
.and_then(|v| v.as_str())
.map(|s| s.to_string()),
r#type: r
.payload
.get("type")
.and_then(|v| v.as_str())
.map(|s| s.to_string()),
};
FaceEmbeddingPoint {
id,
vector: vec![], // Not returned with_vector=false
payload,
score: r.score,
}
})
.collect();
Ok(results)
}
pub async fn get_embeddings_by_trace(
&self,
file_uuid: &str,
trace_id: i32,
) -> Result<Vec<(String, Vec<f32>)>> {
let url = format!(
"{}/collections/{}/points/scroll",
self.base_url, self.collection_name
);
let body = serde_json::json!({
"limit": 1000,
"with_payload": true,
"with_vector": true,
"filter": {
"must": [
{"key": "file_uuid", "match": { "value": file_uuid }},
{"key": "trace_id", "match": { "value": trace_id }}
]
}
});
let response = self
.client
.post(&url)
.header("api-key", &self.api_key)
.header("Content-Type", "application/json")
.json(&body)
.send()
.await
.context("Failed to scroll face embeddings")?;
let status = response.status();
let text = response.text().await.unwrap_or_default();
if !status.is_success() {
anyhow::bail!("Qdrant scroll failed: {} - {}", status, text);
}
#[derive(Deserialize)]
struct ScrollResult {
result: ScrollPoints,
}
#[derive(Deserialize)]
struct ScrollPoints {
points: Vec<PointResult>,
}
#[derive(Deserialize)]
struct PointResult {
id: serde_json::Value,
vector: Vec<f32>,
}
let parsed: ScrollResult =
serde_json::from_str(&text).context("Failed to parse Qdrant scroll response")?;
let results: Vec<(String, Vec<f32>)> = parsed
.result
.points
.into_iter()
.map(|r| {
let id = match r.id {
serde_json::Value::String(s) => s,
serde_json::Value::Number(n) => n.to_string(),
_ => "unknown".to_string(),
};
(id, r.vector)
})
.collect();
Ok(results)
}
pub async fn get_all_embeddings_for_file(
&self,
file_uuid: &str,
) -> Result<Vec<(String, Vec<f32>, FaceEmbeddingPayload)>> {
let url = format!(
"{}/collections/{}/points/scroll",
self.base_url, self.collection_name
);
let body = serde_json::json!({
"limit": 10000,
"with_payload": true,
"with_vector": true,
"filter": {
"must": [
{"key": "file_uuid", "match": { "value": file_uuid }}
]
}
});
let response = self
.client
.post(&url)
.header("api-key", &self.api_key)
.header("Content-Type", "application/json")
.json(&body)
.send()
.await
.context("Failed to scroll face embeddings")?;
let status = response.status();
let text = response.text().await.unwrap_or_default();
if !status.is_success() {
anyhow::bail!("Qdrant scroll failed: {} - {}", status, text);
}
#[derive(Deserialize)]
struct ScrollResult {
result: ScrollPoints,
}
#[derive(Deserialize)]
struct ScrollPoints {
points: Vec<PointResult>,
}
#[derive(Deserialize)]
struct PointResult {
id: serde_json::Value,
vector: Vec<f32>,
payload: HashMap<String, serde_json::Value>,
}
let parsed: ScrollResult =
serde_json::from_str(&text).context("Failed to parse Qdrant scroll response")?;
let results: Vec<(String, Vec<f32>, FaceEmbeddingPayload)> = parsed
.result
.points
.into_iter()
.map(|r| {
let id = match r.id {
serde_json::Value::String(s) => s,
serde_json::Value::Number(n) => n.to_string(),
_ => "unknown".to_string(),
};
let payload = FaceEmbeddingPayload {
file_uuid: r
.payload
.get("file_uuid")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
trace_id: r
.payload
.get("trace_id")
.and_then(|v| v.as_i64())
.unwrap_or(0) as i32,
frame: r.payload.get("frame").and_then(|v| v.as_i64()).unwrap_or(0),
bbox_x: r
.payload
.get("bbox_x")
.and_then(|v| v.as_f64())
.unwrap_or(0.0),
bbox_y: r
.payload
.get("bbox_y")
.and_then(|v| v.as_f64())
.unwrap_or(0.0),
bbox_w: r
.payload
.get("bbox_w")
.and_then(|v| v.as_f64())
.unwrap_or(0.0),
bbox_h: r
.payload
.get("bbox_h")
.and_then(|v| v.as_f64())
.unwrap_or(0.0),
confidence: r
.payload
.get("confidence")
.and_then(|v| v.as_f64())
.unwrap_or(0.0),
yaw: r.payload.get("yaw").and_then(|v| v.as_f64()).unwrap_or(0.0),
pitch: r
.payload
.get("pitch")
.and_then(|v| v.as_f64())
.unwrap_or(0.0),
roll: r
.payload
.get("roll")
.and_then(|v| v.as_f64())
.unwrap_or(0.0),
identity_uuid: r
.payload
.get("identity_uuid")
.and_then(|v| v.as_str())
.map(|s| s.to_string()),
identity_ref: r
.payload
.get("identity_ref")
.and_then(|v| v.as_str())
.map(|s| s.to_string()),
stranger_ref: r
.payload
.get("stranger_ref")
.and_then(|v| v.as_str())
.map(|s| s.to_string()),
r#type: r
.payload
.get("type")
.and_then(|v| v.as_str())
.map(|s| s.to_string()),
};
(id, r.vector, payload)
})
.collect();
Ok(results)
}
pub async fn delete_file_embeddings(&self, file_uuid: &str) -> Result<usize> {
let url = format!(
"{}/collections/{}/points/delete?wait=true",
self.base_url, self.collection_name
);
let body = serde_json::json!({
"filter": {
"must": [
{"key": "file_uuid", "match": { "value": file_uuid }}
]
}
});
let response = self
.client
.post(&url)
.header("api-key", &self.api_key)
.header("Content-Type", "application/json")
.json(&body)
.send()
.await
.context("Failed to delete face embeddings")?;
if !response.status().is_success() {
let text = response.text().await.unwrap_or_default();
anyhow::bail!("Qdrant delete failed: {}", text);
}
Ok(0)
}
pub async fn upsert_seed_embedding(
&self,
identity_uuid: &str,
identity_name: &str,
tmdb_id: i32,
embedding: &[f32],
) -> Result<()> {
let url = format!(
"{}/collections/{}/points?wait=true",
self.base_url, self.collection_name
);
let point_id = identity_uuid.to_string();
let payload = serde_json::json!({
"file_uuid": "",
"trace_id": 0,
"frame": 0,
"bbox_x": 0.0,
"bbox_y": 0.0,
"bbox_w": 0.0,
"bbox_h": 0.0,
"confidence": 0.0,
"yaw": 0.0,
"pitch": 0.0,
"roll": 0.0,
"identity_uuid": identity_uuid,
"identity_ref": serde_json::Value::Null,
"stranger_ref": serde_json::Value::Null,
"identity_name": identity_name,
"tmdb_id": tmdb_id,
"type": "identity_seed",
});
let body = serde_json::json!({
"points": [{
"id": point_id,
"vector": embedding,
"payload": payload
}]
});
let response = self
.client
.put(&url)
.header("api-key", &self.api_key)
.header("Content-Type", "application/json")
.json(&body)
.send()
.await
.context("Failed to upsert seed embedding")?;
if !response.status().is_success() {
let text = response.text().await.unwrap_or_default();
anyhow::bail!("Qdrant seed upsert failed: {}", text);
}
tracing::info!(
"[SeedEmbedding] Stored seed for identity_uuid={}, name={}",
identity_uuid, identity_name
);
Ok(())
}
pub async fn get_seed_embeddings(
&self,
) -> Result<Vec<(String, String, Vec<f32>)>> {
let url = format!(
"{}/collections/{}/points/scroll",
self.base_url, self.collection_name
);
let body = serde_json::json!({
"limit": 10000,
"with_payload": true,
"with_vector": true,
"filter": {
"must": [
{"key": "type", "match": { "value": "identity_seed" }}
]
}
});
let response = self
.client
.post(&url)
.header("api-key", &self.api_key)
.header("Content-Type", "application/json")
.json(&body)
.send()
.await
.context("Failed to scroll seed embeddings")?;
let status = response.status();
let text = response.text().await.unwrap_or_default();
if !status.is_success() {
anyhow::bail!("Qdrant scroll failed: {} - {}", status, text);
}
#[derive(Deserialize)]
struct ScrollResult {
result: ScrollPoints,
}
#[derive(Deserialize)]
struct ScrollPoints {
points: Vec<PointResult>,
}
#[derive(Deserialize)]
struct PointResult {
id: serde_json::Value,
vector: Vec<f32>,
payload: HashMap<String, serde_json::Value>,
}
let parsed: ScrollResult =
serde_json::from_str(&text).context("Failed to parse Qdrant scroll response")?;
let results: Vec<(String, String, Vec<f32>)> = parsed
.result
.points
.into_iter()
.filter_map(|r| {
let identity_uuid = r
.payload
.get("identity_uuid")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
let identity_name = r
.payload
.get("identity_name")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
if identity_uuid.is_empty() {
None
} else {
Some((identity_uuid, identity_name, r.vector))
}
})
.collect();
Ok(results)
}
pub async fn update_identity_ref_by_trace(
&self,
file_uuid: &str,
trace_id: i32,
identity_ref: &str,
) -> Result<usize> {
let url = format!(
"{}/collections/{}/points/payload",
self.base_url, self.collection_name
);
let body = serde_json::json!({
"filter": {
"must": [
{
"key": "file_uuid",
"match": { "value": file_uuid }
},
{
"key": "trace_id",
"match": { "value": trace_id }
}
]
},
"payload": {
"identity_ref": identity_ref
}
});
let response = self
.client
.post(&url)
.header("api-key", &self.api_key)
.header("Content-Type", "application/json")
.json(&body)
.send()
.await
.context("Failed to update identity_ref in Qdrant")?;
if !response.status().is_success() {
let text = response.text().await.unwrap_or_default();
anyhow::bail!("Qdrant identity_ref update failed: {}", text);
}
tracing::info!(
"[FaceEmbedding] Updated identity_ref={} for file={}, trace={}",
identity_ref, file_uuid, trace_id
);
Ok(1)
}
pub async fn update_stranger_ref_by_trace(
&self,
file_uuid: &str,
trace_id: i32,
stranger_ref: &str,
) -> Result<usize> {
let url = format!(
"{}/collections/{}/points/payload",
self.base_url, self.collection_name
);
let body = serde_json::json!({
"filter": {
"must": [
{
"key": "file_uuid",
"match": { "value": file_uuid }
},
{
"key": "trace_id",
"match": { "value": trace_id }
}
]
},
"payload": {
"stranger_ref": stranger_ref
}
});
let response = self
.client
.post(&url)
.header("api-key", &self.api_key)
.header("Content-Type", "application/json")
.json(&body)
.send()
.await
.context("Failed to update stranger_ref in Qdrant")?;
if !response.status().is_success() {
let text = response.text().await.unwrap_or_default();
anyhow::bail!("Qdrant stranger_ref update failed: {}", text);
}
tracing::info!(
"[FaceEmbedding] Updated stranger_ref={} for file={}, trace={}",
stranger_ref, file_uuid, trace_id
);
Ok(1)
}
}
impl Default for FaceEmbeddingDb {
fn default() -> Self {
Self::new()
}
}
src/core/db/mod.rs
-2
View File
@@ -32,14 +32,12 @@ pub trait VectorStore: Send + Sync {
async fn search(&self, query_vector: &[f32], limit: usize) -> Result<Vec<SearchResult>>;
}
pub mod face_embedding_db;
pub mod identity_merge_history;
pub mod mongodb_db;
pub mod postgres_db;
pub mod qdrant_db;
pub mod redis_client;
pub mod redis_db;
pub use face_embedding_db::{FaceEmbeddingDb, FaceEmbeddingPayload, FaceEmbeddingPoint};
pub use identity_merge_history::{
AliasEntry, FacesTransferred, IdentityMergeHistory, IdentityMergeHistoryStore,
IdentitySnapshot, MergeHistoryEntry, MergeHistoryQuery, MergeParams, TargetIdentitySnapshot,
src/core/db/postgres_db.rs
+17 -92
View File
@@ -448,10 +448,7 @@ pub enum ProcessorType {
Hand,
Asrx,
Scene,
Story,
FiveW1H,
Appearance,
MediaPipe,
FaceCluster,
}
@@ -488,10 +485,7 @@ impl ProcessorType {
ProcessorType::Hand => "hand",
ProcessorType::Asrx => "asrx",
ProcessorType::Scene => "scene",
ProcessorType::Story => "story",
ProcessorType::FiveW1H => "5w1h",
ProcessorType::Appearance => "appearance",
ProcessorType::MediaPipe => "mediapipe",
ProcessorType::FaceCluster => "face_cluster",
}
}
@@ -507,10 +501,7 @@ impl ProcessorType {
"hand" => Some(ProcessorType::Hand),
"asrx" => Some(ProcessorType::Asrx),
"scene" => Some(ProcessorType::Scene),
"story" => Some(ProcessorType::Story),
"5w1h" => Some(ProcessorType::FiveW1H),
"appearance" => Some(ProcessorType::Appearance),
"mediapipe" => Some(ProcessorType::MediaPipe),
"face_cluster" => Some(ProcessorType::FaceCluster),
_ => None,
}
@@ -527,10 +518,7 @@ impl ProcessorType {
ProcessorType::Hand => 0.4,
ProcessorType::Asrx => 0.8,
ProcessorType::Scene => 0.3,
ProcessorType::Story => 0.1,
ProcessorType::FiveW1H => 0.1,
ProcessorType::Appearance => 0.3,
ProcessorType::MediaPipe => 0.3,
ProcessorType::FaceCluster => 0.7,
}
}
@@ -538,7 +526,6 @@ impl ProcessorType {
pub fn uses_gpu(&self) -> bool {
match self {
ProcessorType::Yolo | ProcessorType::Face | ProcessorType::Pose | ProcessorType::Hand => true,
ProcessorType::MediaPipe | ProcessorType::FaceCluster => false,
_ => false,
}
}
@@ -554,10 +541,7 @@ impl ProcessorType {
ProcessorType::Hand => 1024,
ProcessorType::Asrx => 2048,
ProcessorType::Scene => 512,
ProcessorType::Story => 256,
ProcessorType::FiveW1H => 256,
ProcessorType::Appearance => 512,
ProcessorType::MediaPipe => 1024,
ProcessorType::FaceCluster => 1024,
}
}
@@ -573,10 +557,7 @@ impl ProcessorType {
ProcessorType::Hand => Some("vision/hand_pose"),
ProcessorType::Asrx => Some("speechbrain/ecapa-tdnn"),
ProcessorType::Scene => Some("places365"),
ProcessorType::Story => None,
ProcessorType::FiveW1H => Some("gemma4"),
ProcessorType::Appearance => None,
ProcessorType::MediaPipe => Some("mediapipe/holistic"),
ProcessorType::FaceCluster => Some("sklearn/agglomerative"),
}
}
@@ -585,17 +566,8 @@ impl ProcessorType {
match self {
ProcessorType::Asrx => vec![ProcessorType::Cut, ProcessorType::Asr],
ProcessorType::Scene => vec![ProcessorType::Cut],
ProcessorType::Story => vec![
ProcessorType::Asrx,
ProcessorType::Cut,
ProcessorType::Yolo,
ProcessorType::Face,
],
ProcessorType::FiveW1H => vec![ProcessorType::Story],
ProcessorType::Appearance => vec![ProcessorType::Pose],
ProcessorType::FaceCluster => vec![ProcessorType::Face],
ProcessorType::Hand => vec![],
ProcessorType::MediaPipe => vec![],
_ => vec![],
}
}
@@ -623,15 +595,12 @@ impl ProcessorType {
| ProcessorType::Pose
| ProcessorType::Hand
| ProcessorType::Appearance
| ProcessorType::MediaPipe
| ProcessorType::FaceCluster => PipelineType::Frame,
ProcessorType::Cut
| ProcessorType::Asr
| ProcessorType::Asrx
| ProcessorType::Scene
| ProcessorType::Story
| ProcessorType::FiveW1H => PipelineType::Time,
| ProcessorType::Scene => PipelineType::Time,
}
}
}
@@ -2612,76 +2581,32 @@ sqlx::query(
Ok(results)
}
/// Face clustering: group unregistered faces within same trace by embedding similarity
/// Face clustering: disabled - embedding column no longer used
pub async fn cluster_face_embeddings(
&self,
file_uuid: &str,
similarity_threshold: f64,
_similarity_threshold: f64,
) -> Result<Vec<FaceClusterGroup>> {
let table = schema::table_name("face_detections");
let rows = sqlx::query_as::<_, (String, i64)>(&format!(
r#"
SELECT trace_id::text, COUNT(DISTINCT frame_number) as frame_count
FROM {}
WHERE file_uuid = $1
AND embedding IS NOT NULL
AND identity_id IS NULL
GROUP BY trace_id
ORDER BY frame_count DESC
"#,
table
))
.bind(file_uuid)
.fetch_all(&self.pool)
.await?;
Ok(rows
.into_iter()
.map(|(trace_id, frame_count)| FaceClusterGroup {
trace_id,
frame_count: frame_count as i32,
})
.collect())
tracing::warn!(
"[cluster_face_embeddings] Disabled - embedding column removed for {}",
file_uuid
);
Ok(Vec::new())
}
/// Search similar faces by embedding via pgvector cosine distance
/// Search similar faces: disabled - embedding column no longer used
pub async fn search_similar_faces(
&self,
query_embedding: &[f32],
_query_embedding: &[f32],
file_uuid: &str,
limit: i64,
threshold: f64,
_limit: i64,
_threshold: f64,
) -> Result<Vec<SimilarFaceResult>> {
let table = schema::table_name("face_detections");
let rows = sqlx::query_as::<_, (i32, i32, f64)>(&format!(
r#"
SELECT id, trace_id,
1 - (embedding::vector <=> $1::vector) as similarity
FROM {}
WHERE file_uuid = $2
AND embedding IS NOT NULL
AND 1 - (embedding::vector <=> $1::vector) >= $3
ORDER BY embedding::vector <=> $1::vector
LIMIT $4
"#,
table
))
.bind(query_embedding)
.bind(file_uuid)
.bind(threshold)
.bind(limit)
.fetch_all(&self.pool)
.await?;
Ok(rows
.into_iter()
.map(|(id, trace_id, similarity)| SimilarFaceResult {
id,
trace_id,
similarity,
bbox: String::new(),
})
.collect())
tracing::warn!(
"[search_similar_faces] Disabled - embedding column removed for {}",
file_uuid
);
Ok(Vec::new())
}
// ==========================================
src/core/db/qdrant_db.rs
-149
View File
@@ -768,45 +768,6 @@ impl QdrantDb {
Ok(result.result.points_count)
}
/// Store face embedding with trace_id + frame_number payload
pub async fn upsert_face_embedding(
&self,
point_id: u64,
vector: &[f32],
file_uuid: &str,
trace_id: i32,
frame_number: i64,
) -> Result<()> {
let url = format!(
"{}/collections/{}/points?wait=true",
self.base_url, self.collection_name
);
let mut payload_map = std::collections::HashMap::new();
payload_map.insert("file_uuid".to_string(), serde_json::json!(file_uuid));
payload_map.insert("trace_id".to_string(), serde_json::json!(trace_id));
payload_map.insert("frame_number".to_string(), serde_json::json!(frame_number));
payload_map.insert("type".to_string(), serde_json::json!("face_embedding"));
let point = serde_json::json!({
"points": [{
"id": point_id,
"vector": vector,
"payload": payload_map
}]
});
let resp = self
.client
.put(&url)
.header("api-key", &self.api_key)
.json(&point)
.send()
.await?;
if !resp.status().is_success() {
anyhow::bail!("Qdrant upsert face failed: {}", resp.status());
}
Ok(())
}
/// Store chunk embedding with parent-child metadata
pub async fn upsert_chunk_embedding(
&self,
@@ -883,113 +844,3 @@ impl VectorStore for QdrantDb {
self.search(query_vector, limit).await
}
}
pub async fn sync_trace_embeddings(file_uuid: &str) -> Result<()> {
use crate::core::config::DATABASE_URL;
use sqlx::Row;
let pool = sqlx::PgPool::connect(&DATABASE_URL).await?;
let table = crate::core::db::schema::table_name("face_detections");
let qdrant = QdrantDb::new();
let collection = format!(
"{}_traces",
crate::core::config::REDIS_KEY_PREFIX
.as_str()
.trim_end_matches(':')
);
qdrant.ensure_collection(&collection, 512).await?;
// Read all face_detections with embeddings, grouped by trace_id in Rust
let rows = sqlx::query(&format!(
"SELECT trace_id, embedding FROM {} \
WHERE file_uuid = $1 AND embedding IS NOT NULL AND trace_id IS NOT NULL \
AND ((metadata->>'qc_ok')::boolean IS NULL OR (metadata->>'qc_ok')::boolean = true)",
table
))
.bind(file_uuid)
.fetch_all(&pool)
.await?;
let mut trace_faces: std::collections::HashMap<i32, Vec<Vec<f32>>> =
std::collections::HashMap::new();
let mut trace_stats: std::collections::HashMap<i32, (i64, i64, i64)> =
std::collections::HashMap::new(); // (count, min_frame, max_frame)
for row in &rows {
let tid: Option<i32> = row.get(0);
let emb: Option<Vec<f32>> = row.get(1);
if let (Some(tid), Some(emb)) = (tid, emb) {
trace_faces.entry(tid).or_default().push(emb);
let entry = trace_stats.entry(tid).or_insert((0, i64::MAX, i64::MIN));
entry.0 += 1;
}
}
// Compute average embedding per trace
struct AvgTrace {
tid: i32,
avg_emb: Vec<f32>,
frame_count: i64,
}
let mut trace_avgs: Vec<AvgTrace> = Vec::new();
for (&tid, faces) in &trace_faces {
let dim = faces[0].len();
let mut avg = vec![0.0f32; dim];
for face in faces {
for (i, &v) in face.iter().enumerate() {
avg[i] += v;
}
}
let n = faces.len() as f32;
for v in &mut avg {
*v /= n;
}
let stats = trace_stats.get(&tid).unwrap_or(&(0, 0, 0));
trace_avgs.push(AvgTrace {
tid,
avg_emb: avg,
frame_count: stats.0,
});
}
// Push to Qdrant in batches
// Point ID: hash(file_uuid + trace_id) for global uniqueness
for chunk in trace_avgs.chunks(500) {
let batch: Vec<(u64, &[f32], Option<serde_json::Value>)> = chunk
.iter()
.map(|t| {
let point_id = {
use sha2::{Digest, Sha256};
let mut hasher = Sha256::new();
hasher.update(file_uuid.as_bytes());
hasher.update(b"_");
hasher.update(t.tid.to_string().as_bytes());
let hash = hasher.finalize();
u64::from_be_bytes(hash[0..8].try_into().unwrap())
};
(
point_id,
t.avg_emb.as_slice(),
Some(serde_json::json!({
"trace_id": t.tid,
"file_uuid": file_uuid,
"frame_count": t.frame_count,
"source": "trace",
})),
)
})
.collect();
qdrant.upsert_vectors_batch(&collection, &batch).await?;
}
tracing::info!(
"Synced {} trace embeddings to Qdrant for {}",
trace_faces.len(),
file_uuid
);
Ok(())
}
src/core/db/qdrant_workspace.rs
+1 -22
View File
@@ -187,34 +187,13 @@ impl QdrantWorkspace {
.await
}
pub async fn upsert_face_embedding(
&self,
point_id: u64,
vector: &[f32],
file_uuid: &str,
trace_id: i32,
frame_number: i64,
) -> Result<()> {
let payload = serde_json::json!({
"file_uuid": file_uuid,
"trace_id": trace_id,
"frame_number": frame_number,
"type": "face_embedding",
});
self.upsert_vector(&self.traces_collection(), point_id, vector, Some(payload))
.await
}
/// Scroll all points for a file from all workspace collections.
/// Used during checkin to read vectors before moving to production.
pub async fn scroll_by_file_uuid(&self, file_uuid: &str) -> Result<WorkspaceScrollResult> {
let chunks = self
.scroll_collection(&self.chunks_collection(), file_uuid)
.await?;
let traces = self
.scroll_collection(&self.traces_collection(), file_uuid)
.await?;
Ok(WorkspaceScrollResult { chunks, traces })
Ok(WorkspaceScrollResult { chunks, traces: Vec::new() })
}
async fn scroll_collection(
src/core/llm/client.rs
+5 -32
View File
@@ -1,7 +1,7 @@
use anyhow::Result;
use serde::{Deserialize, Serialize};
use std::time::Duration;
use tracing::{debug, error, warn};
use tracing::{debug, error};
use crate::core::config;
use crate::core::llm::function_calling::LLM_CLIENT;
@@ -31,44 +31,17 @@ struct Choice {
message: ChatMessage,
}
/// Generates a 5W1H+ summary for a given scene context.
/// Context should include the combined text of all sentences in the scene.
pub async fn generate_5w1h_summary(scene_text: &str) -> Result<String> {
if !*config::llm::SUMMARY_ENABLED {
warn!("LLM Summary is disabled via config");
return Ok("LLM Disabled".to_string());
}
let prompt = format!(
r#"Analyze the following video scene transcript and provide a concise 5W1H+ summary in JSON format.
Focus on: Who, What, Where, When, Why, How, and Key Objects/Actions.
Transcript:
"{}"
Output format:
{{
"who": "...",
"what": "...",
"where": "...",
"when": "...",
"why": "...",
"how": "...",
"summary": "..."
}}"#,
scene_text
);
pub async fn ask_llm(prompt: &str, system_prompt: &str) -> Result<String> {
let req = ChatRequest {
model: (*config::llm::SUMMARY_MODEL).clone(),
messages: vec![
ChatMessage {
role: "system".to_string(),
content: "You are an expert video analyst assistant.".to_string(),
content: system_prompt.to_string(),
},
ChatMessage {
role: "user".to_string(),
content: prompt,
content: prompt.to_string(),
},
],
temperature: 0.1,
@@ -76,7 +49,7 @@ pub async fn generate_5w1h_summary(scene_text: &str) -> Result<String> {
stream: false,
};
debug!("Calling LLM for summary: {}", *config::llm::SUMMARY_URL);
debug!("Calling LLM: {}", *config::llm::SUMMARY_URL);
let res = LLM_CLIENT
.post(&*config::llm::SUMMARY_URL)
src/core/person_identity.rs
+1
View File
@@ -71,6 +71,7 @@ pub struct BindIdentityRequest {
pub file_uuid: String,
pub face_id: Option<String>,
pub id: Option<i64>,
pub expand_to_trace: Option<bool>,
}
#[derive(Debug, Clone, Deserialize, Serialize)]
src/core/processor/face.rs
+3
View File
@@ -103,6 +103,7 @@ mod tests {
confidence: 0.95,
embedding: Some(vec![0.1, 0.2, 0.3]),
landmarks: Some(serde_json::json!([[10.0, 20.0], [30.0, 40.0]])),
pose_angle: None,
attributes: Some(FaceAttributes {
age: Some(30),
gender: Some("male".to_string()),
@@ -174,6 +175,7 @@ mod tests {
confidence: 0.5,
embedding: None,
landmarks: None,
pose_angle: None,
attributes: None,
};
assert!(face.confidence >= 0.0 && face.confidence <= 1.0);
@@ -190,6 +192,7 @@ mod tests {
confidence: 0.95,
embedding: Some(vec![0.1; 512]),
landmarks: None,
pose_angle: None,
attributes: Some(FaceAttributes {
age: Some(35),
gender: Some("male".to_string()),
src/core/processor/mediapipe.rs
-96
View File
@@ -1,96 +0,0 @@
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use std::time::Duration;
use super::executor::PythonExecutor;
const MEDIAPIPE_TIMEOUT: Duration = Duration::from_secs(7200);
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipeResult {
pub frame_count: u64,
pub fps: f64,
pub frames: Vec<MediaPipeFrame>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipeFrame {
pub frame: u64,
pub timestamp: f64,
pub persons: Vec<MediaPipePerson>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipePerson {
pub person_id: u64,
pub pose: Option<MediaPipePose>,
pub left_hand: Option<MediaPipeHand>,
pub right_hand: Option<MediaPipeHand>,
pub face_mesh: Option<MediaPipeFaceMesh>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipePose {
pub landmarks: Vec<Vec<f64>>,
pub keypoints_33: Option<Vec<String>>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipeHand {
pub landmarks: Vec<Vec<f64>>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipeFaceMesh {
pub landmarks: Vec<Vec<f64>>,
}
pub async fn process_mediapipe(
video_path: &str,
output_path: &str,
uuid: Option<&str>,
) -> Result<MediaPipeResult> {
// If mediapipe.json already exists (written by face_processor), skip
if std::path::Path::new(output_path).exists() {
let json_str = std::fs::read_to_string(output_path).context("Failed to read MEDIAPIPE output")?;
let result: MediaPipeResult =
serde_json::from_str(&json_str).context("Failed to parse MEDIAPIPE output")?;
tracing::info!("[MEDIAPIPE] Skipping (already exists): {} frames", result.frames.len());
return Ok(result);
}
let executor = PythonExecutor::new()?;
let script_name = "mediapipe_processor_v1.11.py";
let script_path = executor.script_path(script_name);
tracing::info!("[MEDIAPIPE] Starting MediaPipe Holistic: {}", video_path);
if !script_path.exists() {
tracing::warn!("[MEDIAPIPE] Script not found, returning empty result");
return Ok(MediaPipeResult {
frame_count: 0,
fps: 0.0,
frames: vec![],
});
}
executor
.run(
script_name,
&[video_path, output_path],
uuid,
"MEDIAPIPE",
Some(MEDIAPIPE_TIMEOUT),
)
.await
.with_context(|| format!("Failed to run {:?}", script_path))?;
let json_str =
std::fs::read_to_string(output_path).context("Failed to read MEDIAPIPE output")?;
let result: MediaPipeResult =
serde_json::from_str(&json_str).context("Failed to parse MEDIAPIPE output")?;
tracing::info!("[MEDIAPIPE] Result: {} frames", result.frames.len());
Ok(result)
}
src/core/processor/mediapipe_v2.rs
-203
View File
@@ -1,203 +0,0 @@
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::time::Duration;
use tokio::process::Command;
use tokio::time::timeout;
use super::executor::PythonExecutor;
const MEDIAPIPE_TIMEOUT: Duration = Duration::from_secs(7200);
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipeResult {
pub metadata: MediaPipeMetadata,
pub frames: HashMap<String, MediaPipeDictEntry>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipeMetadata {
pub fps: f64,
pub total_frames: i64,
pub processed_frames: i64,
pub sample_interval: i64,
pub width: i64,
pub height: i64,
pub processor: String,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipeDictEntry {
pub frame_number: i64,
pub timestamp: f64,
pub persons: Vec<MediaPipePerson>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipePerson {
pub person_id: i64,
#[serde(default)]
pub bbox: Option<MediaPipeBBox>,
pub face_mesh: Option<serde_json::Value>,
pub pose: Option<serde_json::Value>,
pub hands: MediaPipeHands,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipeBBox {
pub x: i64,
pub y: i64,
pub width: i64,
pub height: i64,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct MediaPipeHands {
pub left: Option<serde_json::Value>,
pub right: Option<serde_json::Value>,
}
pub async fn process_mediapipe_v2(
video_path: &str,
output_path: &str,
uuid: Option<&str>,
frames: Option<&[i64]>,
) -> Result<MediaPipeResult> {
let executor = PythonExecutor::new()?;
let script_path = executor.script_path("mediapipe_holistic_processor.py");
tracing::info!("[MEDIAPIPE] Starting MediaPipe Holistic: {}", video_path);
if !script_path.exists() {
anyhow::bail!("mediapipe_holistic_processor.py not found");
}
let mut cmd = Command::new(executor.python_path());
cmd.arg(&script_path).arg(video_path).arg(output_path);
// Use explicit frame list if provided, otherwise calculate sample_interval for ~8Hz
if let Some(frames) = frames {
let frames_str = frames
.iter()
.map(|f| f.to_string())
.collect::<Vec<_>>()
.join(",");
cmd.arg("--frames").arg(&frames_str);
tracing::info!("[MEDIAPIPE] 8Hz sampling: {} frames", frames.len());
} else {
let sample_interval = calculate_sample_interval(video_path).await;
cmd.arg("--sample-interval")
.arg(sample_interval.to_string());
}
if let Some(u) = uuid {
cmd.arg("--uuid").arg(u);
}
cmd.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped());
let child = cmd.spawn().context("Failed to run MEDIAPIPE processor")?;
let output = match timeout(MEDIAPIPE_TIMEOUT, child.wait_with_output()).await {
Ok(Ok(output)) => output,
Ok(Err(e)) => return Err(e).context("Failed to run MEDIAPIPE processor"),
Err(_) => anyhow::bail!(
"MEDIAPIPE processing timed out after {:?}",
MEDIAPIPE_TIMEOUT
),
};
let stderr = String::from_utf8_lossy(&output.stderr);
for line in stderr.lines() {
let trimmed = line.trim();
if trimmed.starts_with("MEDIAPIPE_START") {
tracing::info!("[MEDIAPIPE] Loading model...");
} else if trimmed.starts_with("MEDIAPIPE_FRAME:") {
let count = trimmed.trim_start_matches("MEDIAPIPE_FRAME:");
tracing::info!("[MEDIAPIPE] Processed {} frames...", count);
} else if trimmed.starts_with("MEDIAPIPE_COMPLETE:") {
let count = trimmed.trim_start_matches("MEDIAPIPE_COMPLETE:");
tracing::info!("[MEDIAPIPE] Completed! Total: {} frames", count);
} else if trimmed.starts_with("MEDIAPIPE_INFO:") {
let info = trimmed.trim_start_matches("MEDIAPIPE_INFO:");
tracing::info!("[MEDIAPIPE] {}", info);
} else if trimmed.starts_with("MEDIAPIPE_ERROR:") {
let err = trimmed.trim_start_matches("MEDIAPIPE_ERROR:");
tracing::error!("[MEDIAPIPE] {}", err);
}
}
tracing::info!("[MEDIAPIPE] stderr output:\n{}", stderr);
if !output.status.success() {
anyhow::bail!("MEDIAPIPE failed: {}", stderr);
}
let json_str =
std::fs::read_to_string(output_path).context("Failed to read MEDIAPIPE output")?;
let result: MediaPipeResult =
serde_json::from_str(&json_str).context("Failed to parse MEDIAPIPE output")?;
tracing::info!("[MEDIAPIPE] Result: {} frames", result.frames.len());
Ok(result)
}
async fn calculate_sample_interval(video_path: &str) -> i64 {
// Try ffprobe to get FPS, calculate sample_interval for ~8Hz
let probe_cmd = Command::new("ffprobe")
.args([
"-v",
"quiet",
"-print_format",
"json",
"-show_streams",
video_path,
])
.output()
.await;
if let Ok(output) = probe_cmd {
if output.status.success() {
if let Ok(json_str) = String::from_utf8(output.stdout) {
if let Ok(probe_data) = serde_json::from_str::<serde_json::Value>(&json_str) {
if let Some(streams) = probe_data["streams"].as_array() {
for stream in streams {
if stream["codec_type"] == "video" {
if let Some(fps_str) = stream["r_frame_rate"].as_str() {
// Parse "30000/1001" style fps
if let Some(fps) = parse_fractional_fps(fps_str) {
let interval = (fps / 8.0).round() as i64;
return interval.max(1);
}
}
if let Some(fps_val) = stream["avg_frame_rate"].as_str() {
if let Some(fps) = parse_fractional_fps(fps_val) {
let interval = (fps / 8.0).round() as i64;
return interval.max(1);
}
}
}
}
}
}
}
}
}
4 // Default: assume 30fps / 8 = ~4
}
fn parse_fractional_fps(s: &str) -> Option<f64> {
let parts: Vec<&str> = s.split('/').collect();
if parts.len() == 2 {
let num: f64 = parts[0].parse().ok()?;
let den: f64 = parts[1].parse().ok()?;
if den > 0.0 {
return Some(num / den);
}
}
s.parse::<f64>().ok()
}
src/core/processor/mod.rs
-7
View File
@@ -11,11 +11,9 @@ pub mod face_clustering;
pub mod face_recognition;
pub mod hand;
pub mod heuristic_scene;
pub mod mediapipe_v2;
pub mod ocr;
pub mod pose;
pub mod scene_classification;
pub mod story;
pub mod tkg;
pub mod yolo;
@@ -48,17 +46,12 @@ pub use heuristic_scene::{
build_heuristic_scene_meta, generate_scene_meta, CrowdSize, HeuristicSceneMeta,
SceneSegmentMeta,
};
pub use mediapipe_v2::{
process_mediapipe_v2, MediaPipeBBox, MediaPipeDictEntry, MediaPipeHands, MediaPipeMetadata,
MediaPipePerson, MediaPipeResult,
};
pub use ocr::{process_ocr, OcrFrame, OcrResult, OcrText};
pub use pose::{process_pose, Bbox, Keypoint, PersonPose, PoseFrame, PoseResult};
pub use scene_classification::{
load_scene_from_file, process_scene_classification, SceneClassificationResult, ScenePrediction,
SceneSegment,
};
pub use story::{process_story, StoryChildChunk, StoryParentChunk, StoryResult, StoryStats};
pub use tkg::{
build_tkg, query_auto_representative_frame, FrameTraceInfo, MainIdentityInfo,
RepresentativeFrameResult, TkgResult,
src/core/processor/story.rs
-690
View File
@@ -1,690 +0,0 @@
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use std::path::Path;
use std::time::Duration;
use super::executor::PythonExecutor;
const STORY_TIMEOUT: Duration = Duration::from_secs(3600);
// ── Input data structs (from JSON files) ──────────────────────────
#[derive(Debug, Deserialize)]
struct AsrData {
segments: Vec<AsrSegmentInput>,
}
#[derive(Debug, Deserialize)]
struct AsrSegmentInput {
#[serde(default, alias = "start")]
start_time: f64,
#[serde(default, alias = "end")]
end_time: f64,
#[serde(default)]
text: String,
#[serde(default)]
confidence: f64,
}
#[derive(Debug, Deserialize)]
struct CutData {
scenes: Vec<CutSceneInput>,
}
#[derive(Debug, Deserialize)]
struct CutSceneInput {
scene_number: Option<i64>,
#[allow(dead_code)]
start_frame: Option<i64>,
#[allow(dead_code)]
end_frame: Option<i64>,
start_time: Option<f64>,
end_time: Option<f64>,
}
// ── Output data structs ───────────────────────────────────────────
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct StoryResult {
pub child_chunks: Vec<StoryChildChunk>,
pub parent_chunks: Vec<StoryParentChunk>,
pub stats: StoryStats,
#[serde(default)]
pub metadata: serde_json::Value,
#[serde(default)]
pub parent_chunk_size: usize,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct StoryStats {
pub total_child_chunks: usize,
pub total_parent_chunks: usize,
pub asr_children: usize,
pub cut_children: usize,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct StoryChildChunk {
pub chunk_id: String,
pub chunk_type: String,
pub source: String,
pub start_time: f64,
pub end_time: f64,
#[serde(skip_serializing_if = "Option::is_none")]
pub text_content: Option<String>,
pub content: serde_json::Value,
#[serde(default)]
pub child_chunk_ids: Vec<String>,
pub parent_chunk_id: Option<String>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct StoryParentChunk {
pub chunk_id: String,
pub chunk_type: String,
pub source: String,
pub start_time: f64,
pub end_time: f64,
pub text_content: String,
pub content: serde_json::Value,
#[serde(default)]
pub child_chunk_ids: Vec<String>,
pub parent_chunk_id: Option<String>,
}
// ── Public API ────────────────────────────────────────────────────
pub async fn process_story(
video_path: &str,
output_path: &str,
uuid: Option<&str>,
) -> Result<StoryResult> {
// Try native Rust implementation first
let result = try_native_story(video_path, output_path, uuid);
if let Ok(r) = result {
return Ok(r);
}
// Fallback: Python script
tracing::warn!(
"[STORY] Native impl failed, falling back to Python: {:?}",
result.err()
);
let executor = PythonExecutor::new()?;
let script_path = executor.script_path("story_processor.py");
if !script_path.exists() {
return Ok(StoryResult {
child_chunks: vec![],
parent_chunks: vec![],
stats: StoryStats {
total_child_chunks: 0,
total_parent_chunks: 0,
asr_children: 0,
cut_children: 0,
},
metadata: serde_json::json!({}),
parent_chunk_size: 5,
});
}
executor
.run(
"story_processor.py",
&[video_path, output_path],
uuid,
"STORY",
Some(STORY_TIMEOUT),
)
.await
.with_context(|| format!("Failed to run {:?}", script_path))?;
let json_str = std::fs::read_to_string(output_path).context("Failed to read STORY output")?;
let result: StoryResult =
serde_json::from_str(&json_str).context("Failed to parse STORY output")?;
Ok(result)
}
// ── Native implementation ─────────────────────────────────────────
fn try_native_story(
_video_path: &str,
output_path: &str,
_uuid: Option<&str>,
) -> Result<StoryResult> {
let output_dir = Path::new(output_path).parent().unwrap_or(Path::new("."));
let basename = Path::new(output_path)
.file_stem()
.and_then(|s| s.to_str())
.and_then(|s| s.split('.').next())
.unwrap_or("unknown");
let asr_path = output_dir.join(format!("{}.asr.json", basename));
let cut_path = output_dir.join(format!("{}.cut.json", basename));
// ASR data is required; CUT is optional
let asr_data: AsrData = if asr_path.exists() {
let content = std::fs::read_to_string(&asr_path)
.with_context(|| format!("Failed to read {:?}", asr_path))?;
serde_json::from_str(&content).with_context(|| format!("Failed to parse {:?}", asr_path))?
} else {
AsrData { segments: vec![] }
};
let cut_data: CutData = if cut_path.exists() {
let content = std::fs::read_to_string(&cut_path)
.with_context(|| format!("Failed to read {:?}", cut_path))?;
serde_json::from_str(&content).with_context(|| format!("Failed to parse {:?}", cut_path))?
} else {
CutData { scenes: vec![] }
};
let parent_chunk_size: usize = 5;
// ── Build child chunks ────────────────────────────────────────
let mut child_chunks: Vec<StoryChildChunk> = Vec::new();
// ASR child chunks
for seg in &asr_data.segments {
let chunk_id = format!("asr_{:.1}_{:.1}", seg.start_time, seg.end_time);
child_chunks.push(StoryChildChunk {
chunk_id,
chunk_type: "asr".to_string(),
source: "asr".to_string(),
start_time: seg.start_time,
end_time: seg.end_time,
text_content: Some(seg.text.clone()),
content: serde_json::json!({
"text": seg.text,
"confidence": seg.confidence,
}),
child_chunk_ids: vec![],
parent_chunk_id: None,
});
}
// CUT child chunks
for scene in &cut_data.scenes {
let scene_num = scene.scene_number.unwrap_or(0);
let start_time = scene.start_time.unwrap_or(0.0);
let end_time = scene.end_time.unwrap_or(0.0);
let chunk_id = format!("cut_{}", scene_num);
child_chunks.push(StoryChildChunk {
chunk_id,
chunk_type: "cut".to_string(),
source: "cut".to_string(),
start_time,
end_time,
text_content: Some(format!("Scene {}", scene_num)),
content: serde_json::json!({
"scene_number": scene_num,
"start_time": start_time,
"end_time": end_time,
}),
child_chunk_ids: vec![],
parent_chunk_id: None,
});
}
let asr_child_ids: Vec<String> = child_chunks
.iter()
.filter(|c| c.source == "asr")
.map(|c| c.chunk_id.clone())
.collect();
let cut_child_ids: Vec<String> = child_chunks
.iter()
.filter(|c| c.source == "cut")
.map(|c| c.chunk_id.clone())
.collect();
// ── Build parent chunks from ASR ──────────────────────────────
let mut parent_chunks: Vec<StoryParentChunk> = Vec::new();
for (i, batch) in asr_child_ids.chunks(parent_chunk_size).enumerate() {
if batch.is_empty() {
continue;
}
let mut texts: Vec<String> = Vec::new();
let mut times: Vec<(f64, f64)> = Vec::new();
for child_id in batch {
if let Some(child) = child_chunks.iter().find(|c| &c.chunk_id == child_id) {
if let Some(ref t) = child.text_content {
texts.push(t.clone());
}
times.push((child.start_time, child.end_time));
}
}
let start_time = times.first().map(|t| t.0).unwrap_or(0.0);
let end_time = times.last().map(|t| t.1).unwrap_or(0.0);
let narrative = generate_narrative(&texts, &[], start_time, end_time);
let chunk_id = format!("story_asr_{:04}", i);
parent_chunks.push(StoryParentChunk {
chunk_id: chunk_id.clone(),
chunk_type: "story".to_string(),
source: "story_asr".to_string(),
start_time,
end_time,
text_content: narrative.clone(),
content: serde_json::json!({
"description": narrative,
"child_count": batch.len(),
"speech_preview": texts.iter().take(3).cloned().collect::<Vec<_>>().join(" "),
}),
child_chunk_ids: batch.to_vec(),
parent_chunk_id: None,
});
// Link children to parent
for child in &mut child_chunks {
if batch.contains(&child.chunk_id) {
child.parent_chunk_id = Some(chunk_id.clone());
}
}
}
// ── Build parent chunks from CUT ──────────────────────────────
for (i, batch) in cut_child_ids.chunks(parent_chunk_size).enumerate() {
if batch.is_empty() {
continue;
}
let mut times: Vec<(f64, f64)> = Vec::new();
for child_id in batch {
if let Some(child) = child_chunks.iter().find(|c| &c.chunk_id == child_id) {
times.push((child.start_time, child.end_time));
}
}
let start_time = times.first().map(|t| t.0).unwrap_or(0.0);
let end_time = times.last().map(|t| t.1).unwrap_or(0.0);
let narrative = generate_scene_narrative(&[], start_time, end_time, batch.len());
let chunk_id = format!("story_cut_{:04}", i);
parent_chunks.push(StoryParentChunk {
chunk_id: chunk_id.clone(),
chunk_type: "story".to_string(),
source: "story_cut".to_string(),
start_time,
end_time,
text_content: narrative.clone(),
content: serde_json::json!({
"description": narrative,
"child_count": batch.len(),
"scenes": batch,
}),
child_chunk_ids: batch.to_vec(),
parent_chunk_id: None,
});
for child in &mut child_chunks {
if batch.contains(&child.chunk_id) {
child.parent_chunk_id = Some(chunk_id.clone());
}
}
}
// ── Build result ──────────────────────────────────────────────
let total_child = asr_child_ids.len() + cut_child_ids.len();
let total_parent = parent_chunks.len();
let asr_count = asr_child_ids.len();
let cut_count = cut_child_ids.len();
let result = StoryResult {
child_chunks,
parent_chunks,
stats: StoryStats {
total_child_chunks: total_child,
total_parent_chunks: total_parent,
asr_children: asr_count,
cut_children: cut_count,
},
metadata: serde_json::json!({}),
parent_chunk_size,
};
// Write output (for compatibility with Python path)
let json_str = serde_json::to_string_pretty(&result)?;
std::fs::write(output_path, &json_str)
.with_context(|| format!("Failed to write {:?}", output_path))?;
Ok(result)
}
// ── Narrative generation (matching Python logic) ──────────────────
fn generate_narrative(texts: &[String], objects: &[String], start: f64, end: f64) -> String {
if texts.is_empty() && objects.is_empty() {
return format!("Video segment from {:.1}s to {:.1}s", start, end);
}
let mut parts: Vec<String> = Vec::new();
if !texts.is_empty() {
let combined = texts.join(" ");
let truncated = if combined.len() > 150 {
format!("{}...", &combined[..150])
} else {
combined
};
parts.push(format!("Speech: {}", truncated));
}
if !objects.is_empty() {
let mut unique: Vec<&String> = objects.iter().collect();
unique.sort();
unique.dedup();
let objs = unique
.iter()
.take(5)
.map(|s| (*s).as_str())
.collect::<Vec<_>>()
.join(", ");
parts.push(format!("Visuals: {}", objs));
}
format!("[{:.0}s-{:.0}s] {}", start, end, parts.join(" | "))
}
fn generate_scene_narrative(
objects: &[String],
start: f64,
end: f64,
scene_count: usize,
) -> String {
let mut unique: Vec<&String> = objects.iter().collect();
unique.sort();
unique.dedup();
let top5: Vec<&String> = unique.iter().take(5).cloned().collect();
if !top5.is_empty() {
let obj_str = top5
.iter()
.map(|s| s.as_str())
.collect::<Vec<_>>()
.join(", ");
format!(
"[{:.0}s-{:.0}s] {} scenes. Visuals: {}.",
start, end, scene_count, obj_str
)
} else {
format!("[{:.0}s-{:.0}s] {} video scenes.", start, end, scene_count)
}
}
// ── Tests ─────────────────────────────────────────────────────────
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_generate_narrative_with_text() {
let text = generate_narrative(
&["Hello world".to_string()],
&["person".to_string()],
0.0,
5.0,
);
assert!(text.contains("[0s-5s]"));
assert!(text.contains("Speech:"));
assert!(text.contains("Visuals:"));
}
#[test]
fn test_generate_narrative_empty() {
let text = generate_narrative(&[], &[], 10.0, 20.0);
assert!(text.contains("10.0s to 20.0s"));
}
#[test]
fn test_generate_scene_narrative() {
let text = generate_scene_narrative(&["person".to_string()], 0.0, 10.0, 3);
assert!(text.contains("3 scenes"));
assert!(text.contains("person"));
}
#[test]
fn test_generate_scene_narrative_empty() {
let text = generate_scene_narrative(&[], 0.0, 10.0, 1);
assert!(text.contains("1 video scenes"));
}
#[test]
fn test_narrative_truncation() {
let long_text = "a".repeat(200);
let text = generate_narrative(&[long_text], &[], 0.0, 5.0);
assert!(text.len() < 200 + 50); // truncated with "..."
assert!(text.ends_with("..."));
}
#[test]
fn test_story_result_serialization() {
let result = StoryResult {
child_chunks: vec![StoryChildChunk {
chunk_id: "asr_0001".to_string(),
chunk_type: "sentence".to_string(),
source: "asr".to_string(),
start_time: 0.0,
end_time: 5.0,
text_content: Some("Hello world".to_string()),
content: serde_json::json!({}),
child_chunk_ids: vec![],
parent_chunk_id: Some("story_asr_0000".to_string()),
}],
parent_chunks: vec![StoryParentChunk {
chunk_id: "story_asr_0000".to_string(),
chunk_type: "story".to_string(),
source: "story_asr".to_string(),
start_time: 0.0,
end_time: 25.0,
text_content: "[0s-25s] Hello world...".to_string(),
content: serde_json::json!({
"description": "[0s-25s] Hello world...",
"child_count": 5
}),
child_chunk_ids: vec!["asr_0001".to_string()],
parent_chunk_id: None,
}],
stats: StoryStats {
total_child_chunks: 10,
total_parent_chunks: 2,
asr_children: 10,
cut_children: 0,
},
metadata: serde_json::json!({}),
parent_chunk_size: 5,
};
let json = serde_json::to_string(&result).unwrap();
assert!(json.contains("asr_0001"));
assert!(json.contains("story_asr_0000"));
assert!(json.contains("Hello world"));
}
#[test]
fn test_story_result_deserialization() {
let json = r#"{
"child_chunks": [{
"chunk_id": "asr_0001",
"chunk_type": "sentence",
"source": "asr",
"start_time": 0.0,
"end_time": 5.0,
"text_content": "Hello",
"content": {},
"child_chunk_ids": [],
"parent_chunk_id": null
}],
"parent_chunks": [{
"chunk_id": "story_asr_0000",
"chunk_type": "story",
"source": "story_asr",
"start_time": 0.0,
"end_time": 5.0,
"text_content": "Hello segment",
"content": {"description": "Hello segment"},
"child_chunk_ids": ["asr_0001"],
"parent_chunk_id": null
}],
"stats": {
"total_child_chunks": 1,
"total_parent_chunks": 1,
"asr_children": 1,
"cut_children": 0
},
"metadata": {},
"parent_chunk_size": 5
}"#;
let result: StoryResult = serde_json::from_str(json).unwrap();
assert_eq!(result.child_chunks.len(), 1);
assert_eq!(result.parent_chunks.len(), 1);
assert_eq!(result.stats.total_child_chunks, 1);
}
#[test]
fn test_parent_child_relationship() {
let result = StoryResult {
child_chunks: vec![
StoryChildChunk {
chunk_id: "asr_0001".to_string(),
chunk_type: "sentence".to_string(),
source: "asr".to_string(),
start_time: 0.0,
end_time: 5.0,
text_content: Some("First".to_string()),
content: serde_json::json!({}),
child_chunk_ids: vec![],
parent_chunk_id: Some("story_asr_0000".to_string()),
},
StoryChildChunk {
chunk_id: "asr_0002".to_string(),
chunk_type: "sentence".to_string(),
source: "asr".to_string(),
start_time: 5.0,
end_time: 10.0,
text_content: Some("Second".to_string()),
content: serde_json::json!({}),
child_chunk_ids: vec![],
parent_chunk_id: Some("story_asr_0000".to_string()),
},
],
parent_chunks: vec![StoryParentChunk {
chunk_id: "story_asr_0000".to_string(),
chunk_type: "story".to_string(),
source: "story_asr".to_string(),
start_time: 0.0,
end_time: 10.0,
text_content: "Combined narrative".to_string(),
content: serde_json::json!({}),
child_chunk_ids: vec!["asr_0001".to_string(), "asr_0002".to_string()],
parent_chunk_id: None,
}],
stats: StoryStats {
total_child_chunks: 2,
total_parent_chunks: 1,
asr_children: 2,
cut_children: 0,
},
metadata: serde_json::json!({}),
parent_chunk_size: 5,
};
assert_eq!(result.parent_chunks[0].child_chunk_ids.len(), 2);
assert!(result
.child_chunks
.iter()
.all(|c| c.parent_chunk_id.is_some()));
assert!(result.parent_chunks[0].parent_chunk_id.is_none());
}
#[test]
fn test_native_story_empty_data() {
// Write empty ASR and CUT files, then test try_native_story
let dir = std::env::temp_dir().join("story_test_empty");
let _ = std::fs::create_dir_all(&dir);
let basename = "test_video";
let asr_path = dir.join(format!("{}.asr.json", basename));
let cut_path = dir.join(format!("{}.cut.json", basename));
let out_path = dir.join(format!("{}.story.json", basename));
std::fs::write(&asr_path, r#"{"segments":[]}"#).unwrap();
std::fs::write(&cut_path, r#"{"scenes":[]}"#).unwrap();
let result = try_native_story("/dummy.mp4", out_path.to_str().unwrap(), None).unwrap();
assert_eq!(result.stats.total_child_chunks, 0);
assert_eq!(result.stats.total_parent_chunks, 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn test_native_story_with_data() {
let dir = std::env::temp_dir().join("story_test_data");
let _ = std::fs::create_dir_all(&dir);
let basename = "test_video";
let asr_path = dir.join(format!("{}.asr.json", basename));
let cut_path = dir.join(format!("{}.cut.json", basename));
let out_path = dir.join(format!("{}.story.json", basename));
std::fs::write(
&asr_path,
r#"{
"segments": [
{"start": 0.0, "end": 2.5, "text": "Hello", "confidence": 0.95},
{"start": 2.5, "end": 5.0, "text": "World", "confidence": 0.92},
{"start": 5.0, "end": 7.5, "text": "Foo", "confidence": 0.90}
]
}"#,
)
.unwrap();
std::fs::write(&cut_path, r#"{
"scenes": [
{"scene_number": 1, "start_frame": 0, "end_frame": 150, "start_time": 0.0, "end_time": 5.0},
{"scene_number": 2, "start_frame": 150, "end_frame": 300, "start_time": 5.0, "end_time": 10.0}
]
}"#).unwrap();
let result = try_native_story("/dummy.mp4", out_path.to_str().unwrap(), None).unwrap();
assert_eq!(result.stats.asr_children, 3);
assert_eq!(result.stats.cut_children, 2);
assert_eq!(result.stats.total_child_chunks, 5);
// 3 ASR segments, parent_chunk_size=5 → 1 parent
// 2 CUT scenes, parent_chunk_size=5 → 1 parent
assert_eq!(result.stats.total_parent_chunks, 2);
// Verify child-parent linking
for child in &result.child_chunks {
if child.source == "asr" {
assert!(child.parent_chunk_id.is_some());
assert!(child
.parent_chunk_id
.as_ref()
.unwrap()
.starts_with("story_asr_"));
}
}
// Verify output file was written
assert!(out_path.exists());
let content = std::fs::read_to_string(&out_path).unwrap();
assert!(content.contains("Hello"));
assert!(content.contains("World"));
let _ = std::fs::remove_dir_all(&dir);
}
}
src/core/processor/tkg.rs
+5 -1113
View File
File diff suppressed because it is too large Load Diff