feat: add cluster-agent endpoint, fix OCR labeling, fix ingestion blocking
- Add POST /api/v1/file/:file_uuid/cluster-agent endpoint for on-demand face clustering
- Fix OCR chunks being labeled as ASRX: use ChunkType.as_str() instead of {:?}
- Rule 1 now deletes old chunks before re-inserting to avoid stale data
- Add fallback face_traced.json when store_traced_faces.py fails
- ingestion_complete now handles status='error' to unblock jobs
- VLM describe tool now uses trace_id with pre-extracted face crops
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
use base64::{engine::general_purpose::STANDARD as BASE64, Engine};
|
||||
use serde_json;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::core::db::qdrant_db::QdrantDb;
|
||||
use crate::core::db::schema;
|
||||
@@ -1233,3 +1234,115 @@ pub async fn exec_search_by_appearance(
|
||||
Err("Color search output not found".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
fn face_crop_path(file_uuid: &str, trace_id: i32) -> Option<std::path::PathBuf> {
|
||||
let base = std::env::var("MOMENTRY_OUTPUT_DIR")
|
||||
.unwrap_or_else(|_| "/Users/accusys/momentry/output".to_string());
|
||||
let dir = std::path::PathBuf::from(base)
|
||||
.join(".faces")
|
||||
.join(file_uuid)
|
||||
.join(trace_id.to_string());
|
||||
if !dir.exists() {
|
||||
return None;
|
||||
}
|
||||
let mut entries: Vec<_> = match std::fs::read_dir(&dir) {
|
||||
Ok(e) => e.filter_map(|e| e.ok()).collect(),
|
||||
Err(_) => return None,
|
||||
};
|
||||
entries.sort_by_key(|e| e.file_name());
|
||||
entries.first().map(|e| e.path())
|
||||
}
|
||||
|
||||
pub async fn exec_vlm_describe(
|
||||
pool: &sqlx::PgPool,
|
||||
args: &serde_json::Value,
|
||||
) -> Result<String, String> {
|
||||
let file_uuid = args.get("file_uuid").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let trace_id = args.get("trace_id").and_then(|v| v.as_i64()).unwrap_or(0) as i32;
|
||||
let prompt = args
|
||||
.get("prompt")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("Describe this person's clothing and appearance. Focus on colors, clothing type, and any distinctive visual features.");
|
||||
|
||||
if file_uuid.is_empty() {
|
||||
return Ok(serde_json::json!({"error": "file_uuid is required"}).to_string());
|
||||
}
|
||||
if trace_id <= 0 {
|
||||
return Ok(serde_json::json!({"error": "trace_id is required and must be > 0"}).to_string());
|
||||
}
|
||||
|
||||
let crop_path = face_crop_path(file_uuid, trace_id)
|
||||
.ok_or_else(|| format!("No face crop found for {} trace {}", file_uuid, trace_id))?;
|
||||
|
||||
let jpeg_bytes = std::fs::read(&crop_path)
|
||||
.map_err(|e| format!("Failed to read face crop: {}", e))?;
|
||||
|
||||
let videos = schema::table_name("videos");
|
||||
let fps: f64 = sqlx::query_scalar(&format!(
|
||||
"SELECT COALESCE(fps, 25.0) FROM {} WHERE file_uuid = $1",
|
||||
videos
|
||||
))
|
||||
.bind(file_uuid)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.unwrap_or(25.0);
|
||||
|
||||
let frame_name = crop_path
|
||||
.file_stem()
|
||||
.and_then(|s| s.to_str())
|
||||
.and_then(|s| s.parse::<i64>().ok())
|
||||
.unwrap_or(0);
|
||||
let timestamp_secs = frame_name as f64 / fps;
|
||||
|
||||
let base64_img = BASE64.encode(&jpeg_bytes);
|
||||
|
||||
let ollama_url = std::env::var("OLLAMA_URL")
|
||||
.unwrap_or_else(|_| "http://localhost:11434".to_string());
|
||||
let model = std::env::var("VLM_MODEL").unwrap_or_else(|_| "llava".to_string());
|
||||
|
||||
let body = serde_json::json!({
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"images": [base64_img],
|
||||
"stream": false,
|
||||
"options": {
|
||||
"num_predict": 80
|
||||
}
|
||||
});
|
||||
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(30))
|
||||
.build()
|
||||
.map_err(|e| format!("Failed to create HTTP client: {}", e))?;
|
||||
|
||||
let resp = client
|
||||
.post(format!("{}/api/generate", ollama_url))
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("Ollama request failed: {}", e))?;
|
||||
|
||||
let resp_json: serde_json::Value = resp
|
||||
.json()
|
||||
.await
|
||||
.map_err(|e| format!("Failed to parse Ollama response: {}", e))?;
|
||||
|
||||
let description = resp_json
|
||||
.get("response")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("No description returned")
|
||||
.to_string();
|
||||
|
||||
Ok(serde_json::json!({
|
||||
"tool": "vlm_describe",
|
||||
"result": {
|
||||
"file_uuid": file_uuid,
|
||||
"trace_id": trace_id,
|
||||
"frame": frame_name,
|
||||
"description": description,
|
||||
"time_sec": (timestamp_secs * 100.0).round() / 100.0
|
||||
}
|
||||
})
|
||||
.to_string())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user