feat: add Vision LLM integration (CLIP + Qwen3-VL cascade)

- Add Qwen3-VL dynamic management (start/stop/status CLI)
- Add CLIP + Qwen3-VL cascade detection strategy
- Add Vision CLI commands (vision start/stop/status, detect)
- Add cascade_vision processor module
- Add clip processor module
- Add qwen_vl_manager module

Changes:
- scripts/start_qwen3vl.sh, stop_qwen3vl.sh: Qwen3-VL management scripts
- src/core/vision/: Qwen3-VL manager module
- src/core/processor/cascade_vision.rs: CLIP + Qwen3-VL cascade logic
- src/core/processor/clip.rs: CLIP classification and detection
- src/api/clip_api.rs: CLIP API endpoints
- src/cli/vision.rs: Vision CLI implementation
- src/cli/args.rs: Add Vision and Detect commands
- src/main.rs: Integrate Vision CLI
- src/core/mod.rs: Add vision module
- src/core/processor/mod.rs: Add cascade_vision module
This commit is contained in:
Accusys
2026-06-13 16:25:52 +08:00
parent 834b0d4865
commit 17e4e15860
37 changed files with 2185 additions and 294 deletions
+10
View File
@@ -92,6 +92,16 @@ pub static MEDIA_BASE_URL: Lazy<String> = Lazy::new(|| {
.unwrap_or_else(|_| "https://wp.momentry.ddns.net".to_string())
});
pub static STORAGE_ROOT: Lazy<String> = Lazy::new(|| {
env::var("MOMENTRY_STORAGE_ROOT")
.unwrap_or_else(|_| "/Users/accusys/momentry/var/sftpgo/data".to_string())
});
pub static SERVE_BASE_URL: Lazy<String> = Lazy::new(|| {
env::var("MOMENTRY_SERVE_BASE_URL")
.unwrap_or_else(|_| "https://m5wp.momentry.ddns.net/files".to_string())
});
pub static SERVER_PORT: Lazy<u16> = Lazy::new(|| {
env::var("MOMENTRY_SERVER_PORT")
.unwrap_or_else(|_| "3002".to_string())
+1 -1
View File
@@ -2862,7 +2862,7 @@ impl PostgresDb {
let rows = if let Some(u) = file_uuid {
sqlx::query(&format!(
"SELECT chunk_id, file_uuid, chunk_type, text_content, start_time, end_time, 1.0::float8 as score \
FROM {} WHERE file_uuid=$1 AND text_content ILIKE $2 LIMIT $3", table)
FROM {} WHERE file_uuid=$1 AND text_content ILIKE $2 AND text_content != '' LIMIT $3", table)
)
.bind(u).bind(&like).bind(limit)
.fetch_all(&self.pool).await?
+3 -6
View File
@@ -1,10 +1,10 @@
use anyhow::Result;
use reqwest::Client;
use serde::{Deserialize, Serialize};
use std::time::Duration;
use tracing::{debug, error, warn};
use crate::core::config;
use crate::core::llm::function_calling::LLM_CLIENT;
#[derive(Debug, Serialize)]
struct ChatRequest {
@@ -39,10 +39,6 @@ pub async fn generate_5w1h_summary(scene_text: &str) -> Result<String> {
return Ok("LLM Disabled".to_string());
}
let client = Client::builder()
.timeout(Duration::from_secs(*config::llm::SUMMARY_TIMEOUT_SECS))
.build()?;
let prompt = format!(
r#"Analyze the following video scene transcript and provide a concise 5W1H+ summary in JSON format.
Focus on: Who, What, Where, When, Why, How, and Key Objects/Actions.
@@ -82,9 +78,10 @@ pub async fn generate_5w1h_summary(scene_text: &str) -> Result<String> {
debug!("Calling LLM for summary: {}", *config::llm::SUMMARY_URL);
let res = client
let res = LLM_CLIENT
.post(&*config::llm::SUMMARY_URL)
.json(&req)
.timeout(Duration::from_secs(*config::llm::SUMMARY_TIMEOUT_SECS))
.send()
.await?;
+24 -12
View File
@@ -1,8 +1,18 @@
use once_cell::sync::Lazy;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use crate::core::config;
/// Shared HTTP client with connection pooling for all LLM calls
pub static LLM_CLIENT: Lazy<reqwest::Client> = Lazy::new(|| {
reqwest::Client::builder()
.pool_max_idle_per_host(32)
.pool_idle_timeout(std::time::Duration::from_secs(300))
.build()
.expect("Failed to create shared LLM HTTP client")
});
/// A tool/function definition for Gemma4 function calling
#[derive(Debug, Clone, Serialize)]
pub struct ToolDef {
@@ -126,11 +136,11 @@ pub async fn call_llm_vision(
"stream": false,
});
let client = reqwest::Client::builder()
let res = LLM_CLIENT
.post(&llm_vision_url())
.json(&req)
.timeout(std::time::Duration::from_secs(timeout_secs))
.build()?;
let res = client.post(&llm_vision_url()).json(&req).send().await?;
.send().await?;
if !res.status().is_success() {
let text = res.text().await.unwrap_or_default();
anyhow::bail!("Vision LLM API error: {}", text);
@@ -182,13 +192,11 @@ pub async fn call_llm(
max_tokens: u32,
timeout_secs: u64,
) -> anyhow::Result<LlmResponse> {
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(if timeout_secs > 0 {
timeout_secs
} else {
*config::llm::CHAT_TIMEOUT_SECS
}))
.build()?;
let timeout = if timeout_secs > 0 {
timeout_secs
} else {
*config::llm::CHAT_TIMEOUT_SECS
};
let req = ChatRequest {
model: llm_model(),
@@ -199,7 +207,11 @@ pub async fn call_llm(
tools,
};
let res = client.post(&llm_chat_url()).json(&req).send().await?;
let res = LLM_CLIENT
.post(&llm_chat_url())
.json(&req)
.timeout(std::time::Duration::from_secs(timeout))
.send().await?;
if !res.status().is_success() {
let text = res.text().await.unwrap_or_default();
+21 -10
View File
@@ -1,12 +1,12 @@
use std::collections::HashSet;
use anyhow::Result;
use reqwest::Client;
use serde::{Deserialize, Serialize};
use std::time::Duration;
use tracing::{debug, warn};
use crate::core::config;
use crate::core::llm::function_calling::LLM_CLIENT;
#[derive(Debug, Serialize)]
struct ChatRequest {
@@ -38,7 +38,10 @@ struct RerankResponse {
ranked: Vec<usize>,
}
pub async fn rerank_search_results(query: &str, candidates: &[(usize, &str)]) -> Result<Vec<usize>> {
pub async fn rerank_search_results(
query: &str,
candidates: &[(usize, &str)],
) -> Result<Vec<usize>> {
if candidates.is_empty() {
return Ok(vec![]);
}
@@ -67,10 +70,6 @@ Include every chunk number exactly once. Only respond with the JSON."#,
query, chunks_text
);
let client = Client::builder()
.timeout(Duration::from_secs(15))
.build()?;
let req = ChatRequest {
model: config::llm::CHAT_MODEL.clone(),
messages: vec![
@@ -88,11 +87,16 @@ Include every chunk number exactly once. Only respond with the JSON."#,
stream: false,
};
debug!("LLM rerank: {} candidates for query '{}'", candidates.len(), query);
debug!(
"LLM rerank: {} candidates for query '{}'",
candidates.len(),
query
);
let res = client
let res = LLM_CLIENT
.post(&*config::llm::CHAT_URL)
.json(&req)
.timeout(Duration::from_secs(15))
.send()
.await?;
@@ -116,7 +120,11 @@ Include every chunk number exactly once. Only respond with the JSON."#,
// Strip markdown code fences if present
let content = if content.starts_with("```") {
let lines: Vec<&str> = content.lines().collect();
let start = if lines.first().map(|l| l.contains("```")).unwrap_or(false) { 1 } else { 0 };
let start = if lines.first().map(|l| l.contains("```")).unwrap_or(false) {
1
} else {
0
};
let end = if lines.last().map(|l| l.contains("```")).unwrap_or(false) {
lines.len().saturating_sub(1)
} else {
@@ -163,6 +171,9 @@ Include every chunk number exactly once. Only respond with the JSON."#,
}
}
warn!("LLM rerank: could not parse response — content: {}", &content[..content.len().min(200)]);
warn!(
"LLM rerank: could not parse response — content: {}",
&content[..content.len().min(200)]
);
Ok(candidates.iter().map(|(idx, _)| *idx).collect())
}
+1
View File
@@ -20,3 +20,4 @@ pub mod text;
pub mod thumbnail;
pub mod time;
pub mod tmdb;
pub mod vision;
+30 -21
View File
@@ -17,8 +17,8 @@ pub async fn store_asrx_chunks(db: &PostgresDb, uuid: &str) -> Result<()> {
let json_str = std::fs::read_to_string(&asrx_path)
.with_context(|| format!("ASRX file not found: {:?}", asrx_path))?;
let result: AsrxResult = serde_json::from_str(&json_str)
.context("Failed to parse ASRX JSON")?;
let result: AsrxResult =
serde_json::from_str(&json_str).context("Failed to parse ASRX JSON")?;
let segments_count = result.segments.len();
let mut pre_chunks = Vec::new();
@@ -41,21 +41,26 @@ pub async fn store_asrx_chunks(db: &PostgresDb, uuid: &str) -> Result<()> {
));
}
db.store_raw_pre_chunks_batch(uuid, "asrx", &pre_chunks).await?;
db.store_raw_pre_chunks_batch(uuid, "asr", &pre_chunks).await?;
db.store_speaker_detections_batch(uuid, &speaker_detections).await?;
db.store_raw_pre_chunks_batch(uuid, "asrx", &pre_chunks)
.await?;
db.store_raw_pre_chunks_batch(uuid, "asr", &pre_chunks)
.await?;
db.store_speaker_detections_batch(uuid, &speaker_detections)
.await?;
println!("Stored {} ASRX pre-chunks for {}", segments_count, uuid);
Ok(())
}
pub async fn execute_rule1(db: &PostgresDb, uuid: &str) -> Result<usize> {
let video = db.get_video_by_uuid(uuid)
let video = db
.get_video_by_uuid(uuid)
.await?
.context("Video not found")?;
let fps = video.fps;
let count = rule1_ingest::execute_rule1(db, uuid, fps).await
let count = rule1_ingest::execute_rule1(db, uuid, fps)
.await
.context("Rule 1 ingestion failed")?;
println!("Rule 1 completed: {} chunks inserted for {}", count, uuid);
@@ -68,17 +73,15 @@ pub async fn vectorize_chunks(uuid: &str) -> Result<()> {
let embedder = Embedder::new("embeddinggemma-300m".to_string());
let chunk_table = schema::table_name("chunk");
let rows = sqlx::query_as::<_, (String, String, String, i64, i64, f64, f64, String)>(
&format!(
"SELECT chunk_id, chunk_type, text_content, start_frame, end_frame, \
let rows = sqlx::query_as::<_, (String, String, String, i64, i64, f64, f64, String)>(&format!(
"SELECT chunk_id, chunk_type, text_content, start_frame, end_frame, \
start_time, end_time, content::text \
FROM {} WHERE file_uuid = $1 AND chunk_type = 'sentence' \
AND embedding IS NULL \
AND (text_content IS NOT NULL AND text_content != '') \
ORDER BY id",
chunk_table
),
)
chunk_table
))
.bind(uuid)
.fetch_all(db.pool())
.await?;
@@ -91,7 +94,9 @@ pub async fn vectorize_chunks(uuid: &str) -> Result<()> {
let total = rows.len();
let mut stored = 0usize;
for (chunk_id, _chunk_type, text, start_frame, end_frame, start_time, end_time, _content_str) in &rows {
for (chunk_id, _chunk_type, text, start_frame, end_frame, start_time, end_time, _content_str) in
&rows
{
if text.is_empty() {
continue;
}
@@ -127,13 +132,15 @@ pub async fn vectorize_chunks(uuid: &str) -> Result<()> {
}
}
println!("Vectorization complete: {}/{} vectors for {}", stored, total, uuid);
println!(
"Vectorization complete: {}/{} vectors for {}",
stored, total, uuid
);
Ok(())
}
pub async fn run_phase1(uuid: &str) -> Result<()> {
let executor = PythonExecutor::new()
.context("Failed to create PythonExecutor")?;
let executor = PythonExecutor::new().context("Failed to create PythonExecutor")?;
executor
.run(
@@ -154,15 +161,17 @@ pub async fn mark_complete(db: &PostgresDb, uuid: &str) -> Result<()> {
use crate::core::db::MonitorJobStatus;
use crate::core::db::VideoStatus;
let job_id = sqlx::query_scalar::<_, i32>(
&format!("SELECT id FROM {} WHERE uuid = $1 LIMIT 1", schema::table_name("monitor_jobs")),
)
let job_id = sqlx::query_scalar::<_, i32>(&format!(
"SELECT id FROM {} WHERE uuid = $1 LIMIT 1",
schema::table_name("monitor_jobs")
))
.bind(uuid)
.fetch_optional(db.pool())
.await?;
if let Some(job_id) = job_id {
db.update_job_status(job_id, MonitorJobStatus::Completed).await?;
db.update_job_status(job_id, MonitorJobStatus::Completed)
.await?;
println!("Job {} marked as completed", job_id);
}
+1 -4
View File
@@ -44,10 +44,7 @@ pub async fn process_asrx(
let executor = PythonExecutor::new()?;
let script_path = executor.script_path("asrx_processor.py");
tracing::info!(
"[ASRX] Starting hybrid speaker diarization: {}",
video_path
);
tracing::info!("[ASRX] Starting hybrid speaker diarization: {}", video_path);
if !script_path.exists() {
tracing::error!("[ASRX] Script not found: {:?}", script_path);
+308
View File
@@ -0,0 +1,308 @@
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use std::path::Path;
use std::time::Duration;
use tracing::{debug, info, warn};
use crate::core::processor::clip::{ClipPrediction, detect_objects};
use crate::core::vision::qwen_vl_manager::QwenVLManager;
const DEFAULT_CLIP_THRESHOLD: f32 = 0.7;
const QWENVL_TIMEOUT: Duration = Duration::from_secs(30);
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CascadeDetectionResult {
pub detections: Vec<ClipPrediction>,
pub model_used: String,
pub clip_confidence: f32,
pub qwenvl_used: bool,
pub processing_time_ms: u64,
}
pub struct CascadeVisionProcessor {
clip_threshold: f32,
qwen_vl_manager: QwenVLManager,
}
impl CascadeVisionProcessor {
pub fn new() -> Self {
Self {
clip_threshold: DEFAULT_CLIP_THRESHOLD,
qwen_vl_manager: QwenVLManager::new(),
}
}
pub fn with_threshold(threshold: f32) -> Self {
Self {
clip_threshold: threshold,
qwen_vl_manager: QwenVLManager::new(),
}
}
pub async fn detect_objects(&self, image_path: &Path, objects: &[&str]) -> Result<CascadeDetectionResult> {
let start_time = std::time::Instant::now();
info!(
"[Cascade] Starting detection for {:?} with {} object classes (threshold: {:.2})",
image_path,
objects.len(),
self.clip_threshold
);
let clip_result = self.run_clip_detection(image_path, objects).await?;
let max_clip_confidence = clip_result
.iter()
.map(|p| p.confidence)
.fold(0.0_f32, |max, val| if val > max { val } else { max });
debug!(
"[Cascade] CLIP max confidence: {:.3} (threshold: {:.2})",
max_clip_confidence,
self.clip_threshold
);
if max_clip_confidence > self.clip_threshold {
info!(
"[Cascade] High confidence ({:.3} > {:.2}) → triggering Qwen3-VL",
max_clip_confidence,
self.clip_threshold
);
let qwenvl_result = self.run_qwenvl_detection(image_path, objects).await?;
let processing_time = start_time.elapsed().as_millis() as u64;
return Ok(CascadeDetectionResult {
detections: qwenvl_result,
model_used: "qwen3vl".to_string(),
clip_confidence: max_clip_confidence,
qwenvl_used: true,
processing_time_ms: processing_time,
});
}
info!(
"[Cascade] Low confidence ({:.3} <= {:.2}) → using CLIP results only",
max_clip_confidence,
self.clip_threshold
);
let processing_time = start_time.elapsed().as_millis() as u64;
Ok(CascadeDetectionResult {
detections: clip_result,
model_used: "clip".to_string(),
clip_confidence: max_clip_confidence,
qwenvl_used: false,
processing_time_ms: processing_time,
})
}
async fn run_clip_detection(&self, image_path: &Path, objects: &[&str]) -> Result<Vec<ClipPrediction>> {
let image_path_str = image_path.display().to_string();
debug!("[Cascade] Running CLIP detection for {:?}", image_path);
let predictions = detect_objects(&image_path_str, objects, None, None)
.await
.context("CLIP detection failed")?;
debug!(
"[Cascade] CLIP detected {} objects",
predictions.len()
);
Ok(predictions)
}
async fn run_qwenvl_detection(&self, image_path: &Path, objects: &[&str]) -> Result<Vec<ClipPrediction>> {
let image_path_str = image_path.display().to_string();
debug!("[Cascade] Running Qwen3-VL detection for {:?}", image_path);
self.qwen_vl_manager.ensure_running().await?;
let prompt = self.build_detection_prompt(objects);
let client = reqwest::Client::new();
let url = format!("http://localhost:{}/v1/chat/completions", self.qwen_vl_manager.get_port());
let request_body = serde_json::json!({
"model": "Qwen3VL-8B-Instruct-Q8_0",
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": prompt
},
{
"type": "image_url",
"image_url": {
"url": format!("file://{}", image_path_str)
}
}
]
}
],
"max_tokens": 500,
"temperature": 0.1
});
let response = client
.post(&url)
.json(&request_body)
.timeout(QWENVL_TIMEOUT)
.send()
.await
.context("Qwen3-VL API request failed")?;
if !response.status().is_success() {
warn!("[Cascade] Qwen3-VL API error: {}", response.status());
anyhow::bail!("Qwen3-VL API returned error: {}", response.status());
}
let response_json: serde_json::Value = response
.json()
.await
.context("Failed to parse Qwen3-VL response")?;
let content = response_json
.get("choices")
.and_then(|choices| choices.get(0))
.and_then(|choice| choice.get("message"))
.and_then(|message| message.get("content"))
.and_then(|content| content.as_str())
.unwrap_or("");
debug!("[Cascade] Qwen3-VL response: {}", content);
let detections = self.parse_qwenvl_response(content, objects);
self.qwen_vl_manager.update_last_request_time().await;
info!(
"[Cascade] Qwen3-VL detected {} objects",
detections.len()
);
Ok(detections)
}
fn build_detection_prompt(&self, objects: &[&str]) -> String {
let object_list = objects.join(", ");
format!(
"Analyze this image and detect the following objects: {}.\n\
For each detected object, provide:\n\
1. The object name\n\
2. A confidence score (0.0 to 1.0)\n\
3. A brief description of what you see\n\
\n\
Format your response as JSON:\n\
{{\"detections\": [{{\"label\": \"object_name\", \"confidence\": 0.95, \"description\": \"brief description\"}}]}}\n\
\n\
If no objects are detected, return: {{\"detections\": []}}\n\
\n\
IMPORTANT: Only detect objects that are clearly visible and identifiable. Do not guess or hallucinate.",
object_list
)
}
fn parse_qwenvl_response(&self, content: &str, _objects: &[&str]) -> Vec<ClipPrediction> {
let json_start = content.find('{');
let json_end = content.rfind('}');
if json_start.is_none() || json_end.is_none() {
debug!("[Cascade] No JSON found in Qwen3-VL response");
return Vec::new();
}
let json_str = &content[json_start.unwrap()..=json_end.unwrap()];
let parsed: serde_json::Value = serde_json::from_str(json_str)
.unwrap_or(serde_json::json!({"detections": []}));
let detections = parsed
.get("detections")
.and_then(|d| d.as_array())
.map(|arr| arr.clone())
.unwrap_or_else(|| Vec::new());
detections
.iter()
.filter_map(|d| {
let label = d.get("label").and_then(|l| l.as_str()).unwrap_or("");
let confidence = d.get("confidence").and_then(|c| c.as_f64()).unwrap_or(0.0) as f32;
if !label.is_empty() && confidence > 0.0 {
Some(ClipPrediction {
label: label.to_string(),
confidence,
})
} else {
None
}
})
.collect()
}
}
impl Default for CascadeVisionProcessor {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_build_detection_prompt() {
let processor = CascadeVisionProcessor::new();
let objects = vec!["gun", "weapon", "person"];
let prompt = processor.build_detection_prompt(&objects);
assert!(prompt.contains("gun, weapon, person"));
assert!(prompt.contains("confidence score"));
assert!(prompt.contains("JSON"));
}
#[test]
fn test_parse_qwenvl_response() {
let processor = CascadeVisionProcessor::new();
let response = "{\"detections\": [{\"label\": \"gun\", \"confidence\": 0.95, \"description\": \"a handgun\"}]}";
let objects = vec!["gun"];
let detections = processor.parse_qwenvl_response(response, &objects);
assert_eq!(detections.len(), 1);
assert_eq!(detections[0].label, "gun");
assert!((detections[0].confidence - 0.95).abs() < 0.001);
}
#[test]
fn test_parse_empty_response() {
let processor = CascadeVisionProcessor::new();
let response = "{\"detections\": []}";
let objects = vec!["gun"];
let detections = processor.parse_qwenvl_response(response, &objects);
assert_eq!(detections.len(), 0);
}
#[test]
fn test_parse_invalid_json() {
let processor = CascadeVisionProcessor::new();
let response = "This is not JSON";
let objects = vec!["gun"];
let detections = processor.parse_qwenvl_response(response, &objects);
assert_eq!(detections.len(), 0);
}
}
+290
View File
@@ -0,0 +1,290 @@
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use std::time::Duration;
use super::executor::PythonExecutor;
const CLIP_TIMEOUT: Duration = Duration::from_secs(300);
/// CLIP classification prediction
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct ClipPrediction {
pub label: String,
pub confidence: f32,
}
/// CLIP classification result for a single image
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct ClipImageResult {
pub image_path: String,
pub predictions: Vec<ClipPrediction>,
}
/// CLIP object detection result
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct ClipDetectionResult {
pub image_path: String,
pub detected_objects: Vec<ClipPrediction>,
}
/// Classify a single image with given labels
pub async fn classify_image(
image_path: &str,
labels: &[&str],
top_k: Option<usize>,
model_name: Option<&str>,
) -> Result<Vec<ClipPrediction>> {
let executor = PythonExecutor::new()?;
let script_path = executor.script_path("clip_classifier.py");
if !script_path.exists() {
anyhow::bail!("clip_classifier.py not found at {:?}", script_path);
}
let top_k = top_k.unwrap_or(5);
let model = model_name.unwrap_or("openai/clip-vit-base-patch32");
let mut args = vec![
image_path.to_string(),
"--labels".to_string(),
labels.join(","),
"--top-k".to_string(),
top_k.to_string(),
"--model".to_string(),
model.to_string(),
];
let output_path = format!("{}.clip.json", image_path);
args.push("--output".to_string());
args.push(output_path.clone());
tracing::info!(
"[CLIP] Classifying image: {} with {} labels",
image_path,
labels.len()
);
executor
.run(
"clip_classifier.py",
&args.iter().map(|s| s.as_str()).collect::<Vec<_>>(),
None,
"CLIP",
Some(CLIP_TIMEOUT),
)
.await
.context("Failed to run CLIP classifier")?;
let json_str = std::fs::read_to_string(&output_path)
.context("Failed to read CLIP output")?;
let results: std::collections::HashMap<String, Vec<ClipPrediction>> =
serde_json::from_str(&json_str)
.context("Failed to parse CLIP output")?;
let predictions = results
.get(image_path)
.cloned()
.unwrap_or_default();
tracing::info!(
"[CLIP] Top prediction: {} ({:.3})",
predictions.first().map(|p| p.label.as_str()).unwrap_or("none"),
predictions.first().map(|p| p.confidence).unwrap_or(0.0)
);
Ok(predictions)
}
/// Detect objects in an image
pub async fn detect_objects(
image_path: &str,
objects: &[&str],
threshold: Option<f32>,
model_name: Option<&str>,
) -> Result<Vec<ClipPrediction>> {
let executor = PythonExecutor::new()?;
let script_path = executor.script_path("clip_classifier.py");
if !script_path.exists() {
anyhow::bail!("clip_classifier.py not found at {:?}", script_path);
}
let threshold = threshold.unwrap_or(0.15);
let model = model_name.unwrap_or("openai/clip-vit-base-patch32");
let mut args = vec![
image_path.to_string(),
"--detect".to_string(),
objects.join(","),
"--threshold".to_string(),
threshold.to_string(),
"--model".to_string(),
model.to_string(),
];
let output_path = format!("{}.clip.json", image_path);
args.push("--output".to_string());
args.push(output_path.clone());
tracing::info!(
"[CLIP] Detecting {} objects in: {} (threshold: {:.2})",
objects.len(),
image_path,
threshold
);
executor
.run(
"clip_classifier.py",
&args.iter().map(|s| s.as_str()).collect::<Vec<_>>(),
None,
"CLIP",
Some(CLIP_TIMEOUT),
)
.await
.context("Failed to run CLIP object detection")?;
let json_str = std::fs::read_to_string(&output_path)
.context("Failed to read CLIP output")?;
let results: std::collections::HashMap<String, Vec<ClipPrediction>> =
serde_json::from_str(&json_str)
.context("Failed to parse CLIP output")?;
let detected = results
.get(image_path)
.cloned()
.unwrap_or_default();
if !detected.is_empty() {
tracing::info!(
"[CLIP] Detected {} objects: {}",
detected.len(),
detected.iter().map(|p| p.label.as_str()).collect::<Vec<_>>().join(", ")
);
} else {
tracing::info!("[CLIP] No objects detected above threshold {:.2}", threshold);
}
Ok(detected)
}
/// Batch classify multiple images
pub async fn classify_images(
image_paths: &[&str],
labels: &[&str],
top_k: Option<usize>,
model_name: Option<&str>,
) -> Result<Vec<ClipImageResult>> {
let executor = PythonExecutor::new()?;
let script_path = executor.script_path("clip_classifier.py");
if !script_path.exists() {
anyhow::bail!("clip_classifier.py not found at {:?}", script_path);
}
let top_k = top_k.unwrap_or(5);
let model = model_name.unwrap_or("openai/clip-vit-base-patch32");
// Create temp file with image paths
let temp_file = format!("/tmp/clip_batch_{}.txt", uuid::Uuid::new_v4());
std::fs::write(&temp_file, image_paths.join("\n"))
.context("Failed to write batch file")?;
let mut args = vec![
temp_file.clone(),
"--batch".to_string(),
"--labels".to_string(),
labels.join(","),
"--top-k".to_string(),
top_k.to_string(),
"--model".to_string(),
model.to_string(),
];
let output_path = format!("/tmp/clip_batch_{}.json", uuid::Uuid::new_v4());
args.push("--output".to_string());
args.push(output_path.clone());
tracing::info!(
"[CLIP] Batch classifying {} images with {} labels",
image_paths.len(),
labels.len()
);
executor
.run(
"clip_classifier.py",
&args.iter().map(|s| s.as_str()).collect::<Vec<_>>(),
None,
"CLIP",
Some(CLIP_TIMEOUT),
)
.await
.context("Failed to run batch CLIP classification")?;
let json_str = std::fs::read_to_string(&output_path)
.context("Failed to read CLIP batch output")?;
let results_map: std::collections::HashMap<String, Vec<ClipPrediction>> =
serde_json::from_str(&json_str)
.context("Failed to parse CLIP batch output")?;
let results: Vec<ClipImageResult> = image_paths
.iter()
.map(|path| ClipImageResult {
image_path: path.to_string(),
predictions: results_map.get(*path).cloned().unwrap_or_default(),
})
.collect();
// Cleanup temp files
let _ = std::fs::remove_file(&temp_file);
let _ = std::fs::remove_file(&output_path);
Ok(results)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_clip_prediction_serialization() {
let pred = ClipPrediction {
label: "person in room".to_string(),
confidence: 0.876,
};
let json = serde_json::to_string(&pred).unwrap();
assert!(json.contains("person in room"));
assert!(json.contains("0.876"));
}
#[test]
fn test_clip_prediction_deserialization() {
let json = r#"{"label":"outdoor scene","confidence":0.945}"#;
let pred: ClipPrediction = serde_json::from_str(json).unwrap();
assert_eq!(pred.label, "outdoor scene");
assert!((pred.confidence - 0.945).abs() < 0.001);
}
#[test]
fn test_clip_image_result() {
let result = ClipImageResult {
image_path: "/test/image.jpg".to_string(),
predictions: vec![
ClipPrediction {
label: "indoor".to_string(),
confidence: 0.92,
},
ClipPrediction {
label: "outdoor".to_string(),
confidence: 0.08,
},
],
};
assert_eq!(result.predictions.len(), 2);
assert_eq!(result.predictions[0].label, "indoor");
}
}
+4
View File
@@ -1,6 +1,8 @@
pub mod asr;
pub mod asrx;
pub mod caption;
pub mod cascade_vision;
pub mod clip;
pub mod cut;
pub mod executor;
pub mod face;
@@ -16,6 +18,8 @@ pub mod yolo;
pub use asr::{process_asr, AsrResult, AsrSegment};
pub use asrx::{process_asrx, AsrxResult, AsrxSegment};
pub use caption::{process_caption, CaptionResult, CaptionSummary, FrameCaption};
pub use cascade_vision::{CascadeDetectionResult, CascadeVisionProcessor};
pub use clip::{classify_image, classify_images, detect_objects, ClipDetectionResult, ClipImageResult, ClipPrediction};
pub use cut::{process_cut, CutResult, CutScene};
pub use executor::{validate_python_env, PythonExecutor, RetryConfig};
pub use face::{process_face, Face, FaceFrame, FaceResult};
+1
View File
@@ -0,0 +1 @@
pub mod qwen_vl_manager;
+218
View File
@@ -0,0 +1,218 @@
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::Mutex;
use tracing::{debug, error, info, warn};
pub struct QwenVLManager {
port: u16,
model_path: PathBuf,
mmproj_path: PathBuf,
log_file: PathBuf,
pid_file: PathBuf,
start_script: PathBuf,
stop_script: PathBuf,
last_request_time: Arc<Mutex<Instant>>,
max_startup_time: Duration,
}
impl QwenVLManager {
pub fn new() -> Self {
Self {
port: 8086,
model_path: PathBuf::from("/Users/accusys/models/Qwen3VL-8B-Instruct-Q8_0.gguf"),
mmproj_path: PathBuf::from("/Users/accusys/models/mmproj-Qwen3VL-8B-Instruct-F16.gguf"),
log_file: PathBuf::from("logs/qwen3vl_8086.log"),
pid_file: PathBuf::from("/tmp/qwen3vl.pid"),
start_script: PathBuf::from("scripts/start_qwen3vl.sh"),
stop_script: PathBuf::from("scripts/stop_qwen3vl.sh"),
last_request_time: Arc::new(Mutex::new(Instant::now())),
max_startup_time: Duration::from_secs(60),
}
}
pub fn with_port(port: u16) -> Self {
let mut manager = Self::new();
manager.port = port;
manager.pid_file = PathBuf::from(format!("/tmp/qwen3vl_{}.pid", port));
manager.log_file = PathBuf::from(format!("logs/qwen3vl_{}.log", port));
manager
}
pub fn get_port(&self) -> u16 {
self.port
}
pub async fn is_running(&self) -> Result<bool> {
let health_url = format!("http://localhost:{}/health", self.port);
let client = reqwest::Client::new();
let response = client
.get(&health_url)
.timeout(Duration::from_secs(5))
.send()
.await;
match response {
Ok(resp) => {
let status = resp.status();
let body = resp.text().await?;
if status.is_success() && body.contains("\"status\":\"ok\"") {
debug!("Qwen3-VL is running on port {}", self.port);
return Ok(true);
}
debug!("Qwen3-VL health check failed: {}", status);
Ok(false)
}
Err(e) => {
debug!("Qwen3-VL not reachable: {}", e);
Ok(false)
}
}
}
pub async fn ensure_running(&self) -> Result<()> {
if self.is_running().await? {
debug!("Qwen3-VL already running");
self.update_last_request_time().await;
return Ok(());
}
info!("Starting Qwen3-VL server on port {}", self.port);
self.start_server().await?;
self.wait_for_ready().await?;
self.update_last_request_time().await;
info!("Qwen3-VL server started successfully");
Ok(())
}
pub async fn start_server(&self) -> Result<()> {
let script_path = self.start_script.canonicalize()
.context("Failed to resolve start script path")?;
debug!("Running start script: {}", script_path.display());
let output = Command::new("bash")
.arg(&script_path)
.output()
.context("Failed to execute start script")?;
if !output.status.success() {
error!("Start script failed: {}", String::from_utf8_lossy(&output.stderr));
anyhow::bail!("Failed to start Qwen3-VL server");
}
debug!("Start script output: {}", String::from_utf8_lossy(&output.stdout));
Ok(())
}
pub async fn stop_server(&self) -> Result<()> {
let script_path = self.stop_script.canonicalize()
.context("Failed to resolve stop script path")?;
debug!("Running stop script: {}", script_path.display());
let output = Command::new("bash")
.arg(&script_path)
.output()
.context("Failed to execute stop script")?;
if !output.status.success() {
warn!("Stop script returned error: {}", String::from_utf8_lossy(&output.stderr));
}
debug!("Stop script output: {}", String::from_utf8_lossy(&output.stdout));
tokio::time::sleep(Duration::from_secs(2)).await;
if self.is_running().await? {
warn!("Qwen3-VL still running after stop script");
}
info!("Qwen3-VL server stopped");
Ok(())
}
pub async fn wait_for_ready(&self) -> Result<()> {
let health_url = format!("http://localhost:{}/health", self.port);
let client = reqwest::Client::new();
let start_time = Instant::now();
while start_time.elapsed() < self.max_startup_time {
let response = client
.get(&health_url)
.timeout(Duration::from_secs(2))
.send()
.await;
match response {
Ok(resp) => {
if resp.status().is_success() {
let body = resp.text().await?;
if body.contains("\"status\":\"ok\"") {
debug!("Qwen3-VL ready after {} seconds", start_time.elapsed().as_secs());
return Ok(());
}
}
}
Err(_) => {}
}
tokio::time::sleep(Duration::from_secs(2)).await;
}
error!("Qwen3-VL failed to start within {} seconds", self.max_startup_time.as_secs());
anyhow::bail!("Qwen3-VL startup timeout");
}
pub async fn update_last_request_time(&self) {
let mut last_request = self.last_request_time.lock().await;
*last_request = Instant::now();
debug!("Updated last request time");
}
pub async fn get_status(&self) -> Result<QwenVLStatus> {
let is_running = self.is_running().await?;
let last_request = self.last_request_time.lock().await.clone();
Ok(QwenVLStatus {
running: is_running,
port: self.port,
model_path: self.model_path.display().to_string(),
last_request: last_request.elapsed().as_secs(),
pid_file: self.pid_file.display().to_string(),
log_file: self.log_file.display().to_string(),
})
}
pub async fn auto_stop_if_idle(&self, idle_timeout: Duration) -> Result<()> {
let last_request = self.last_request_time.lock().await.clone();
if last_request.elapsed() > idle_timeout && self.is_running().await? {
info!("Qwen3-VL idle for {} seconds, stopping server", last_request.elapsed().as_secs());
self.stop_server().await?;
}
Ok(())
}
}
#[derive(Debug, Clone, serde::Serialize)]
pub struct QwenVLStatus {
pub running: bool,
pub port: u16,
pub model_path: String,
pub last_request: u64,
pub pid_file: String,
pub log_file: String,
}
impl Default for QwenVLManager {
fn default() -> Self {
Self::new()
}
}