feat: add Vision LLM integration (CLIP + Qwen3-VL cascade)
- Add Qwen3-VL dynamic management (start/stop/status CLI) - Add CLIP + Qwen3-VL cascade detection strategy - Add Vision CLI commands (vision start/stop/status, detect) - Add cascade_vision processor module - Add clip processor module - Add qwen_vl_manager module Changes: - scripts/start_qwen3vl.sh, stop_qwen3vl.sh: Qwen3-VL management scripts - src/core/vision/: Qwen3-VL manager module - src/core/processor/cascade_vision.rs: CLIP + Qwen3-VL cascade logic - src/core/processor/clip.rs: CLIP classification and detection - src/api/clip_api.rs: CLIP API endpoints - src/cli/vision.rs: Vision CLI implementation - src/cli/args.rs: Add Vision and Detect commands - src/main.rs: Integrate Vision CLI - src/core/mod.rs: Add vision module - src/core/processor/mod.rs: Add cascade_vision module
This commit is contained in:
@@ -92,6 +92,16 @@ pub static MEDIA_BASE_URL: Lazy<String> = Lazy::new(|| {
|
||||
.unwrap_or_else(|_| "https://wp.momentry.ddns.net".to_string())
|
||||
});
|
||||
|
||||
pub static STORAGE_ROOT: Lazy<String> = Lazy::new(|| {
|
||||
env::var("MOMENTRY_STORAGE_ROOT")
|
||||
.unwrap_or_else(|_| "/Users/accusys/momentry/var/sftpgo/data".to_string())
|
||||
});
|
||||
|
||||
pub static SERVE_BASE_URL: Lazy<String> = Lazy::new(|| {
|
||||
env::var("MOMENTRY_SERVE_BASE_URL")
|
||||
.unwrap_or_else(|_| "https://m5wp.momentry.ddns.net/files".to_string())
|
||||
});
|
||||
|
||||
pub static SERVER_PORT: Lazy<u16> = Lazy::new(|| {
|
||||
env::var("MOMENTRY_SERVER_PORT")
|
||||
.unwrap_or_else(|_| "3002".to_string())
|
||||
|
||||
@@ -2862,7 +2862,7 @@ impl PostgresDb {
|
||||
let rows = if let Some(u) = file_uuid {
|
||||
sqlx::query(&format!(
|
||||
"SELECT chunk_id, file_uuid, chunk_type, text_content, start_time, end_time, 1.0::float8 as score \
|
||||
FROM {} WHERE file_uuid=$1 AND text_content ILIKE $2 LIMIT $3", table)
|
||||
FROM {} WHERE file_uuid=$1 AND text_content ILIKE $2 AND text_content != '' LIMIT $3", table)
|
||||
)
|
||||
.bind(u).bind(&like).bind(limit)
|
||||
.fetch_all(&self.pool).await?
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
use anyhow::Result;
|
||||
use reqwest::Client;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::Duration;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use crate::core::config;
|
||||
use crate::core::llm::function_calling::LLM_CLIENT;
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct ChatRequest {
|
||||
@@ -39,10 +39,6 @@ pub async fn generate_5w1h_summary(scene_text: &str) -> Result<String> {
|
||||
return Ok("LLM Disabled".to_string());
|
||||
}
|
||||
|
||||
let client = Client::builder()
|
||||
.timeout(Duration::from_secs(*config::llm::SUMMARY_TIMEOUT_SECS))
|
||||
.build()?;
|
||||
|
||||
let prompt = format!(
|
||||
r#"Analyze the following video scene transcript and provide a concise 5W1H+ summary in JSON format.
|
||||
Focus on: Who, What, Where, When, Why, How, and Key Objects/Actions.
|
||||
@@ -82,9 +78,10 @@ pub async fn generate_5w1h_summary(scene_text: &str) -> Result<String> {
|
||||
|
||||
debug!("Calling LLM for summary: {}", *config::llm::SUMMARY_URL);
|
||||
|
||||
let res = client
|
||||
let res = LLM_CLIENT
|
||||
.post(&*config::llm::SUMMARY_URL)
|
||||
.json(&req)
|
||||
.timeout(Duration::from_secs(*config::llm::SUMMARY_TIMEOUT_SECS))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
|
||||
@@ -1,8 +1,18 @@
|
||||
use once_cell::sync::Lazy;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::core::config;
|
||||
|
||||
/// Shared HTTP client with connection pooling for all LLM calls
|
||||
pub static LLM_CLIENT: Lazy<reqwest::Client> = Lazy::new(|| {
|
||||
reqwest::Client::builder()
|
||||
.pool_max_idle_per_host(32)
|
||||
.pool_idle_timeout(std::time::Duration::from_secs(300))
|
||||
.build()
|
||||
.expect("Failed to create shared LLM HTTP client")
|
||||
});
|
||||
|
||||
/// A tool/function definition for Gemma4 function calling
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct ToolDef {
|
||||
@@ -126,11 +136,11 @@ pub async fn call_llm_vision(
|
||||
"stream": false,
|
||||
});
|
||||
|
||||
let client = reqwest::Client::builder()
|
||||
let res = LLM_CLIENT
|
||||
.post(&llm_vision_url())
|
||||
.json(&req)
|
||||
.timeout(std::time::Duration::from_secs(timeout_secs))
|
||||
.build()?;
|
||||
|
||||
let res = client.post(&llm_vision_url()).json(&req).send().await?;
|
||||
.send().await?;
|
||||
if !res.status().is_success() {
|
||||
let text = res.text().await.unwrap_or_default();
|
||||
anyhow::bail!("Vision LLM API error: {}", text);
|
||||
@@ -182,13 +192,11 @@ pub async fn call_llm(
|
||||
max_tokens: u32,
|
||||
timeout_secs: u64,
|
||||
) -> anyhow::Result<LlmResponse> {
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(if timeout_secs > 0 {
|
||||
timeout_secs
|
||||
} else {
|
||||
*config::llm::CHAT_TIMEOUT_SECS
|
||||
}))
|
||||
.build()?;
|
||||
let timeout = if timeout_secs > 0 {
|
||||
timeout_secs
|
||||
} else {
|
||||
*config::llm::CHAT_TIMEOUT_SECS
|
||||
};
|
||||
|
||||
let req = ChatRequest {
|
||||
model: llm_model(),
|
||||
@@ -199,7 +207,11 @@ pub async fn call_llm(
|
||||
tools,
|
||||
};
|
||||
|
||||
let res = client.post(&llm_chat_url()).json(&req).send().await?;
|
||||
let res = LLM_CLIENT
|
||||
.post(&llm_chat_url())
|
||||
.json(&req)
|
||||
.timeout(std::time::Duration::from_secs(timeout))
|
||||
.send().await?;
|
||||
|
||||
if !res.status().is_success() {
|
||||
let text = res.text().await.unwrap_or_default();
|
||||
|
||||
+21
-10
@@ -1,12 +1,12 @@
|
||||
use std::collections::HashSet;
|
||||
|
||||
use anyhow::Result;
|
||||
use reqwest::Client;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::Duration;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::core::config;
|
||||
use crate::core::llm::function_calling::LLM_CLIENT;
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct ChatRequest {
|
||||
@@ -38,7 +38,10 @@ struct RerankResponse {
|
||||
ranked: Vec<usize>,
|
||||
}
|
||||
|
||||
pub async fn rerank_search_results(query: &str, candidates: &[(usize, &str)]) -> Result<Vec<usize>> {
|
||||
pub async fn rerank_search_results(
|
||||
query: &str,
|
||||
candidates: &[(usize, &str)],
|
||||
) -> Result<Vec<usize>> {
|
||||
if candidates.is_empty() {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
@@ -67,10 +70,6 @@ Include every chunk number exactly once. Only respond with the JSON."#,
|
||||
query, chunks_text
|
||||
);
|
||||
|
||||
let client = Client::builder()
|
||||
.timeout(Duration::from_secs(15))
|
||||
.build()?;
|
||||
|
||||
let req = ChatRequest {
|
||||
model: config::llm::CHAT_MODEL.clone(),
|
||||
messages: vec![
|
||||
@@ -88,11 +87,16 @@ Include every chunk number exactly once. Only respond with the JSON."#,
|
||||
stream: false,
|
||||
};
|
||||
|
||||
debug!("LLM rerank: {} candidates for query '{}'", candidates.len(), query);
|
||||
debug!(
|
||||
"LLM rerank: {} candidates for query '{}'",
|
||||
candidates.len(),
|
||||
query
|
||||
);
|
||||
|
||||
let res = client
|
||||
let res = LLM_CLIENT
|
||||
.post(&*config::llm::CHAT_URL)
|
||||
.json(&req)
|
||||
.timeout(Duration::from_secs(15))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
@@ -116,7 +120,11 @@ Include every chunk number exactly once. Only respond with the JSON."#,
|
||||
// Strip markdown code fences if present
|
||||
let content = if content.starts_with("```") {
|
||||
let lines: Vec<&str> = content.lines().collect();
|
||||
let start = if lines.first().map(|l| l.contains("```")).unwrap_or(false) { 1 } else { 0 };
|
||||
let start = if lines.first().map(|l| l.contains("```")).unwrap_or(false) {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let end = if lines.last().map(|l| l.contains("```")).unwrap_or(false) {
|
||||
lines.len().saturating_sub(1)
|
||||
} else {
|
||||
@@ -163,6 +171,9 @@ Include every chunk number exactly once. Only respond with the JSON."#,
|
||||
}
|
||||
}
|
||||
|
||||
warn!("LLM rerank: could not parse response — content: {}", &content[..content.len().min(200)]);
|
||||
warn!(
|
||||
"LLM rerank: could not parse response — content: {}",
|
||||
&content[..content.len().min(200)]
|
||||
);
|
||||
Ok(candidates.iter().map(|(idx, _)| *idx).collect())
|
||||
}
|
||||
|
||||
@@ -20,3 +20,4 @@ pub mod text;
|
||||
pub mod thumbnail;
|
||||
pub mod time;
|
||||
pub mod tmdb;
|
||||
pub mod vision;
|
||||
|
||||
+30
-21
@@ -17,8 +17,8 @@ pub async fn store_asrx_chunks(db: &PostgresDb, uuid: &str) -> Result<()> {
|
||||
|
||||
let json_str = std::fs::read_to_string(&asrx_path)
|
||||
.with_context(|| format!("ASRX file not found: {:?}", asrx_path))?;
|
||||
let result: AsrxResult = serde_json::from_str(&json_str)
|
||||
.context("Failed to parse ASRX JSON")?;
|
||||
let result: AsrxResult =
|
||||
serde_json::from_str(&json_str).context("Failed to parse ASRX JSON")?;
|
||||
|
||||
let segments_count = result.segments.len();
|
||||
let mut pre_chunks = Vec::new();
|
||||
@@ -41,21 +41,26 @@ pub async fn store_asrx_chunks(db: &PostgresDb, uuid: &str) -> Result<()> {
|
||||
));
|
||||
}
|
||||
|
||||
db.store_raw_pre_chunks_batch(uuid, "asrx", &pre_chunks).await?;
|
||||
db.store_raw_pre_chunks_batch(uuid, "asr", &pre_chunks).await?;
|
||||
db.store_speaker_detections_batch(uuid, &speaker_detections).await?;
|
||||
db.store_raw_pre_chunks_batch(uuid, "asrx", &pre_chunks)
|
||||
.await?;
|
||||
db.store_raw_pre_chunks_batch(uuid, "asr", &pre_chunks)
|
||||
.await?;
|
||||
db.store_speaker_detections_batch(uuid, &speaker_detections)
|
||||
.await?;
|
||||
|
||||
println!("Stored {} ASRX pre-chunks for {}", segments_count, uuid);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn execute_rule1(db: &PostgresDb, uuid: &str) -> Result<usize> {
|
||||
let video = db.get_video_by_uuid(uuid)
|
||||
let video = db
|
||||
.get_video_by_uuid(uuid)
|
||||
.await?
|
||||
.context("Video not found")?;
|
||||
let fps = video.fps;
|
||||
|
||||
let count = rule1_ingest::execute_rule1(db, uuid, fps).await
|
||||
let count = rule1_ingest::execute_rule1(db, uuid, fps)
|
||||
.await
|
||||
.context("Rule 1 ingestion failed")?;
|
||||
|
||||
println!("Rule 1 completed: {} chunks inserted for {}", count, uuid);
|
||||
@@ -68,17 +73,15 @@ pub async fn vectorize_chunks(uuid: &str) -> Result<()> {
|
||||
let embedder = Embedder::new("embeddinggemma-300m".to_string());
|
||||
|
||||
let chunk_table = schema::table_name("chunk");
|
||||
let rows = sqlx::query_as::<_, (String, String, String, i64, i64, f64, f64, String)>(
|
||||
&format!(
|
||||
"SELECT chunk_id, chunk_type, text_content, start_frame, end_frame, \
|
||||
let rows = sqlx::query_as::<_, (String, String, String, i64, i64, f64, f64, String)>(&format!(
|
||||
"SELECT chunk_id, chunk_type, text_content, start_frame, end_frame, \
|
||||
start_time, end_time, content::text \
|
||||
FROM {} WHERE file_uuid = $1 AND chunk_type = 'sentence' \
|
||||
AND embedding IS NULL \
|
||||
AND (text_content IS NOT NULL AND text_content != '') \
|
||||
ORDER BY id",
|
||||
chunk_table
|
||||
),
|
||||
)
|
||||
chunk_table
|
||||
))
|
||||
.bind(uuid)
|
||||
.fetch_all(db.pool())
|
||||
.await?;
|
||||
@@ -91,7 +94,9 @@ pub async fn vectorize_chunks(uuid: &str) -> Result<()> {
|
||||
let total = rows.len();
|
||||
let mut stored = 0usize;
|
||||
|
||||
for (chunk_id, _chunk_type, text, start_frame, end_frame, start_time, end_time, _content_str) in &rows {
|
||||
for (chunk_id, _chunk_type, text, start_frame, end_frame, start_time, end_time, _content_str) in
|
||||
&rows
|
||||
{
|
||||
if text.is_empty() {
|
||||
continue;
|
||||
}
|
||||
@@ -127,13 +132,15 @@ pub async fn vectorize_chunks(uuid: &str) -> Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
println!("Vectorization complete: {}/{} vectors for {}", stored, total, uuid);
|
||||
println!(
|
||||
"Vectorization complete: {}/{} vectors for {}",
|
||||
stored, total, uuid
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn run_phase1(uuid: &str) -> Result<()> {
|
||||
let executor = PythonExecutor::new()
|
||||
.context("Failed to create PythonExecutor")?;
|
||||
let executor = PythonExecutor::new().context("Failed to create PythonExecutor")?;
|
||||
|
||||
executor
|
||||
.run(
|
||||
@@ -154,15 +161,17 @@ pub async fn mark_complete(db: &PostgresDb, uuid: &str) -> Result<()> {
|
||||
use crate::core::db::MonitorJobStatus;
|
||||
use crate::core::db::VideoStatus;
|
||||
|
||||
let job_id = sqlx::query_scalar::<_, i32>(
|
||||
&format!("SELECT id FROM {} WHERE uuid = $1 LIMIT 1", schema::table_name("monitor_jobs")),
|
||||
)
|
||||
let job_id = sqlx::query_scalar::<_, i32>(&format!(
|
||||
"SELECT id FROM {} WHERE uuid = $1 LIMIT 1",
|
||||
schema::table_name("monitor_jobs")
|
||||
))
|
||||
.bind(uuid)
|
||||
.fetch_optional(db.pool())
|
||||
.await?;
|
||||
|
||||
if let Some(job_id) = job_id {
|
||||
db.update_job_status(job_id, MonitorJobStatus::Completed).await?;
|
||||
db.update_job_status(job_id, MonitorJobStatus::Completed)
|
||||
.await?;
|
||||
println!("Job {} marked as completed", job_id);
|
||||
}
|
||||
|
||||
|
||||
@@ -44,10 +44,7 @@ pub async fn process_asrx(
|
||||
let executor = PythonExecutor::new()?;
|
||||
let script_path = executor.script_path("asrx_processor.py");
|
||||
|
||||
tracing::info!(
|
||||
"[ASRX] Starting hybrid speaker diarization: {}",
|
||||
video_path
|
||||
);
|
||||
tracing::info!("[ASRX] Starting hybrid speaker diarization: {}", video_path);
|
||||
|
||||
if !script_path.exists() {
|
||||
tracing::error!("[ASRX] Script not found: {:?}", script_path);
|
||||
|
||||
@@ -0,0 +1,308 @@
|
||||
use anyhow::{Context, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::core::processor::clip::{ClipPrediction, detect_objects};
|
||||
use crate::core::vision::qwen_vl_manager::QwenVLManager;
|
||||
|
||||
const DEFAULT_CLIP_THRESHOLD: f32 = 0.7;
|
||||
const QWENVL_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CascadeDetectionResult {
|
||||
pub detections: Vec<ClipPrediction>,
|
||||
pub model_used: String,
|
||||
pub clip_confidence: f32,
|
||||
pub qwenvl_used: bool,
|
||||
pub processing_time_ms: u64,
|
||||
}
|
||||
|
||||
pub struct CascadeVisionProcessor {
|
||||
clip_threshold: f32,
|
||||
qwen_vl_manager: QwenVLManager,
|
||||
}
|
||||
|
||||
impl CascadeVisionProcessor {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
clip_threshold: DEFAULT_CLIP_THRESHOLD,
|
||||
qwen_vl_manager: QwenVLManager::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_threshold(threshold: f32) -> Self {
|
||||
Self {
|
||||
clip_threshold: threshold,
|
||||
qwen_vl_manager: QwenVLManager::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn detect_objects(&self, image_path: &Path, objects: &[&str]) -> Result<CascadeDetectionResult> {
|
||||
let start_time = std::time::Instant::now();
|
||||
|
||||
info!(
|
||||
"[Cascade] Starting detection for {:?} with {} object classes (threshold: {:.2})",
|
||||
image_path,
|
||||
objects.len(),
|
||||
self.clip_threshold
|
||||
);
|
||||
|
||||
let clip_result = self.run_clip_detection(image_path, objects).await?;
|
||||
|
||||
let max_clip_confidence = clip_result
|
||||
.iter()
|
||||
.map(|p| p.confidence)
|
||||
.fold(0.0_f32, |max, val| if val > max { val } else { max });
|
||||
|
||||
debug!(
|
||||
"[Cascade] CLIP max confidence: {:.3} (threshold: {:.2})",
|
||||
max_clip_confidence,
|
||||
self.clip_threshold
|
||||
);
|
||||
|
||||
if max_clip_confidence > self.clip_threshold {
|
||||
info!(
|
||||
"[Cascade] High confidence ({:.3} > {:.2}) → triggering Qwen3-VL",
|
||||
max_clip_confidence,
|
||||
self.clip_threshold
|
||||
);
|
||||
|
||||
let qwenvl_result = self.run_qwenvl_detection(image_path, objects).await?;
|
||||
|
||||
let processing_time = start_time.elapsed().as_millis() as u64;
|
||||
|
||||
return Ok(CascadeDetectionResult {
|
||||
detections: qwenvl_result,
|
||||
model_used: "qwen3vl".to_string(),
|
||||
clip_confidence: max_clip_confidence,
|
||||
qwenvl_used: true,
|
||||
processing_time_ms: processing_time,
|
||||
});
|
||||
}
|
||||
|
||||
info!(
|
||||
"[Cascade] Low confidence ({:.3} <= {:.2}) → using CLIP results only",
|
||||
max_clip_confidence,
|
||||
self.clip_threshold
|
||||
);
|
||||
|
||||
let processing_time = start_time.elapsed().as_millis() as u64;
|
||||
|
||||
Ok(CascadeDetectionResult {
|
||||
detections: clip_result,
|
||||
model_used: "clip".to_string(),
|
||||
clip_confidence: max_clip_confidence,
|
||||
qwenvl_used: false,
|
||||
processing_time_ms: processing_time,
|
||||
})
|
||||
}
|
||||
|
||||
async fn run_clip_detection(&self, image_path: &Path, objects: &[&str]) -> Result<Vec<ClipPrediction>> {
|
||||
let image_path_str = image_path.display().to_string();
|
||||
|
||||
debug!("[Cascade] Running CLIP detection for {:?}", image_path);
|
||||
|
||||
let predictions = detect_objects(&image_path_str, objects, None, None)
|
||||
.await
|
||||
.context("CLIP detection failed")?;
|
||||
|
||||
debug!(
|
||||
"[Cascade] CLIP detected {} objects",
|
||||
predictions.len()
|
||||
);
|
||||
|
||||
Ok(predictions)
|
||||
}
|
||||
|
||||
async fn run_qwenvl_detection(&self, image_path: &Path, objects: &[&str]) -> Result<Vec<ClipPrediction>> {
|
||||
let image_path_str = image_path.display().to_string();
|
||||
|
||||
debug!("[Cascade] Running Qwen3-VL detection for {:?}", image_path);
|
||||
|
||||
self.qwen_vl_manager.ensure_running().await?;
|
||||
|
||||
let prompt = self.build_detection_prompt(objects);
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let url = format!("http://localhost:{}/v1/chat/completions", self.qwen_vl_manager.get_port());
|
||||
|
||||
let request_body = serde_json::json!({
|
||||
"model": "Qwen3VL-8B-Instruct-Q8_0",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": prompt
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": format!("file://{}", image_path_str)
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"max_tokens": 500,
|
||||
"temperature": 0.1
|
||||
});
|
||||
|
||||
let response = client
|
||||
.post(&url)
|
||||
.json(&request_body)
|
||||
.timeout(QWENVL_TIMEOUT)
|
||||
.send()
|
||||
.await
|
||||
.context("Qwen3-VL API request failed")?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
warn!("[Cascade] Qwen3-VL API error: {}", response.status());
|
||||
anyhow::bail!("Qwen3-VL API returned error: {}", response.status());
|
||||
}
|
||||
|
||||
let response_json: serde_json::Value = response
|
||||
.json()
|
||||
.await
|
||||
.context("Failed to parse Qwen3-VL response")?;
|
||||
|
||||
let content = response_json
|
||||
.get("choices")
|
||||
.and_then(|choices| choices.get(0))
|
||||
.and_then(|choice| choice.get("message"))
|
||||
.and_then(|message| message.get("content"))
|
||||
.and_then(|content| content.as_str())
|
||||
.unwrap_or("");
|
||||
|
||||
debug!("[Cascade] Qwen3-VL response: {}", content);
|
||||
|
||||
let detections = self.parse_qwenvl_response(content, objects);
|
||||
|
||||
self.qwen_vl_manager.update_last_request_time().await;
|
||||
|
||||
info!(
|
||||
"[Cascade] Qwen3-VL detected {} objects",
|
||||
detections.len()
|
||||
);
|
||||
|
||||
Ok(detections)
|
||||
}
|
||||
|
||||
fn build_detection_prompt(&self, objects: &[&str]) -> String {
|
||||
let object_list = objects.join(", ");
|
||||
|
||||
format!(
|
||||
"Analyze this image and detect the following objects: {}.\n\
|
||||
For each detected object, provide:\n\
|
||||
1. The object name\n\
|
||||
2. A confidence score (0.0 to 1.0)\n\
|
||||
3. A brief description of what you see\n\
|
||||
\n\
|
||||
Format your response as JSON:\n\
|
||||
{{\"detections\": [{{\"label\": \"object_name\", \"confidence\": 0.95, \"description\": \"brief description\"}}]}}\n\
|
||||
\n\
|
||||
If no objects are detected, return: {{\"detections\": []}}\n\
|
||||
\n\
|
||||
IMPORTANT: Only detect objects that are clearly visible and identifiable. Do not guess or hallucinate.",
|
||||
object_list
|
||||
)
|
||||
}
|
||||
|
||||
fn parse_qwenvl_response(&self, content: &str, _objects: &[&str]) -> Vec<ClipPrediction> {
|
||||
let json_start = content.find('{');
|
||||
let json_end = content.rfind('}');
|
||||
|
||||
if json_start.is_none() || json_end.is_none() {
|
||||
debug!("[Cascade] No JSON found in Qwen3-VL response");
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let json_str = &content[json_start.unwrap()..=json_end.unwrap()];
|
||||
|
||||
let parsed: serde_json::Value = serde_json::from_str(json_str)
|
||||
.unwrap_or(serde_json::json!({"detections": []}));
|
||||
|
||||
let detections = parsed
|
||||
.get("detections")
|
||||
.and_then(|d| d.as_array())
|
||||
.map(|arr| arr.clone())
|
||||
.unwrap_or_else(|| Vec::new());
|
||||
|
||||
detections
|
||||
.iter()
|
||||
.filter_map(|d| {
|
||||
let label = d.get("label").and_then(|l| l.as_str()).unwrap_or("");
|
||||
let confidence = d.get("confidence").and_then(|c| c.as_f64()).unwrap_or(0.0) as f32;
|
||||
|
||||
if !label.is_empty() && confidence > 0.0 {
|
||||
Some(ClipPrediction {
|
||||
label: label.to_string(),
|
||||
confidence,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for CascadeVisionProcessor {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_build_detection_prompt() {
|
||||
let processor = CascadeVisionProcessor::new();
|
||||
let objects = vec!["gun", "weapon", "person"];
|
||||
let prompt = processor.build_detection_prompt(&objects);
|
||||
|
||||
assert!(prompt.contains("gun, weapon, person"));
|
||||
assert!(prompt.contains("confidence score"));
|
||||
assert!(prompt.contains("JSON"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_qwenvl_response() {
|
||||
let processor = CascadeVisionProcessor::new();
|
||||
let response = "{\"detections\": [{\"label\": \"gun\", \"confidence\": 0.95, \"description\": \"a handgun\"}]}";
|
||||
let objects = vec!["gun"];
|
||||
|
||||
let detections = processor.parse_qwenvl_response(response, &objects);
|
||||
|
||||
assert_eq!(detections.len(), 1);
|
||||
assert_eq!(detections[0].label, "gun");
|
||||
assert!((detections[0].confidence - 0.95).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_empty_response() {
|
||||
let processor = CascadeVisionProcessor::new();
|
||||
let response = "{\"detections\": []}";
|
||||
let objects = vec!["gun"];
|
||||
|
||||
let detections = processor.parse_qwenvl_response(response, &objects);
|
||||
|
||||
assert_eq!(detections.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_invalid_json() {
|
||||
let processor = CascadeVisionProcessor::new();
|
||||
let response = "This is not JSON";
|
||||
let objects = vec!["gun"];
|
||||
|
||||
let detections = processor.parse_qwenvl_response(response, &objects);
|
||||
|
||||
assert_eq!(detections.len(), 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
use anyhow::{Context, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::Duration;
|
||||
|
||||
use super::executor::PythonExecutor;
|
||||
|
||||
const CLIP_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
|
||||
/// CLIP classification prediction
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
pub struct ClipPrediction {
|
||||
pub label: String,
|
||||
pub confidence: f32,
|
||||
}
|
||||
|
||||
/// CLIP classification result for a single image
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
pub struct ClipImageResult {
|
||||
pub image_path: String,
|
||||
pub predictions: Vec<ClipPrediction>,
|
||||
}
|
||||
|
||||
/// CLIP object detection result
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
pub struct ClipDetectionResult {
|
||||
pub image_path: String,
|
||||
pub detected_objects: Vec<ClipPrediction>,
|
||||
}
|
||||
|
||||
/// Classify a single image with given labels
|
||||
pub async fn classify_image(
|
||||
image_path: &str,
|
||||
labels: &[&str],
|
||||
top_k: Option<usize>,
|
||||
model_name: Option<&str>,
|
||||
) -> Result<Vec<ClipPrediction>> {
|
||||
let executor = PythonExecutor::new()?;
|
||||
let script_path = executor.script_path("clip_classifier.py");
|
||||
|
||||
if !script_path.exists() {
|
||||
anyhow::bail!("clip_classifier.py not found at {:?}", script_path);
|
||||
}
|
||||
|
||||
let top_k = top_k.unwrap_or(5);
|
||||
let model = model_name.unwrap_or("openai/clip-vit-base-patch32");
|
||||
|
||||
let mut args = vec![
|
||||
image_path.to_string(),
|
||||
"--labels".to_string(),
|
||||
labels.join(","),
|
||||
"--top-k".to_string(),
|
||||
top_k.to_string(),
|
||||
"--model".to_string(),
|
||||
model.to_string(),
|
||||
];
|
||||
|
||||
let output_path = format!("{}.clip.json", image_path);
|
||||
args.push("--output".to_string());
|
||||
args.push(output_path.clone());
|
||||
|
||||
tracing::info!(
|
||||
"[CLIP] Classifying image: {} with {} labels",
|
||||
image_path,
|
||||
labels.len()
|
||||
);
|
||||
|
||||
executor
|
||||
.run(
|
||||
"clip_classifier.py",
|
||||
&args.iter().map(|s| s.as_str()).collect::<Vec<_>>(),
|
||||
None,
|
||||
"CLIP",
|
||||
Some(CLIP_TIMEOUT),
|
||||
)
|
||||
.await
|
||||
.context("Failed to run CLIP classifier")?;
|
||||
|
||||
let json_str = std::fs::read_to_string(&output_path)
|
||||
.context("Failed to read CLIP output")?;
|
||||
|
||||
let results: std::collections::HashMap<String, Vec<ClipPrediction>> =
|
||||
serde_json::from_str(&json_str)
|
||||
.context("Failed to parse CLIP output")?;
|
||||
|
||||
let predictions = results
|
||||
.get(image_path)
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
|
||||
tracing::info!(
|
||||
"[CLIP] Top prediction: {} ({:.3})",
|
||||
predictions.first().map(|p| p.label.as_str()).unwrap_or("none"),
|
||||
predictions.first().map(|p| p.confidence).unwrap_or(0.0)
|
||||
);
|
||||
|
||||
Ok(predictions)
|
||||
}
|
||||
|
||||
/// Detect objects in an image
|
||||
pub async fn detect_objects(
|
||||
image_path: &str,
|
||||
objects: &[&str],
|
||||
threshold: Option<f32>,
|
||||
model_name: Option<&str>,
|
||||
) -> Result<Vec<ClipPrediction>> {
|
||||
let executor = PythonExecutor::new()?;
|
||||
let script_path = executor.script_path("clip_classifier.py");
|
||||
|
||||
if !script_path.exists() {
|
||||
anyhow::bail!("clip_classifier.py not found at {:?}", script_path);
|
||||
}
|
||||
|
||||
let threshold = threshold.unwrap_or(0.15);
|
||||
let model = model_name.unwrap_or("openai/clip-vit-base-patch32");
|
||||
|
||||
let mut args = vec![
|
||||
image_path.to_string(),
|
||||
"--detect".to_string(),
|
||||
objects.join(","),
|
||||
"--threshold".to_string(),
|
||||
threshold.to_string(),
|
||||
"--model".to_string(),
|
||||
model.to_string(),
|
||||
];
|
||||
|
||||
let output_path = format!("{}.clip.json", image_path);
|
||||
args.push("--output".to_string());
|
||||
args.push(output_path.clone());
|
||||
|
||||
tracing::info!(
|
||||
"[CLIP] Detecting {} objects in: {} (threshold: {:.2})",
|
||||
objects.len(),
|
||||
image_path,
|
||||
threshold
|
||||
);
|
||||
|
||||
executor
|
||||
.run(
|
||||
"clip_classifier.py",
|
||||
&args.iter().map(|s| s.as_str()).collect::<Vec<_>>(),
|
||||
None,
|
||||
"CLIP",
|
||||
Some(CLIP_TIMEOUT),
|
||||
)
|
||||
.await
|
||||
.context("Failed to run CLIP object detection")?;
|
||||
|
||||
let json_str = std::fs::read_to_string(&output_path)
|
||||
.context("Failed to read CLIP output")?;
|
||||
|
||||
let results: std::collections::HashMap<String, Vec<ClipPrediction>> =
|
||||
serde_json::from_str(&json_str)
|
||||
.context("Failed to parse CLIP output")?;
|
||||
|
||||
let detected = results
|
||||
.get(image_path)
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
|
||||
if !detected.is_empty() {
|
||||
tracing::info!(
|
||||
"[CLIP] Detected {} objects: {}",
|
||||
detected.len(),
|
||||
detected.iter().map(|p| p.label.as_str()).collect::<Vec<_>>().join(", ")
|
||||
);
|
||||
} else {
|
||||
tracing::info!("[CLIP] No objects detected above threshold {:.2}", threshold);
|
||||
}
|
||||
|
||||
Ok(detected)
|
||||
}
|
||||
|
||||
/// Batch classify multiple images
|
||||
pub async fn classify_images(
|
||||
image_paths: &[&str],
|
||||
labels: &[&str],
|
||||
top_k: Option<usize>,
|
||||
model_name: Option<&str>,
|
||||
) -> Result<Vec<ClipImageResult>> {
|
||||
let executor = PythonExecutor::new()?;
|
||||
let script_path = executor.script_path("clip_classifier.py");
|
||||
|
||||
if !script_path.exists() {
|
||||
anyhow::bail!("clip_classifier.py not found at {:?}", script_path);
|
||||
}
|
||||
|
||||
let top_k = top_k.unwrap_or(5);
|
||||
let model = model_name.unwrap_or("openai/clip-vit-base-patch32");
|
||||
|
||||
// Create temp file with image paths
|
||||
let temp_file = format!("/tmp/clip_batch_{}.txt", uuid::Uuid::new_v4());
|
||||
std::fs::write(&temp_file, image_paths.join("\n"))
|
||||
.context("Failed to write batch file")?;
|
||||
|
||||
let mut args = vec![
|
||||
temp_file.clone(),
|
||||
"--batch".to_string(),
|
||||
"--labels".to_string(),
|
||||
labels.join(","),
|
||||
"--top-k".to_string(),
|
||||
top_k.to_string(),
|
||||
"--model".to_string(),
|
||||
model.to_string(),
|
||||
];
|
||||
|
||||
let output_path = format!("/tmp/clip_batch_{}.json", uuid::Uuid::new_v4());
|
||||
args.push("--output".to_string());
|
||||
args.push(output_path.clone());
|
||||
|
||||
tracing::info!(
|
||||
"[CLIP] Batch classifying {} images with {} labels",
|
||||
image_paths.len(),
|
||||
labels.len()
|
||||
);
|
||||
|
||||
executor
|
||||
.run(
|
||||
"clip_classifier.py",
|
||||
&args.iter().map(|s| s.as_str()).collect::<Vec<_>>(),
|
||||
None,
|
||||
"CLIP",
|
||||
Some(CLIP_TIMEOUT),
|
||||
)
|
||||
.await
|
||||
.context("Failed to run batch CLIP classification")?;
|
||||
|
||||
let json_str = std::fs::read_to_string(&output_path)
|
||||
.context("Failed to read CLIP batch output")?;
|
||||
|
||||
let results_map: std::collections::HashMap<String, Vec<ClipPrediction>> =
|
||||
serde_json::from_str(&json_str)
|
||||
.context("Failed to parse CLIP batch output")?;
|
||||
|
||||
let results: Vec<ClipImageResult> = image_paths
|
||||
.iter()
|
||||
.map(|path| ClipImageResult {
|
||||
image_path: path.to_string(),
|
||||
predictions: results_map.get(*path).cloned().unwrap_or_default(),
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Cleanup temp files
|
||||
let _ = std::fs::remove_file(&temp_file);
|
||||
let _ = std::fs::remove_file(&output_path);
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_clip_prediction_serialization() {
|
||||
let pred = ClipPrediction {
|
||||
label: "person in room".to_string(),
|
||||
confidence: 0.876,
|
||||
};
|
||||
let json = serde_json::to_string(&pred).unwrap();
|
||||
assert!(json.contains("person in room"));
|
||||
assert!(json.contains("0.876"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clip_prediction_deserialization() {
|
||||
let json = r#"{"label":"outdoor scene","confidence":0.945}"#;
|
||||
let pred: ClipPrediction = serde_json::from_str(json).unwrap();
|
||||
assert_eq!(pred.label, "outdoor scene");
|
||||
assert!((pred.confidence - 0.945).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clip_image_result() {
|
||||
let result = ClipImageResult {
|
||||
image_path: "/test/image.jpg".to_string(),
|
||||
predictions: vec![
|
||||
ClipPrediction {
|
||||
label: "indoor".to_string(),
|
||||
confidence: 0.92,
|
||||
},
|
||||
ClipPrediction {
|
||||
label: "outdoor".to_string(),
|
||||
confidence: 0.08,
|
||||
},
|
||||
],
|
||||
};
|
||||
assert_eq!(result.predictions.len(), 2);
|
||||
assert_eq!(result.predictions[0].label, "indoor");
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
pub mod asr;
|
||||
pub mod asrx;
|
||||
pub mod caption;
|
||||
pub mod cascade_vision;
|
||||
pub mod clip;
|
||||
pub mod cut;
|
||||
pub mod executor;
|
||||
pub mod face;
|
||||
@@ -16,6 +18,8 @@ pub mod yolo;
|
||||
pub use asr::{process_asr, AsrResult, AsrSegment};
|
||||
pub use asrx::{process_asrx, AsrxResult, AsrxSegment};
|
||||
pub use caption::{process_caption, CaptionResult, CaptionSummary, FrameCaption};
|
||||
pub use cascade_vision::{CascadeDetectionResult, CascadeVisionProcessor};
|
||||
pub use clip::{classify_image, classify_images, detect_objects, ClipDetectionResult, ClipImageResult, ClipPrediction};
|
||||
pub use cut::{process_cut, CutResult, CutScene};
|
||||
pub use executor::{validate_python_env, PythonExecutor, RetryConfig};
|
||||
pub use face::{process_face, Face, FaceFrame, FaceResult};
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
pub mod qwen_vl_manager;
|
||||
@@ -0,0 +1,218 @@
|
||||
use anyhow::{Context, Result};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::Mutex;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
pub struct QwenVLManager {
|
||||
port: u16,
|
||||
model_path: PathBuf,
|
||||
mmproj_path: PathBuf,
|
||||
log_file: PathBuf,
|
||||
pid_file: PathBuf,
|
||||
start_script: PathBuf,
|
||||
stop_script: PathBuf,
|
||||
last_request_time: Arc<Mutex<Instant>>,
|
||||
max_startup_time: Duration,
|
||||
}
|
||||
|
||||
impl QwenVLManager {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
port: 8086,
|
||||
model_path: PathBuf::from("/Users/accusys/models/Qwen3VL-8B-Instruct-Q8_0.gguf"),
|
||||
mmproj_path: PathBuf::from("/Users/accusys/models/mmproj-Qwen3VL-8B-Instruct-F16.gguf"),
|
||||
log_file: PathBuf::from("logs/qwen3vl_8086.log"),
|
||||
pid_file: PathBuf::from("/tmp/qwen3vl.pid"),
|
||||
start_script: PathBuf::from("scripts/start_qwen3vl.sh"),
|
||||
stop_script: PathBuf::from("scripts/stop_qwen3vl.sh"),
|
||||
last_request_time: Arc::new(Mutex::new(Instant::now())),
|
||||
max_startup_time: Duration::from_secs(60),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_port(port: u16) -> Self {
|
||||
let mut manager = Self::new();
|
||||
manager.port = port;
|
||||
manager.pid_file = PathBuf::from(format!("/tmp/qwen3vl_{}.pid", port));
|
||||
manager.log_file = PathBuf::from(format!("logs/qwen3vl_{}.log", port));
|
||||
manager
|
||||
}
|
||||
|
||||
pub fn get_port(&self) -> u16 {
|
||||
self.port
|
||||
}
|
||||
|
||||
pub async fn is_running(&self) -> Result<bool> {
|
||||
let health_url = format!("http://localhost:{}/health", self.port);
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let response = client
|
||||
.get(&health_url)
|
||||
.timeout(Duration::from_secs(5))
|
||||
.send()
|
||||
.await;
|
||||
|
||||
match response {
|
||||
Ok(resp) => {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await?;
|
||||
if status.is_success() && body.contains("\"status\":\"ok\"") {
|
||||
debug!("Qwen3-VL is running on port {}", self.port);
|
||||
return Ok(true);
|
||||
}
|
||||
debug!("Qwen3-VL health check failed: {}", status);
|
||||
Ok(false)
|
||||
}
|
||||
Err(e) => {
|
||||
debug!("Qwen3-VL not reachable: {}", e);
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn ensure_running(&self) -> Result<()> {
|
||||
if self.is_running().await? {
|
||||
debug!("Qwen3-VL already running");
|
||||
self.update_last_request_time().await;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
info!("Starting Qwen3-VL server on port {}", self.port);
|
||||
self.start_server().await?;
|
||||
self.wait_for_ready().await?;
|
||||
self.update_last_request_time().await;
|
||||
|
||||
info!("Qwen3-VL server started successfully");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn start_server(&self) -> Result<()> {
|
||||
let script_path = self.start_script.canonicalize()
|
||||
.context("Failed to resolve start script path")?;
|
||||
|
||||
debug!("Running start script: {}", script_path.display());
|
||||
|
||||
let output = Command::new("bash")
|
||||
.arg(&script_path)
|
||||
.output()
|
||||
.context("Failed to execute start script")?;
|
||||
|
||||
if !output.status.success() {
|
||||
error!("Start script failed: {}", String::from_utf8_lossy(&output.stderr));
|
||||
anyhow::bail!("Failed to start Qwen3-VL server");
|
||||
}
|
||||
|
||||
debug!("Start script output: {}", String::from_utf8_lossy(&output.stdout));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn stop_server(&self) -> Result<()> {
|
||||
let script_path = self.stop_script.canonicalize()
|
||||
.context("Failed to resolve stop script path")?;
|
||||
|
||||
debug!("Running stop script: {}", script_path.display());
|
||||
|
||||
let output = Command::new("bash")
|
||||
.arg(&script_path)
|
||||
.output()
|
||||
.context("Failed to execute stop script")?;
|
||||
|
||||
if !output.status.success() {
|
||||
warn!("Stop script returned error: {}", String::from_utf8_lossy(&output.stderr));
|
||||
}
|
||||
|
||||
debug!("Stop script output: {}", String::from_utf8_lossy(&output.stdout));
|
||||
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
|
||||
if self.is_running().await? {
|
||||
warn!("Qwen3-VL still running after stop script");
|
||||
}
|
||||
|
||||
info!("Qwen3-VL server stopped");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn wait_for_ready(&self) -> Result<()> {
|
||||
let health_url = format!("http://localhost:{}/health", self.port);
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let start_time = Instant::now();
|
||||
|
||||
while start_time.elapsed() < self.max_startup_time {
|
||||
let response = client
|
||||
.get(&health_url)
|
||||
.timeout(Duration::from_secs(2))
|
||||
.send()
|
||||
.await;
|
||||
|
||||
match response {
|
||||
Ok(resp) => {
|
||||
if resp.status().is_success() {
|
||||
let body = resp.text().await?;
|
||||
if body.contains("\"status\":\"ok\"") {
|
||||
debug!("Qwen3-VL ready after {} seconds", start_time.elapsed().as_secs());
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
}
|
||||
|
||||
error!("Qwen3-VL failed to start within {} seconds", self.max_startup_time.as_secs());
|
||||
anyhow::bail!("Qwen3-VL startup timeout");
|
||||
}
|
||||
|
||||
pub async fn update_last_request_time(&self) {
|
||||
let mut last_request = self.last_request_time.lock().await;
|
||||
*last_request = Instant::now();
|
||||
debug!("Updated last request time");
|
||||
}
|
||||
|
||||
pub async fn get_status(&self) -> Result<QwenVLStatus> {
|
||||
let is_running = self.is_running().await?;
|
||||
let last_request = self.last_request_time.lock().await.clone();
|
||||
|
||||
Ok(QwenVLStatus {
|
||||
running: is_running,
|
||||
port: self.port,
|
||||
model_path: self.model_path.display().to_string(),
|
||||
last_request: last_request.elapsed().as_secs(),
|
||||
pid_file: self.pid_file.display().to_string(),
|
||||
log_file: self.log_file.display().to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn auto_stop_if_idle(&self, idle_timeout: Duration) -> Result<()> {
|
||||
let last_request = self.last_request_time.lock().await.clone();
|
||||
|
||||
if last_request.elapsed() > idle_timeout && self.is_running().await? {
|
||||
info!("Qwen3-VL idle for {} seconds, stopping server", last_request.elapsed().as_secs());
|
||||
self.stop_server().await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct QwenVLStatus {
|
||||
pub running: bool,
|
||||
pub port: u16,
|
||||
pub model_path: String,
|
||||
pub last_request: u64,
|
||||
pub pid_file: String,
|
||||
pub log_file: String,
|
||||
}
|
||||
|
||||
impl Default for QwenVLManager {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user