feat: add Vision LLM integration (CLIP + Qwen3-VL cascade)

- Add Qwen3-VL dynamic management (start/stop/status CLI)
- Add CLIP + Qwen3-VL cascade detection strategy
- Add Vision CLI commands (vision start/stop/status, detect)
- Add cascade_vision processor module
- Add clip processor module
- Add qwen_vl_manager module

Changes:
- scripts/start_qwen3vl.sh, stop_qwen3vl.sh: Qwen3-VL management scripts
- src/core/vision/: Qwen3-VL manager module
- src/core/processor/cascade_vision.rs: CLIP + Qwen3-VL cascade logic
- src/core/processor/clip.rs: CLIP classification and detection
- src/api/clip_api.rs: CLIP API endpoints
- src/cli/vision.rs: Vision CLI implementation
- src/cli/args.rs: Add Vision and Detect commands
- src/main.rs: Integrate Vision CLI
- src/core/mod.rs: Add vision module
- src/core/processor/mod.rs: Add cascade_vision module
This commit is contained in:
Accusys
2026-06-13 16:25:52 +08:00
parent 834b0d4865
commit 17e4e15860
37 changed files with 2185 additions and 294 deletions
+28
View File
@@ -50,6 +50,24 @@ pub enum Commands {
/// UUID
uuid: String,
},
/// Detect objects in an image using CLIP or Qwen3-VL
Detect {
/// Image path
#[arg(short, long)]
image: String,
/// Objects to detect (comma separated)
#[arg(short, long, value_delimiter = ',')]
objects: Vec<String>,
/// Use cascade mode (CLIP first, then Qwen3-VL for high confidence)
#[arg(long, default_value = "false")]
cascade: bool,
/// CLIP confidence threshold for cascade (default: 0.7)
#[arg(long, default_value = "0.7")]
threshold: f32,
},
/// Vision LLM management
#[command(subcommand)]
Vision(VisionCommands),
/// Vectorize chunks
Vectorize {
/// UUID (or 'all' for all)
@@ -215,6 +233,16 @@ pub enum N8nAction {
Verify,
}
#[derive(Subcommand)]
pub enum VisionCommands {
/// Start Qwen3-VL server
Start,
/// Stop Qwen3-VL server
Stop,
/// Check Qwen3-VL status
Status,
}
/// Parse key type from string
pub fn parse_key_type(s: Option<&str>) -> momentry_core::core::api_key::ApiKeyType {
use momentry_core::core::api_key::ApiKeyType;
+1
View File
@@ -1,5 +1,6 @@
//! CLI command definitions and argument parsing
pub mod args;
pub mod vision;
pub use args::*;
+95
View File
@@ -0,0 +1,95 @@
use anyhow::Result;
use std::path::PathBuf;
use momentry_core::core::vision::qwen_vl_manager::QwenVLManager;
use momentry_core::core::processor::cascade_vision::CascadeVisionProcessor;
pub async fn handle_vision_command(cmd: crate::cli::args::VisionCommands) -> Result<()> {
let manager = QwenVLManager::new();
match cmd {
crate::cli::args::VisionCommands::Start => {
println!("Starting Qwen3-VL server...");
manager.ensure_running().await?;
println!("✅ Qwen3-VL server started successfully");
println!("Health check: http://localhost:8086/health");
}
crate::cli::args::VisionCommands::Stop => {
println!("Stopping Qwen3-VL server...");
manager.stop_server().await?;
println!("✅ Qwen3-VL server stopped");
}
crate::cli::args::VisionCommands::Status => {
println!("Checking Qwen3-VL status...");
let status = manager.get_status().await?;
println!("Status:");
println!(" Running: {}", if status.running { "✅ Yes" } else { "❌ No" });
println!(" Port: {}", status.port);
println!(" Model: {}", status.model_path);
println!(" Last request: {} seconds ago", status.last_request);
println!(" PID file: {}", status.pid_file);
println!(" Log file: {}", status.log_file);
}
}
Ok(())
}
pub async fn handle_detect_command(
image: String,
objects: Vec<String>,
cascade: bool,
threshold: f32,
) -> Result<()> {
let image_path = PathBuf::from(&image);
if !image_path.exists() {
anyhow::bail!("Image file not found: {}", image);
}
println!("Detecting objects in: {}", image);
println!("Objects: {}", objects.join(", "));
println!("Mode: {}", if cascade { "Cascade (CLIP + Qwen3-VL)" } else { "CLIP only" });
println!("Threshold: {:.2}", threshold);
println!();
if cascade {
let processor = CascadeVisionProcessor::with_threshold(threshold);
let result = processor.detect_objects(&image_path, &objects.iter().map(|s| s.as_str()).collect::<Vec<_>>()).await?;
println!("Detection Results:");
println!(" Model used: {}", result.model_used);
println!(" CLIP confidence: {:.3}", result.clip_confidence);
println!(" Qwen3-VL used: {}", if result.qwenvl_used { "✅ Yes" } else { "❌ No" });
println!(" Processing time: {} ms", result.processing_time_ms);
println!(" Detections:");
for detection in &result.detections {
println!(" - {}: {:.3}", detection.label, detection.confidence);
}
if result.detections.is_empty() {
println!(" (No objects detected)");
}
} else {
use momentry_core::core::processor::clip::detect_objects;
let objects_str: Vec<&str> = objects.iter().map(|s| s.as_str()).collect();
let predictions = detect_objects(&image, &objects_str, Some(threshold), None).await?;
println!("Detection Results:");
println!(" Model used: CLIP");
println!(" Detections:");
for prediction in &predictions {
println!(" - {}: {:.3}", prediction.label, prediction.confidence);
}
if predictions.is_empty() {
println!(" (No objects detected above threshold {:.2})", threshold);
}
}
Ok(())
}