feat: add Vision LLM integration (CLIP + Qwen3-VL cascade)
- Add Qwen3-VL dynamic management (start/stop/status CLI) - Add CLIP + Qwen3-VL cascade detection strategy - Add Vision CLI commands (vision start/stop/status, detect) - Add cascade_vision processor module - Add clip processor module - Add qwen_vl_manager module Changes: - scripts/start_qwen3vl.sh, stop_qwen3vl.sh: Qwen3-VL management scripts - src/core/vision/: Qwen3-VL manager module - src/core/processor/cascade_vision.rs: CLIP + Qwen3-VL cascade logic - src/core/processor/clip.rs: CLIP classification and detection - src/api/clip_api.rs: CLIP API endpoints - src/cli/vision.rs: Vision CLI implementation - src/cli/args.rs: Add Vision and Detect commands - src/main.rs: Integrate Vision CLI - src/core/mod.rs: Add vision module - src/core/processor/mod.rs: Add cascade_vision module
This commit is contained in:
@@ -50,6 +50,24 @@ pub enum Commands {
|
||||
/// UUID
|
||||
uuid: String,
|
||||
},
|
||||
/// Detect objects in an image using CLIP or Qwen3-VL
|
||||
Detect {
|
||||
/// Image path
|
||||
#[arg(short, long)]
|
||||
image: String,
|
||||
/// Objects to detect (comma separated)
|
||||
#[arg(short, long, value_delimiter = ',')]
|
||||
objects: Vec<String>,
|
||||
/// Use cascade mode (CLIP first, then Qwen3-VL for high confidence)
|
||||
#[arg(long, default_value = "false")]
|
||||
cascade: bool,
|
||||
/// CLIP confidence threshold for cascade (default: 0.7)
|
||||
#[arg(long, default_value = "0.7")]
|
||||
threshold: f32,
|
||||
},
|
||||
/// Vision LLM management
|
||||
#[command(subcommand)]
|
||||
Vision(VisionCommands),
|
||||
/// Vectorize chunks
|
||||
Vectorize {
|
||||
/// UUID (or 'all' for all)
|
||||
@@ -215,6 +233,16 @@ pub enum N8nAction {
|
||||
Verify,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
pub enum VisionCommands {
|
||||
/// Start Qwen3-VL server
|
||||
Start,
|
||||
/// Stop Qwen3-VL server
|
||||
Stop,
|
||||
/// Check Qwen3-VL status
|
||||
Status,
|
||||
}
|
||||
|
||||
/// Parse key type from string
|
||||
pub fn parse_key_type(s: Option<&str>) -> momentry_core::core::api_key::ApiKeyType {
|
||||
use momentry_core::core::api_key::ApiKeyType;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
//! CLI command definitions and argument parsing
|
||||
|
||||
pub mod args;
|
||||
pub mod vision;
|
||||
|
||||
pub use args::*;
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
use anyhow::Result;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use momentry_core::core::vision::qwen_vl_manager::QwenVLManager;
|
||||
use momentry_core::core::processor::cascade_vision::CascadeVisionProcessor;
|
||||
|
||||
pub async fn handle_vision_command(cmd: crate::cli::args::VisionCommands) -> Result<()> {
|
||||
let manager = QwenVLManager::new();
|
||||
|
||||
match cmd {
|
||||
crate::cli::args::VisionCommands::Start => {
|
||||
println!("Starting Qwen3-VL server...");
|
||||
manager.ensure_running().await?;
|
||||
println!("✅ Qwen3-VL server started successfully");
|
||||
println!("Health check: http://localhost:8086/health");
|
||||
}
|
||||
crate::cli::args::VisionCommands::Stop => {
|
||||
println!("Stopping Qwen3-VL server...");
|
||||
manager.stop_server().await?;
|
||||
println!("✅ Qwen3-VL server stopped");
|
||||
}
|
||||
crate::cli::args::VisionCommands::Status => {
|
||||
println!("Checking Qwen3-VL status...");
|
||||
let status = manager.get_status().await?;
|
||||
|
||||
println!("Status:");
|
||||
println!(" Running: {}", if status.running { "✅ Yes" } else { "❌ No" });
|
||||
println!(" Port: {}", status.port);
|
||||
println!(" Model: {}", status.model_path);
|
||||
println!(" Last request: {} seconds ago", status.last_request);
|
||||
println!(" PID file: {}", status.pid_file);
|
||||
println!(" Log file: {}", status.log_file);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn handle_detect_command(
|
||||
image: String,
|
||||
objects: Vec<String>,
|
||||
cascade: bool,
|
||||
threshold: f32,
|
||||
) -> Result<()> {
|
||||
let image_path = PathBuf::from(&image);
|
||||
|
||||
if !image_path.exists() {
|
||||
anyhow::bail!("Image file not found: {}", image);
|
||||
}
|
||||
|
||||
println!("Detecting objects in: {}", image);
|
||||
println!("Objects: {}", objects.join(", "));
|
||||
println!("Mode: {}", if cascade { "Cascade (CLIP + Qwen3-VL)" } else { "CLIP only" });
|
||||
println!("Threshold: {:.2}", threshold);
|
||||
println!();
|
||||
|
||||
if cascade {
|
||||
let processor = CascadeVisionProcessor::with_threshold(threshold);
|
||||
let result = processor.detect_objects(&image_path, &objects.iter().map(|s| s.as_str()).collect::<Vec<_>>()).await?;
|
||||
|
||||
println!("Detection Results:");
|
||||
println!(" Model used: {}", result.model_used);
|
||||
println!(" CLIP confidence: {:.3}", result.clip_confidence);
|
||||
println!(" Qwen3-VL used: {}", if result.qwenvl_used { "✅ Yes" } else { "❌ No" });
|
||||
println!(" Processing time: {} ms", result.processing_time_ms);
|
||||
println!(" Detections:");
|
||||
|
||||
for detection in &result.detections {
|
||||
println!(" - {}: {:.3}", detection.label, detection.confidence);
|
||||
}
|
||||
|
||||
if result.detections.is_empty() {
|
||||
println!(" (No objects detected)");
|
||||
}
|
||||
} else {
|
||||
use momentry_core::core::processor::clip::detect_objects;
|
||||
|
||||
let objects_str: Vec<&str> = objects.iter().map(|s| s.as_str()).collect();
|
||||
let predictions = detect_objects(&image, &objects_str, Some(threshold), None).await?;
|
||||
|
||||
println!("Detection Results:");
|
||||
println!(" Model used: CLIP");
|
||||
println!(" Detections:");
|
||||
|
||||
for prediction in &predictions {
|
||||
println!(" - {}: {:.3}", prediction.label, prediction.confidence);
|
||||
}
|
||||
|
||||
if predictions.is_empty() {
|
||||
println!(" (No objects detected above threshold {:.2})", threshold);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user