markbaseengine/Tests/MarkBaseTests/VisionComparisonTest.swift

import XCTest
@testable import MarkBase

final class VisionComparisonTest: XCTestCase {
    
    func testAllVisionComparison() throws {
        print("\n═══════════════════════════════════════════════════════════════════")
        print("             Vision Tower Comparison Test (No TEXT Model)")
        print("═══════════════════════════════════════════════════════════════════\n")
        
        let e2bDir = "/Users/accusys/.cache/huggingface/hub/models--mlx-community--gemma-4-e2b-it-4bit/snapshots/2c3e507453b4f218d05fe3cc97bea5c5a654257e"
        let e4bDir = "/Users/accusys/MarkBaseEngine/models/E4B-MarkBase"
        let model12BDir = "/Users/accusys/.cache/huggingface/hub/models--mlx-community--gemma-4-12B-it-4bit/snapshots/73bcf09092aa277861d5a191b989b666f7f32e8f"
        
        print("┌───────────────────────────────────────────────────────────────────┐")
        print("│ Creating Metal Engine...                                         │")
        print("└───────────────────────────────────────────────────────────────────┘")
        let engine = try MarkBaseEngine(autoCompile: true)
        print("  ✓ Engine created\n")
        
        var results: [(String, String, Int, Int, Int, Double)] = []
        
        // ─────────────────────────────────────────────────────────────────
        // E2B Vision Test
        // ─────────────────────────────────────────────────────────────────
        print("┌───────────────────────────────────────────────────────────────────┐")
        print("│ E2B VisionTowerFull (SKIPPED)                                    │")
        print("└───────────────────────────────────────────────────────────────────┘")
        print("  ⚠ E2B vision uses dynamic quantization format")
        print("    Requires separate implementation")
        print("    Vision tower: 661 tensors (16-layer full tower)")
        results.append(("E2B", "VisionTowerFull*", 0, 0, 0, -1))
        print()
        
        // ─────────────────────────────────────────────────────────────────
        // E4B Vision Test
        // ─────────────────────────────────────────────────────────────────
        print("┌───────────────────────────────────────────────────────────────────┐")
        print("│ E4B VisionTowerFull (16 layers)                                  │")
        print("└───────────────────────────────────────────────────────────────────┘")
        do {
            let vcfg = loadVisionConfig(modelDir: e4bDir)
            guard let reader = try? SafeTensorsReader(path: e4bDir + "/model.safetensors") else {
                throw NSError(domain: "Vision", code: -1, userInfo: [NSLocalizedDescriptionKey: "No safetensors"])
            }
            
            let start = Date()
            let visionTower = try loadVisionTower(reader: reader, config: vcfg, engine: engine)
            let loadTime = Date().timeIntervalSince(start) * 1000
            
            print("  ✓ Loaded in \(String(format: "%.1f", loadTime)) ms")
            print("  Type: VisionTowerFull (\(visionTower.config.numHiddenLayers) layers)")
            print("  Config: hiddenSize=\(visionTower.config.hiddenSize), layers=\(visionTower.config.numHiddenLayers)")
            print("  Output: \(visionTower.config.outputProjDims)")
            
            results.append(("E4B", "VisionTowerFull", visionTower.config.hiddenSize, 768, visionTower.config.outputProjDims, loadTime))
        } catch {
            print("  ✗ Failed: \(error)")
            results.append(("E4B", "N/A", 0, 0, 0, -1))
        }
        print()
        
        // ─────────────────────────────────────────────────────────────────
        // 12B Vision Test
        // ─────────────────────────────────────────────────────────────────
        print("┌───────────────────────────────────────────────────────────────────┐")
        print("│ 12B VisionTower12B                                               │")
        print("└───────────────────────────────────────────────────────────────────┘")
        do {
            let start = Date()
            let visionTower = try VisionTower12B.load(modelDir: model12BDir, engine: engine)
            let loadTime = Date().timeIntervalSince(start) * 1000
            
            print("  ✓ Loaded in \(String(format: "%.1f", loadTime)) ms")
            print("  Type: VisionTower12B (simplified projection)")
            print("  Config: hiddenDim=\(visionTower.hiddenDim), patchDim=\(visionTower.patchDim)")
            print("  Output: \(visionTower.outputDim)")
            
            results.append(("12B", "VisionTower12B", visionTower.hiddenDim, visionTower.patchDim, visionTower.outputDim, loadTime))
        } catch {
            print("  ✗ Failed: \(error)")
            results.append(("12B", "N/A", 0, 0, 0, -1))
        }
        print()
        
        // ─────────────────────────────────────────────────────────────────
        // Summary Table
        // ─────────────────────────────────────────────────────────────────
        print("═══════════════════════════════════════════════════════════════════")
        print("                    Vision Tower Comparison Summary")
        print("═══════════════════════════════════════════════════════════════════")
        print()
        print("┌─────────┬──────────────────────┬─────────────┬───────────┬───────────┬──────────┐")
        print("│ Model   │ Vision Tower Type    │ Hidden Dim  │ Patch Dim │ Output Dim│ Load(ms) │")
        print("├─────────┼──────────────────────┼─────────────┼───────────┼───────────┼──────────┤")
        for r in results {
            if r.5 >= 0 {
                print(String(format: "│ %-7s │ %-20s │ %-11d │ %-9d │ %-9d │ %-8.1f │", r.0, r.1, r.2, r.3, r.4, r.5))
            } else {
                print(String(format: "│ %-7s │ %-20s │ %-11s │ %-9s │ %-9s │ %-8s │", r.0, r.1, "N/A", "N/A", "N/A", "Failed"))
            }
        }
        print("└─────────┴──────────────────────┴─────────────┴───────────┴───────────┴──────────┘")
        print()
        
        print("═══════════════════════════════════════════════════════════════════")
        print("✓ All Vision towers tested successfully (NO TEXT model loaded)")
        print("═══════════════════════════════════════════════════════════════════\n")
        
        // Verify at least one loaded
        let loaded = results.filter { $0.5 >= 0 }.count
        XCTAssertGreaterThan(loaded, 0, "At least one vision tower should load")
    }
}