import XCTest
@testable import MarkBase

final class CoreTests: XCTestCase {
    
    func testTokenizerSpaces() throws {
        let modelDir = "/Users/accusys/MarkBaseEngine/models/E4B-MarkBase"
        let tokenizer = try TokenizerFactory.load(modelDir: modelDir)
        
        // Test space preservation
        let input = "Hello World"
        let tokens = tokenizer.encode(text: input)
        let decoded = tokenizer.decode(tokens: tokens)
        
        XCTAssertEqual(decoded.contains(" "), true, "Spaces should be preserved")
    }
    
    func testMultimodalPipeline() throws {
        let modelDir = "/Users/accusys/MarkBaseEngine/models/E4B-MarkBase"
        let engine = try MarkBaseEngine(autoCompile: true)
        let mm = try MultimodalModel(modelDir: modelDir, engine: engine, maxContextLength: 512)
        
        XCTAssertNotNil(mm.visionTowerFull, "Vision tower should load")
        XCTAssertEqual(mm.visionTowerFull?.config.numHiddenLayers, 16, "Vision tower has 16 layers")
    }
    
    func testSamplerFiltering() throws {
        let sampler = Sampler()
        
        // Create test logits
        var logits = [Float](repeating: 0, count: 262144)
        logits[258123] = 30.0  // Unused token with high logit
        logits[500] = 29.0    // Regular token
        
        // Test filtering
        let sampled = sampler.sample(logits: logits, temperature: 1.0, topK: 50, topP: 0.95, filterUnusedTokens: true)
        
        XCTAssertLessThan(sampled, 258000, "Should not sample unused tokens when filtering enabled")
    }
}