diff --git a/Tests/01_Model/Model26BTest.swift b/Tests/01_Model/Model26BTest.swift new file mode 100644 index 0000000..440e13a --- /dev/null +++ b/Tests/01_Model/Model26BTest.swift @@ -0,0 +1,65 @@ +import XCTest +@testable import MarkBase + +final class Model26BTest: XCTestCase { + + var engine: MarkBaseEngine! + var model: E4BModel! + let modelDir = "/Users/accusys/MarkBaseEngine/models/gemma-4-26b-standard" + let maxCtx = 128 + + override func setUp() { + super.setUp() + guard FileManager.default.fileExists(atPath: modelDir + "/model.safetensors") else { + return + } + engine = try? MarkBaseEngine(autoCompile: true) + model = try? E4BModel(modelDir: modelDir, engine: engine, maxContextLength: maxCtx) + } + + func testModelLoads() throws { + try XCTSkipIf(model == nil, "gemma-4-26b-standard model not found") + XCTAssertNotNil(model) + XCTAssertEqual(model.hiddenSize, 2816) + XCTAssertEqual(model.numHiddenLayers, 30) + XCTAssertEqual(model.vocabSize, 262144) + } + + func testBosTokenLogitsNoNaN() throws { + try XCTSkipIf(model == nil, "gemma-4-26b-standard model not found") + let logits = try model.forward(tokenId: 2, position: 0) + XCTAssertEqual(logits.count, model.vocabSize) + let nanCount = logits.filter { $0.isNaN }.count + XCTAssertEqual(nanCount, 0, "No NaN values in logits") + } + + func testLogitsNotAllSaturated() throws { + try XCTSkipIf(model == nil, "gemma-4-26b-standard model not found") + let logits = try model.forward(tokenId: 2, position: 0) + // 26B has no softcapping, so logits should have variation + let uniqueCount = Set(logits.map { round($0 * 10) / 10 }).count + XCTAssertGreaterThan(uniqueCount, 100, "Logits should have meaningful variation") + } + + func testLogitsReasonableRange() throws { + try XCTSkipIf(model == nil, "gemma-4-26b-standard model not found") + let logits = try model.forward(tokenId: 2, position: 0) + let maxVal = logits.max() ?? 0 + let minVal = logits.min() ?? 0 + XCTAssertGreaterThan(maxVal, -100) + XCTAssertLessThan(maxVal, 10000) + XCTAssertGreaterThan(minVal, -10000) + XCTAssertLessThan(minVal, 100) + XCTAssertGreaterThan(maxVal, minVal, "Logits should have dynamic range") + } + + func testMultipleTokensProduceDifferentLogits() throws { + try XCTSkipIf(model == nil, "gemma-4-26b-standard model not found") + let tokens = [2, 100, 1000, 10000] + for (pos, tokenId) in tokens.enumerated() { + let logits = try model.forward(tokenId: tokenId, position: pos) + let nanCount = logits.filter { $0.isNaN }.count + XCTAssertEqual(nanCount, 0, "NaN for token=\(tokenId) pos=\(pos)") + } + } +} diff --git a/Tests/01_Model/Model31BTest.swift b/Tests/01_Model/Model31BTest.swift new file mode 100644 index 0000000..25bf166 --- /dev/null +++ b/Tests/01_Model/Model31BTest.swift @@ -0,0 +1,55 @@ +import XCTest +@testable import MarkBase + +final class Model31BTest: XCTestCase { + + var engine: MarkBaseEngine! + var model: E4BModel! + let modelDir = "/Users/accusys/MarkBaseEngine/models/gemma-4-31b-it-4bit" + let maxCtx = 64 + + override func setUp() { + super.setUp() + guard FileManager.default.fileExists(atPath: modelDir + "/model.safetensors.index.json") else { + return + } + engine = try? MarkBaseEngine(autoCompile: true) + model = try? E4BModel(modelDir: modelDir, engine: engine, maxContextLength: maxCtx) + } + + func testModelLoads() throws { + try XCTSkipIf(model == nil, "gemma-4-31b-it-4bit model not found") + XCTAssertNotNil(model) + XCTAssertEqual(model.hiddenSize, 5376) + XCTAssertEqual(model.numHiddenLayers, 60) + XCTAssertEqual(model.vocabSize, 262144) + } + + func testBosTokenLogitsNoNaN() throws { + try XCTSkipIf(model == nil, "gemma-4-31b-it-4bit model not found") + let logits = try model.forward(tokenId: 2, position: 0) + XCTAssertEqual(logits.count, model.vocabSize) + let nanCount = logits.filter { $0.isNaN }.count + XCTAssertEqual(nanCount, 0, "No NaN values in logits") + } + + func testLogitSoftcapping() throws { + try XCTSkipIf(model == nil, "gemma-4-31b-it-4bit model not found") + let logits = try model.forward(tokenId: 2, position: 0) + let softcap: Float = 30.0 + for logit in logits { + XCTAssertLessThanOrEqual(abs(logit), softcap + 0.1, + "Logit \(logit) exceeds softcap \(softcap)") + } + } + + func testMultipleTokensProduceDifferentLogits() throws { + try XCTSkipIf(model == nil, "gemma-4-31b-it-4bit model not found") + let tokens = [2, 100, 1000] + for (pos, tokenId) in tokens.enumerated() { + let logits = try model.forward(tokenId: tokenId, position: pos) + let nanCount = logits.filter { $0.isNaN }.count + XCTAssertEqual(nanCount, 0, "NaN for token=\(tokenId) pos=\(pos)") + } + } +} diff --git a/Tests/Manifest.json b/Tests/Manifest.json index 977338f..15a2590 100644 --- a/Tests/Manifest.json +++ b/Tests/Manifest.json @@ -31,7 +31,23 @@ "memory_gb": 6, "gpu": true, "model": "E4B-MarkBase", - "timeout_seconds": 120, + "timeout_seconds": 180, + "schedule": "on_demand" + }, + "01_Model/Model26BTest.swift": { + "tier": 1, + "memory_gb": 20, + "gpu": true, + "model": "gemma-4-26b-standard", + "timeout_seconds": 300, + "schedule": "on_demand" + }, + "01_Model/Model31BTest.swift": { + "tier": 1, + "memory_gb": 22, + "gpu": true, + "model": "gemma-4-31b-it-4bit", + "timeout_seconds": 360, "schedule": "on_demand" } }, @@ -86,13 +102,13 @@ }, "gemma-4-12b-it-4bit": { "path": "models/gemma-4-12b-it-4bit", - "format": "unknown", + "format": "markbase-4bit", "params": "12B", - "weight_gb": 0.008, - "memory_gb": 0, + "weight_gb": 10, + "memory_gb": 14, "multimodal": true, - "status": "unavailable", - "notes": "Corrupted/incomplete files (8KB only). Full 4-bit 12B needed." + "status": "available", + "notes": "Multimodal - text-only output saturates softcap (gibberish). Full model files (blobs) present." }, "12B-it-MLX-8bit": { "path": "models/12B-it-MLX-8bit",