v2: add 26B + 31B model tests (Phase 3)

Tests/01_Model/Model26BTest.swift

@@ -0,0 +1,65 @@
+import XCTest
+@testable import MarkBase
+
+final class Model26BTest: XCTestCase {
+
+    var engine: MarkBaseEngine!
+    var model: E4BModel!
+    let modelDir = "/Users/accusys/MarkBaseEngine/models/gemma-4-26b-standard"
+    let maxCtx = 128
+
+    override func setUp() {
+        super.setUp()
+        guard FileManager.default.fileExists(atPath: modelDir + "/model.safetensors") else {
+            return
+        }
+        engine = try? MarkBaseEngine(autoCompile: true)
+        model = try? E4BModel(modelDir: modelDir, engine: engine, maxContextLength: maxCtx)
+    }
+
+    func testModelLoads() throws {
+        try XCTSkipIf(model == nil, "gemma-4-26b-standard model not found")
+        XCTAssertNotNil(model)
+        XCTAssertEqual(model.hiddenSize, 2816)
+        XCTAssertEqual(model.numHiddenLayers, 30)
+        XCTAssertEqual(model.vocabSize, 262144)
+    }
+
+    func testBosTokenLogitsNoNaN() throws {
+        try XCTSkipIf(model == nil, "gemma-4-26b-standard model not found")
+        let logits = try model.forward(tokenId: 2, position: 0)
+        XCTAssertEqual(logits.count, model.vocabSize)
+        let nanCount = logits.filter { $0.isNaN }.count
+        XCTAssertEqual(nanCount, 0, "No NaN values in logits")
+    }
+
+    func testLogitsNotAllSaturated() throws {
+        try XCTSkipIf(model == nil, "gemma-4-26b-standard model not found")
+        let logits = try model.forward(tokenId: 2, position: 0)
+        // 26B has no softcapping, so logits should have variation
+        let uniqueCount = Set(logits.map { round($0 * 10) / 10 }).count
+        XCTAssertGreaterThan(uniqueCount, 100, "Logits should have meaningful variation")
+    }
+
+    func testLogitsReasonableRange() throws {
+        try XCTSkipIf(model == nil, "gemma-4-26b-standard model not found")
+        let logits = try model.forward(tokenId: 2, position: 0)
+        let maxVal = logits.max() ?? 0
+        let minVal = logits.min() ?? 0
+        XCTAssertGreaterThan(maxVal, -100)
+        XCTAssertLessThan(maxVal, 10000)
+        XCTAssertGreaterThan(minVal, -10000)
+        XCTAssertLessThan(minVal, 100)
+        XCTAssertGreaterThan(maxVal, minVal, "Logits should have dynamic range")
+    }
+
+    func testMultipleTokensProduceDifferentLogits() throws {
+        try XCTSkipIf(model == nil, "gemma-4-26b-standard model not found")
+        let tokens = [2, 100, 1000, 10000]
+        for (pos, tokenId) in tokens.enumerated() {
+            let logits = try model.forward(tokenId: tokenId, position: pos)
+            let nanCount = logits.filter { $0.isNaN }.count
+            XCTAssertEqual(nanCount, 0, "NaN for token=\(tokenId) pos=\(pos)")
+        }
+    }
+}

Tests/01_Model/Model31BTest.swift

@@ -0,0 +1,55 @@
+import XCTest
+@testable import MarkBase
+
+final class Model31BTest: XCTestCase {
+
+    var engine: MarkBaseEngine!
+    var model: E4BModel!
+    let modelDir = "/Users/accusys/MarkBaseEngine/models/gemma-4-31b-it-4bit"
+    let maxCtx = 64
+
+    override func setUp() {
+        super.setUp()
+        guard FileManager.default.fileExists(atPath: modelDir + "/model.safetensors.index.json") else {
+            return
+        }
+        engine = try? MarkBaseEngine(autoCompile: true)
+        model = try? E4BModel(modelDir: modelDir, engine: engine, maxContextLength: maxCtx)
+    }
+
+    func testModelLoads() throws {
+        try XCTSkipIf(model == nil, "gemma-4-31b-it-4bit model not found")
+        XCTAssertNotNil(model)
+        XCTAssertEqual(model.hiddenSize, 5376)
+        XCTAssertEqual(model.numHiddenLayers, 60)
+        XCTAssertEqual(model.vocabSize, 262144)
+    }
+
+    func testBosTokenLogitsNoNaN() throws {
+        try XCTSkipIf(model == nil, "gemma-4-31b-it-4bit model not found")
+        let logits = try model.forward(tokenId: 2, position: 0)
+        XCTAssertEqual(logits.count, model.vocabSize)
+        let nanCount = logits.filter { $0.isNaN }.count
+        XCTAssertEqual(nanCount, 0, "No NaN values in logits")
+    }
+
+    func testLogitSoftcapping() throws {
+        try XCTSkipIf(model == nil, "gemma-4-31b-it-4bit model not found")
+        let logits = try model.forward(tokenId: 2, position: 0)
+        let softcap: Float = 30.0
+        for logit in logits {
+            XCTAssertLessThanOrEqual(abs(logit), softcap + 0.1,
+                                     "Logit \(logit) exceeds softcap \(softcap)")
+        }
+    }
+
+    func testMultipleTokensProduceDifferentLogits() throws {
+        try XCTSkipIf(model == nil, "gemma-4-31b-it-4bit model not found")
+        let tokens = [2, 100, 1000]
+        for (pos, tokenId) in tokens.enumerated() {
+            let logits = try model.forward(tokenId: tokenId, position: pos)
+            let nanCount = logits.filter { $0.isNaN }.count
+            XCTAssertEqual(nanCount, 0, "NaN for token=\(tokenId) pos=\(pos)")
+        }
+    }
+}

Tests/Manifest.json

@@ -31,7 +31,23 @@
       "memory_gb": 6,
       "gpu": true,
       "model": "E4B-MarkBase",
-      "timeout_seconds": 120,
+      "timeout_seconds": 180,
+      "schedule": "on_demand"
+    },
+    "01_Model/Model26BTest.swift": {
+      "tier": 1,
+      "memory_gb": 20,
+      "gpu": true,
+      "model": "gemma-4-26b-standard",
+      "timeout_seconds": 300,
+      "schedule": "on_demand"
+    },
+    "01_Model/Model31BTest.swift": {
+      "tier": 1,
+      "memory_gb": 22,
+      "gpu": true,
+      "model": "gemma-4-31b-it-4bit",
+      "timeout_seconds": 360,
       "schedule": "on_demand"
     }
   },
@@ -86,13 +102,13 @@
     },
     "gemma-4-12b-it-4bit": {
       "path": "models/gemma-4-12b-it-4bit",
-      "format": "unknown",
+      "format": "markbase-4bit",
       "params": "12B",
-      "weight_gb": 0.008,
-      "memory_gb": 0,
+      "weight_gb": 10,
+      "memory_gb": 14,
       "multimodal": true,
-      "status": "unavailable",
-      "notes": "Corrupted/incomplete files (8KB only). Full 4-bit 12B needed."
+      "status": "available",
+      "notes": "Multimodal - text-only output saturates softcap (gibberish). Full model files (blobs) present."
     },
     "12B-it-MLX-8bit": {
       "path": "models/12B-it-MLX-8bit",