v2: add E4B-MarkBase model tests (Phase 2)

.gitea/workflows/ci.yaml

@@ -25,7 +25,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Run Unit Tests
-        run: swift test --filter "MathTest" --filter "SamplerTest" --filter "TokenizerTest"
+        run: swift test --filter "MathTest" --filter "SamplerTest" --filter "TokenizerTest" --filter "ModelTest"
 
   lint:
     needs: build

.gitignore

@@ -7,4 +7,5 @@ Package.resolved
 *.xcodeproj/
 *.xcworkspace/
 .DS_Store
-test_summary.md
+test_summary.md.runner
+.runner

Tests/01_Model/ModelTest.swift

@@ -0,0 +1,117 @@
+import XCTest
+@testable import MarkBase
+
+final class ModelTest: XCTestCase {
+
+    var engine: MarkBaseEngine!
+    var model: E4BModel!
+    let modelDir = "/Users/accusys/MarkBaseEngine/models/E4B-MarkBase"
+    let maxCtx = 256
+
+    override func setUp() {
+        super.setUp()
+        guard FileManager.default.fileExists(atPath: modelDir + "/model.safetensors") else {
+            return
+        }
+        engine = try? MarkBaseEngine(autoCompile: true)
+        model = try? E4BModel(modelDir: modelDir, engine: engine, maxContextLength: maxCtx)
+    }
+
+    // MARK: - Model Loading
+
+    func testModelLoads() throws {
+        try XCTSkipIf(model == nil, "E4B-MarkBase model not found")
+        XCTAssertNotNil(model)
+        XCTAssertEqual(model.vocabSize, 262144)
+        XCTAssertEqual(model.hiddenSize, 2560)
+        XCTAssertEqual(model.numHiddenLayers, 42)
+    }
+
+    // MARK: - Forward Pass
+
+    func testBosTokenLogits() throws {
+        try XCTSkipIf(model == nil, "E4B-MarkBase model not found")
+        let logits = try model.forward(tokenId: 2, position: 0)
+        XCTAssertEqual(logits.count, model.vocabSize)
+        let nanCount = logits.filter { $0.isNaN }.count
+        XCTAssertEqual(nanCount, 0, "No NaN values in logits")
+        XCTAssertGreaterThan(logits.max() ?? -Float.infinity, -50)
+        XCTAssertLessThan(logits.max() ?? Float.infinity, 50)
+        XCTAssertGreaterThan(logits.min() ?? -Float.infinity, -50)
+        XCTAssertLessThan(logits.min() ?? Float.infinity, 50)
+    }
+
+    func testLogitSoftcapping() throws {
+        try XCTSkipIf(model == nil, "E4B-MarkBase model not found")
+        let logits = try model.forward(tokenId: 2, position: 0)
+        let softcap: Float = 30.0
+        for logit in logits {
+            XCTAssertLessThanOrEqual(abs(logit), softcap + 1e-3,
+                                     "Logit \(logit) exceeds softcap \(softcap)")
+        }
+    }
+
+    func testMultipleTokensDeterministic() throws {
+        try XCTSkipIf(model == nil, "E4B-MarkBase model not found")
+        let tokens = [2, 1024, 2048, 4096]
+        var allLogits: [[Float]] = []
+        for (pos, tokenId) in tokens.enumerated() {
+            let logits = try model.forward(tokenId: tokenId, position: pos)
+            allLogits.append(logits)
+        }
+        XCTAssertEqual(allLogits.count, tokens.count)
+        for logits in allLogits {
+            let nanCount = logits.filter { $0.isNaN }.count
+            XCTAssertEqual(nanCount, 0, "No NaN values in logits")
+        }
+    }
+
+    func testDeterministicOutput() throws {
+        try XCTSkipIf(model == nil, "E4B-MarkBase model not found")
+        let r1 = try model.forward(tokenId: 99, position: 0)
+        let r2 = try model.forward(tokenId: 99, position: 0)
+        XCTAssertEqual(r1, r2)
+    }
+
+    func testKVCacheIncrements() throws {
+        try XCTSkipIf(model == nil, "E4B-MarkBase model not found")
+        let r0 = try model.forward(tokenId: 2, position: 0)
+        let r1 = try model.forward(tokenId: 1024, position: 1)
+        let r2 = try model.forward(tokenId: 2048, position: 2)
+        XCTAssertFalse(r0.elementsEqual(r1))
+        XCTAssertFalse(r1.elementsEqual(r2))
+        for logits in [r0, r1, r2] {
+            let nanCount = logits.filter { $0.isNaN }.count
+            XCTAssertEqual(nanCount, 0, "No NaN values in logits")
+        }
+    }
+
+    func testDifferentTokensDifferentLogits() throws {
+        try XCTSkipIf(model == nil, "E4B-MarkBase model not found")
+        let tokenA: [Float] = try model.forward(tokenId: 100, position: 0)
+        let tokenB: [Float] = try model.forward(tokenId: 200, position: 0)
+        XCTAssertNotEqual(tokenA, tokenB)
+    }
+
+    func testRandomTokenId() throws {
+        try XCTSkipIf(model == nil, "E4B-MarkBase model not found")
+        for tokenId in [0, 1, 100, 1000, 10000, 100000] {
+            let logits = try model.forward(tokenId: tokenId, position: 0)
+            let nanCount = logits.filter { $0.isNaN }.count
+            XCTAssertEqual(nanCount, 0, "No NaN for tokenId=\(tokenId)")
+            XCTAssertEqual(logits.count, model.vocabSize)
+        }
+    }
+
+    // MARK: - Batched context test
+
+    func testFullContextForward() throws {
+        try XCTSkipIf(model == nil, "E4B-MarkBase model not found")
+        let promptTokens = [2] + Array(repeating: 1024, count: 32)
+        for (pos, tokenId) in promptTokens.enumerated() {
+            let logits = try model.forward(tokenId: tokenId, position: pos)
+            let nanCount = logits.filter { $0.isNaN }.count
+            XCTAssertEqual(nanCount, 0, "NaN at position \(pos)")
+        }
+    }
+}

Tests/Manifest.json

@@ -25,6 +25,14 @@
       "model": null,
       "timeout_seconds": 30,
       "schedule": "always"
+    },
+    "01_Model/ModelTest.swift": {
+      "tier": 1,
+      "memory_gb": 6,
+      "gpu": true,
+      "model": "E4B-MarkBase",
+      "timeout_seconds": 120,
+      "schedule": "on_demand"
     }
   },
   "models": {