diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 23a1c6c..0d2be28 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v4 - name: Run Unit Tests - run: swift test --filter "MathTest" --filter "SamplerTest" --filter "TokenizerTest" + run: swift test --filter "MathTest" --filter "SamplerTest" --filter "TokenizerTest" --filter "ModelTest" lint: needs: build diff --git a/.gitignore b/.gitignore index 0d66706..ba0205b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ Package.resolved *.xcodeproj/ *.xcworkspace/ .DS_Store -test_summary.md \ No newline at end of file +test_summary.md.runner +.runner diff --git a/Tests/01_Model/ModelTest.swift b/Tests/01_Model/ModelTest.swift new file mode 100644 index 0000000..353127b --- /dev/null +++ b/Tests/01_Model/ModelTest.swift @@ -0,0 +1,117 @@ +import XCTest +@testable import MarkBase + +final class ModelTest: XCTestCase { + + var engine: MarkBaseEngine! + var model: E4BModel! + let modelDir = "/Users/accusys/MarkBaseEngine/models/E4B-MarkBase" + let maxCtx = 256 + + override func setUp() { + super.setUp() + guard FileManager.default.fileExists(atPath: modelDir + "/model.safetensors") else { + return + } + engine = try? MarkBaseEngine(autoCompile: true) + model = try? E4BModel(modelDir: modelDir, engine: engine, maxContextLength: maxCtx) + } + + // MARK: - Model Loading + + func testModelLoads() throws { + try XCTSkipIf(model == nil, "E4B-MarkBase model not found") + XCTAssertNotNil(model) + XCTAssertEqual(model.vocabSize, 262144) + XCTAssertEqual(model.hiddenSize, 2560) + XCTAssertEqual(model.numHiddenLayers, 42) + } + + // MARK: - Forward Pass + + func testBosTokenLogits() throws { + try XCTSkipIf(model == nil, "E4B-MarkBase model not found") + let logits = try model.forward(tokenId: 2, position: 0) + XCTAssertEqual(logits.count, model.vocabSize) + let nanCount = logits.filter { $0.isNaN }.count + XCTAssertEqual(nanCount, 0, "No NaN values in logits") + XCTAssertGreaterThan(logits.max() ?? -Float.infinity, -50) + XCTAssertLessThan(logits.max() ?? Float.infinity, 50) + XCTAssertGreaterThan(logits.min() ?? -Float.infinity, -50) + XCTAssertLessThan(logits.min() ?? Float.infinity, 50) + } + + func testLogitSoftcapping() throws { + try XCTSkipIf(model == nil, "E4B-MarkBase model not found") + let logits = try model.forward(tokenId: 2, position: 0) + let softcap: Float = 30.0 + for logit in logits { + XCTAssertLessThanOrEqual(abs(logit), softcap + 1e-3, + "Logit \(logit) exceeds softcap \(softcap)") + } + } + + func testMultipleTokensDeterministic() throws { + try XCTSkipIf(model == nil, "E4B-MarkBase model not found") + let tokens = [2, 1024, 2048, 4096] + var allLogits: [[Float]] = [] + for (pos, tokenId) in tokens.enumerated() { + let logits = try model.forward(tokenId: tokenId, position: pos) + allLogits.append(logits) + } + XCTAssertEqual(allLogits.count, tokens.count) + for logits in allLogits { + let nanCount = logits.filter { $0.isNaN }.count + XCTAssertEqual(nanCount, 0, "No NaN values in logits") + } + } + + func testDeterministicOutput() throws { + try XCTSkipIf(model == nil, "E4B-MarkBase model not found") + let r1 = try model.forward(tokenId: 99, position: 0) + let r2 = try model.forward(tokenId: 99, position: 0) + XCTAssertEqual(r1, r2) + } + + func testKVCacheIncrements() throws { + try XCTSkipIf(model == nil, "E4B-MarkBase model not found") + let r0 = try model.forward(tokenId: 2, position: 0) + let r1 = try model.forward(tokenId: 1024, position: 1) + let r2 = try model.forward(tokenId: 2048, position: 2) + XCTAssertFalse(r0.elementsEqual(r1)) + XCTAssertFalse(r1.elementsEqual(r2)) + for logits in [r0, r1, r2] { + let nanCount = logits.filter { $0.isNaN }.count + XCTAssertEqual(nanCount, 0, "No NaN values in logits") + } + } + + func testDifferentTokensDifferentLogits() throws { + try XCTSkipIf(model == nil, "E4B-MarkBase model not found") + let tokenA: [Float] = try model.forward(tokenId: 100, position: 0) + let tokenB: [Float] = try model.forward(tokenId: 200, position: 0) + XCTAssertNotEqual(tokenA, tokenB) + } + + func testRandomTokenId() throws { + try XCTSkipIf(model == nil, "E4B-MarkBase model not found") + for tokenId in [0, 1, 100, 1000, 10000, 100000] { + let logits = try model.forward(tokenId: tokenId, position: 0) + let nanCount = logits.filter { $0.isNaN }.count + XCTAssertEqual(nanCount, 0, "No NaN for tokenId=\(tokenId)") + XCTAssertEqual(logits.count, model.vocabSize) + } + } + + // MARK: - Batched context test + + func testFullContextForward() throws { + try XCTSkipIf(model == nil, "E4B-MarkBase model not found") + let promptTokens = [2] + Array(repeating: 1024, count: 32) + for (pos, tokenId) in promptTokens.enumerated() { + let logits = try model.forward(tokenId: tokenId, position: pos) + let nanCount = logits.filter { $0.isNaN }.count + XCTAssertEqual(nanCount, 0, "NaN at position \(pos)") + } + } +} diff --git a/Tests/Manifest.json b/Tests/Manifest.json index 671265a..977338f 100644 --- a/Tests/Manifest.json +++ b/Tests/Manifest.json @@ -25,6 +25,14 @@ "model": null, "timeout_seconds": 30, "schedule": "always" + }, + "01_Model/ModelTest.swift": { + "tier": 1, + "memory_gb": 6, + "gpu": true, + "model": "E4B-MarkBase", + "timeout_seconds": 120, + "schedule": "on_demand" } }, "models": {