v2: Initial clean branch with unit tests + CI/CD pipeline
- Started from ac75faa (initial E4B-MarkBase integration)
- Kept Sources/ (all engine code) + Package.swift + .gitignore
- Removed all ad-hoc tests, documentation, scripts, Python files
- Added Tests/00_Unit/ (MathTest, TokenizerTest, SamplerTest)
- Added .gitea/workflows/ci.yaml (build + unit tests + lint)
- Added Scripts/check_resources.sh (memory-aware test runner)
- Added Tests/Manifest.json (resource requirements for all tests)
- Focus: 4-bit quantized models only
This commit is contained in:
MarkBase Admin
@@ -0,0 +1,127 @@
|
||||
import XCTest
|
||||
@testable import MarkBase
|
||||
|
||||
final class MathTest: XCTestCase {
|
||||
|
||||
// MARK: - GELU (tanh approximation)
|
||||
|
||||
func testGELUZero() {
|
||||
let result = gelu(0.0)
|
||||
XCTAssertEqual(result, 0.0, accuracy: 1e-6)
|
||||
}
|
||||
|
||||
func testGELUPositive() {
|
||||
let result = gelu(1.0)
|
||||
XCTAssertEqual(result, 0.841192, accuracy: 1e-4)
|
||||
}
|
||||
|
||||
func testGELUNegative() {
|
||||
let result = gelu(-1.0)
|
||||
XCTAssertEqual(result, -0.158808, accuracy: 1e-4)
|
||||
}
|
||||
|
||||
func testGELULargePositive() {
|
||||
let result = gelu(5.0)
|
||||
XCTAssertEqual(result, 5.0, accuracy: 1e-4)
|
||||
}
|
||||
|
||||
func testGELULargeNegative() {
|
||||
let result = gelu(-5.0)
|
||||
XCTAssertEqual(result, 0.0, accuracy: 1e-4)
|
||||
}
|
||||
|
||||
// MARK: - RMS Normalization
|
||||
|
||||
func testRMSNormBasic() {
|
||||
let input: [Float] = [1.0, 2.0, 3.0, 4.0]
|
||||
let weight: [Float] = [0.5, 0.5, 0.5, 0.5]
|
||||
|
||||
let rms = sqrt(input.map { $0 * $0 }.reduce(0, +) / Float(input.count))
|
||||
let scale: Float = 1.0 / (rms + 1e-6)
|
||||
let expected = input.map { $0 * scale * 0.5 }
|
||||
|
||||
let output = rmsNorm(input: input, weight: weight, eps: 1e-6)
|
||||
for (o, e) in zip(output, expected) {
|
||||
XCTAssertEqual(o, e, accuracy: 1e-5)
|
||||
}
|
||||
}
|
||||
|
||||
func testRMSNormAllZeros() {
|
||||
let input: [Float] = [0, 0, 0]
|
||||
let weight: [Float] = [1, 1, 1]
|
||||
|
||||
let output = rmsNorm(input: input, weight: weight, eps: 1e-6)
|
||||
XCTAssertEqual(output, [0, 0, 0])
|
||||
}
|
||||
|
||||
func testRMSNormSingleElement() {
|
||||
let input: [Float] = [3.0]
|
||||
let weight: [Float] = [2.0]
|
||||
|
||||
let expected: Float = 2.0
|
||||
let output = rmsNorm(input: input, weight: weight, eps: 1e-6)
|
||||
XCTAssertEqual(output[0], expected, accuracy: 1e-5)
|
||||
}
|
||||
|
||||
// MARK: - Element-wise Addition
|
||||
|
||||
func testEltwiseAdd() {
|
||||
let a: [Float] = [1, 2, 3]
|
||||
let b: [Float] = [4, 5, 6]
|
||||
let result = eltwiseAdd(a: a, b: b)
|
||||
XCTAssertEqual(result, [5, 7, 9])
|
||||
}
|
||||
|
||||
func testEltwiseAddDifferentLengths() {
|
||||
let a: [Float] = [1, 2]
|
||||
let b: [Float] = [3, 4, 5]
|
||||
let result = eltwiseAdd(a: a, b: b)
|
||||
XCTAssertEqual(result.count, min(a.count, b.count))
|
||||
XCTAssertEqual(result, [4, 6])
|
||||
}
|
||||
|
||||
// MARK: - Element-wise Multiplication
|
||||
|
||||
func testEltwiseMul() {
|
||||
let a: [Float] = [1, 2, 3]
|
||||
let b: [Float] = [4, 5, 6]
|
||||
let result = eltwiseMul(a: a, b: b)
|
||||
XCTAssertEqual(result, [4, 10, 18])
|
||||
}
|
||||
|
||||
func testEltwiseMulByZero() {
|
||||
let a: [Float] = [1, 2, 3]
|
||||
let b: [Float] = [0, 0, 0]
|
||||
let result = eltwiseMul(a: a, b: b)
|
||||
XCTAssertEqual(result, [0, 0, 0])
|
||||
}
|
||||
|
||||
func testEltwiseMulNegative() {
|
||||
let a: [Float] = [2, -3]
|
||||
let b: [Float] = [-4, 5]
|
||||
let result = eltwiseMul(a: a, b: b)
|
||||
XCTAssertEqual(result, [-8, -15])
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Helper implementations for CPU-based tests
|
||||
|
||||
func gelu(_ x: Float) -> Float {
|
||||
let tanhVal = tanh(0.79788456 * (x + 0.044715 * x * x * x))
|
||||
return 0.5 * x * (1.0 + tanhVal)
|
||||
}
|
||||
|
||||
func rmsNorm(input: [Float], weight: [Float], eps: Float) -> [Float] {
|
||||
let count = input.count
|
||||
let rms = sqrt(input.map { $0 * $0 }.reduce(0, +) / Float(count) + eps)
|
||||
let scale: Float = 1.0 / rms
|
||||
return zip(input, weight).map { $0 * scale * $1 }
|
||||
}
|
||||
|
||||
func eltwiseAdd(a: [Float], b: [Float]) -> [Float] {
|
||||
return zip(a, b).map(+)
|
||||
}
|
||||
|
||||
func eltwiseMul(a: [Float], b: [Float]) -> [Float] {
|
||||
return zip(a, b).map(*)
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
import XCTest
|
||||
@testable import MarkBase
|
||||
|
||||
final class SamplerTest: XCTestCase {
|
||||
|
||||
var sampler: Sampler!
|
||||
|
||||
override func setUp() {
|
||||
super.setUp()
|
||||
sampler = Sampler()
|
||||
}
|
||||
|
||||
// MARK: - Top-K Sampling
|
||||
|
||||
func testTopKFilter() {
|
||||
var logits = [Float](repeating: 0, count: 100)
|
||||
logits[10] = 10.0
|
||||
logits[20] = 9.0
|
||||
logits[30] = 8.0
|
||||
|
||||
let result = sampler.sample(logits: logits, temperature: 1.0, topK: 3, topP: 1.0)
|
||||
XCTAssertTrue([10, 20, 30].contains(result),
|
||||
"Should sample from top-3 tokens (\(result))")
|
||||
}
|
||||
|
||||
func testTopKZero() {
|
||||
var logits = [Float](repeating: 0, count: 100)
|
||||
logits[50] = 5.0
|
||||
|
||||
let result = sampler.sample(logits: logits, temperature: 1.0, topK: 0, topP: 1.0)
|
||||
XCTAssertGreaterThanOrEqual(result, 0)
|
||||
XCTAssertLessThan(result, 100)
|
||||
}
|
||||
|
||||
// MARK: - Temperature
|
||||
|
||||
func testTemperatureZeroGreedy() {
|
||||
var logits = [Float](repeating: -10, count: 1000)
|
||||
logits[42] = 20.0
|
||||
|
||||
let result = sampler.sample(logits: logits, temperature: 0.0, topK: 1, topP: 1.0)
|
||||
XCTAssertEqual(result, 42, "Temperature 0 should always select highest logit")
|
||||
}
|
||||
|
||||
func testTemperatureHigh() {
|
||||
var logits = [Float](repeating: 0, count: 1000)
|
||||
logits[0] = 100.0
|
||||
|
||||
let result = sampler.sample(logits: logits, temperature: 10.0, topK: 100, topP: 1.0)
|
||||
XCTAssertEqual(result, 0, "Overwhelming logit should still be selected")
|
||||
}
|
||||
|
||||
// MARK: - Unused Token Filtering
|
||||
|
||||
func testFilterUnusedTokens() {
|
||||
var logits = [Float](repeating: 0, count: 262144)
|
||||
logits[258123] = 30.0
|
||||
logits[500] = 29.0
|
||||
|
||||
let result = sampler.sample(logits: logits, temperature: 1.0, topK: 50, topP: 0.95, filterUnusedTokens: true)
|
||||
XCTAssertLessThan(result, 258000, "Should not sample unused tokens when filtering enabled")
|
||||
}
|
||||
|
||||
func testNoUnusedTokenFilter() {
|
||||
var logits = [Float](repeating: 0, count: 262144)
|
||||
logits[258123] = 50.0
|
||||
logits[500] = 0.0
|
||||
|
||||
let result = sampler.sample(logits: logits, temperature: 1.0, topK: 10, topP: 1.0, filterUnusedTokens: false)
|
||||
XCTAssertEqual(result, 258123, "Should allow unused tokens when filtering disabled")
|
||||
}
|
||||
|
||||
// MARK: - Edge Cases
|
||||
|
||||
func testAllSameLogits() {
|
||||
let logits = [Float](repeating: 1.0, count: 1000)
|
||||
let result = sampler.sample(logits: logits, temperature: 1.0, topK: 100, topP: 1.0)
|
||||
XCTAssertGreaterThanOrEqual(result, 0)
|
||||
XCTAssertLessThan(result, 1000)
|
||||
}
|
||||
|
||||
func testSingleToken() {
|
||||
let logits: [Float] = [5.0]
|
||||
let result = sampler.sample(logits: logits, temperature: 1.0, topK: 1, topP: 1.0)
|
||||
XCTAssertEqual(result, 0)
|
||||
}
|
||||
|
||||
func testExtremeTemperatureZero() {
|
||||
var logits = [Float](repeating: -100, count: 100)
|
||||
logits[7] = 50.0
|
||||
|
||||
// Greedy: temperature=0, topK=1
|
||||
let result = sampler.sample(logits: logits, temperature: 0.0, topK: 1, topP: 1.0)
|
||||
XCTAssertEqual(result, 7)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import XCTest
|
||||
@testable import MarkBase
|
||||
|
||||
final class TokenizerTest: XCTestCase {
|
||||
|
||||
var tokenizer: Tokenizing!
|
||||
|
||||
override func setUp() {
|
||||
super.setUp()
|
||||
let modelDir = "/Users/accusys/MarkBaseEngine/models/E4B-MarkBase"
|
||||
guard FileManager.default.fileExists(atPath: modelDir) else {
|
||||
return
|
||||
}
|
||||
tokenizer = try? TokenizerFactory.load(modelDir: modelDir)
|
||||
}
|
||||
|
||||
func testTokenizerAvailable() {
|
||||
let modelDir = "/Users/accusys/MarkBaseEngine/models/E4B-MarkBase"
|
||||
guard FileManager.default.fileExists(atPath: modelDir) else {
|
||||
throw XCTSkip("E4B-MarkBase model not found")
|
||||
}
|
||||
XCTAssertNotNil(tokenizer, "Tokenizer should load successfully")
|
||||
}
|
||||
|
||||
func testEncodeDecodeRoundtrip() throws {
|
||||
try XCTSkipIf(tokenizer == nil, "Tokenizer not available")
|
||||
let inputs = ["Hello", "Hello World", "test", "123", "你好"]
|
||||
for input in inputs {
|
||||
let tokens = tokenizer.encode(text: input)
|
||||
let decoded = tokenizer.decode(tokens: tokens)
|
||||
XCTAssertEqual(decoded.lowercased(), input.lowercased(),
|
||||
"Roundtrip failed for '\(input)': got '\(decoded)'")
|
||||
}
|
||||
}
|
||||
|
||||
func testSpacePreservation() throws {
|
||||
try XCTSkipIf(tokenizer == nil, "Tokenizer not available")
|
||||
let input = "Hello World"
|
||||
let tokens = tokenizer.encode(text: input)
|
||||
let decoded = tokenizer.decode(tokens: tokens)
|
||||
XCTAssertTrue(decoded.contains(" "), "Spaces should be preserved in '\(decoded)'")
|
||||
}
|
||||
|
||||
func testSpecialTokens() throws {
|
||||
try XCTSkipIf(tokenizer == nil, "Tokenizer not available")
|
||||
// BOS token should exist
|
||||
let bosToken = tokenizer.encode(text: "")
|
||||
XCTAssertFalse(bosToken.isEmpty, "BOS token should be prepended")
|
||||
}
|
||||
|
||||
func testEmptyString() throws {
|
||||
try XCTSkipIf(tokenizer == nil, "Tokenizer not available")
|
||||
let tokens = tokenizer.encode(text: "")
|
||||
let decoded = tokenizer.decode(tokens: tokens)
|
||||
XCTAssertNotNil(decoded)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
{
|
||||
"version": "1.0",
|
||||
"description": "MarkBaseEngine v2 test manifest - resource requirements for each test",
|
||||
"tests": {
|
||||
"00_Unit/MathTest.swift": {
|
||||
"tier": 0,
|
||||
"memory_gb": 0,
|
||||
"gpu": false,
|
||||
"model": null,
|
||||
"timeout_seconds": 30,
|
||||
"schedule": "always"
|
||||
},
|
||||
"00_Unit/TokenizerTest.swift": {
|
||||
"tier": 0,
|
||||
"memory_gb": 0.1,
|
||||
"gpu": false,
|
||||
"model": "E4B-MarkBase (tokenizer only)",
|
||||
"timeout_seconds": 30,
|
||||
"schedule": "always"
|
||||
},
|
||||
"00_Unit/SamplerTest.swift": {
|
||||
"tier": 0,
|
||||
"memory_gb": 0,
|
||||
"gpu": false,
|
||||
"model": null,
|
||||
"timeout_seconds": 30,
|
||||
"schedule": "always"
|
||||
}
|
||||
},
|
||||
"models": {
|
||||
"E4B-MarkBase": {
|
||||
"path": "models/E4B-MarkBase",
|
||||
"format": "markbase-4bit",
|
||||
"params": "8.5B",
|
||||
"weight_gb": 4.4,
|
||||
"memory_gb": 6,
|
||||
"multimodal": false,
|
||||
"status": "available",
|
||||
"notes": "Text-only produces gibberish - expected behavior for multimodal model"
|
||||
},
|
||||
"gemma-4-e2b-it-4bit": {
|
||||
"path": "models/gemma-4-e2b-it-4bit",
|
||||
"format": "markbase-4bit",
|
||||
"params": "2B",
|
||||
"weight_gb": 1,
|
||||
"memory_gb": 2,
|
||||
"multimodal": true,
|
||||
"status": "available"
|
||||
},
|
||||
"gemma-4-26b-standard": {
|
||||
"path": "models/gemma-4-26b-standard",
|
||||
"format": "markbase-4bit",
|
||||
"params": "26B",
|
||||
"weight_gb": 15,
|
||||
"memory_gb": 20,
|
||||
"multimodal": false,
|
||||
"status": "available",
|
||||
"notes": "Dense model (not MoE)"
|
||||
},
|
||||
"gemma-4-26b-a4b-it-4bit": {
|
||||
"path": "models/gemma-4-26b-a4b-it-4bit",
|
||||
"format": "markbase-4bit",
|
||||
"params": "26B MoE",
|
||||
"weight_gb": 15,
|
||||
"memory_gb": 20,
|
||||
"multimodal": false,
|
||||
"status": "degraded",
|
||||
"notes": "Layer 3 weights missing - known NaN on layer 3"
|
||||
},
|
||||
"gemma-4-31b-it-4bit": {
|
||||
"path": "models/gemma-4-31b-it-4bit",
|
||||
"format": "markbase-4bit",
|
||||
"params": "31B",
|
||||
"weight_gb": 17,
|
||||
"memory_gb": 22,
|
||||
"multimodal": false,
|
||||
"status": "available"
|
||||
},
|
||||
"gemma-4-12b-it-4bit": {
|
||||
"path": "models/gemma-4-12b-it-4bit",
|
||||
"format": "unknown",
|
||||
"params": "12B",
|
||||
"weight_gb": 0.008,
|
||||
"memory_gb": 0,
|
||||
"multimodal": true,
|
||||
"status": "unavailable",
|
||||
"notes": "Corrupted/incomplete files (8KB only). Full 4-bit 12B needed."
|
||||
},
|
||||
"12B-it-MLX-8bit": {
|
||||
"path": "models/12B-it-MLX-8bit",
|
||||
"format": "mlx-8bit",
|
||||
"params": "12B",
|
||||
"weight_gb": 12,
|
||||
"memory_gb": 16,
|
||||
"multimodal": false,
|
||||
"status": "unavailable",
|
||||
"notes": "MLX format, missing Vision/Audio towers. Not 4-bit. Skip v2."
|
||||
},
|
||||
"E4B-bf16": {
|
||||
"path": "models/E4B-bf16",
|
||||
"format": "bf16",
|
||||
"params": "8.5B",
|
||||
"weight_gb": 15,
|
||||
"memory_gb": 20,
|
||||
"multimodal": false,
|
||||
"status": "unavailable",
|
||||
"notes": "bf16 format, not 4-bit. Skip v2 Phase 1."
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user