Initial commit: E4B-MarkBase model integration with passing tests
CI / build-and-test (push) Has been cancelled
CI / build-and-test (push) Has been cancelled
- E4B-MarkBase model (42 layers, 4.4GB) loaded successfully - All Phase 1-6 tests passed (model loading, forward pass, vision/audio towers, token generation, performance) - All stress tests passed (5/5 in 127.6s) - Concurrent inference - Memory stress (67.5 tok/s, 0 NaN) - Continuous generation - Batch processing - Long-running stability - Swift Metal inference engine with multimodal support
This commit is contained in:
MarkBase Admin
@@ -0,0 +1,204 @@
|
||||
import XCTest
|
||||
@testable import MarkBase
|
||||
|
||||
class E4BvsE2BComparisonTest: XCTestCase {
|
||||
|
||||
func testFullComparison() throws {
|
||||
print("\n═══════════════════════════════════════════════════════════════════")
|
||||
print(" E4B-MarkBase vs E2B Full Comparison Test")
|
||||
print("═══════════════════════════════════════════════════════════════════\n")
|
||||
|
||||
// E4B Model Path
|
||||
let e4bPath = "/Users/accusys/MarkBaseEngine/models/E4B-MarkBase"
|
||||
let e2bPath = "/Users/accusys/MarkBaseEngine/models/gemma-4-12b-it-4bit"
|
||||
|
||||
guard FileManager.default.fileExists(atPath: e4bPath),
|
||||
FileManager.default.fileExists(atPath: e2bPath) else {
|
||||
print("⚠ Models not found")
|
||||
return
|
||||
}
|
||||
|
||||
let engine = try MarkBaseEngine(autoCompile: true)
|
||||
|
||||
// ===== 1. Architecture Comparison =====
|
||||
print("1. Architecture Comparison:")
|
||||
|
||||
// E4B
|
||||
print("\n E4B-MarkBase:")
|
||||
let e4bModel = try E4BModel(modelDir: e4bPath, engine: engine, maxContextLength: 128)
|
||||
print(" TEXT Layers: \(e4bModel.numHiddenLayers)")
|
||||
print(" Hidden Size: \(e4bModel.hiddenSize)")
|
||||
print(" Vocab Size: \(e4bModel.vocabSize)")
|
||||
|
||||
// Check multimodal components
|
||||
let e4bReader = try SafeTensorsReader(path: "\(e4bPath)/model.safetensors")
|
||||
let e4bTensors = e4bReader.allTensors
|
||||
let e4bAudioTensors = e4bTensors.filter { $0.name.contains("audio_tower") }
|
||||
let e4bVisionTensors = e4bTensors.filter { $0.name.contains("vision_tower") }
|
||||
print(" Audio Tower Tensors: \(e4bAudioTensors.count)")
|
||||
print(" Vision Tower Tensors: \(e4bVisionTensors.count)")
|
||||
|
||||
// E2B (sharded)
|
||||
print("\n E2B:")
|
||||
let e2bModel = try E4BModel(modelDir: e2bPath, engine: engine, maxContextLength: 128)
|
||||
print(" TEXT Layers: \(e2bModel.numHiddenLayers)")
|
||||
print(" Hidden Size: \(e2bModel.hiddenSize)")
|
||||
print(" Vocab Size: \(e2bModel.vocabSize)")
|
||||
|
||||
// E2B has index file
|
||||
let e2bIndex = try SafeTensorsIndex(modelDir: e2bPath)
|
||||
var e2bReaders: [String: SafeTensorsReader] = [:]
|
||||
for shardFile in Set(e2bIndex.weightMap.values) {
|
||||
e2bReaders[shardFile] = try SafeTensorsReader(path: "\(e2bPath)/\(shardFile)")
|
||||
}
|
||||
let e2bTensors = e2bReaders.values.flatMap { $0.allTensors }
|
||||
let e2bPerLayerTensors = e2bTensors.filter { $0.name.contains("per_layer") }
|
||||
print(" Per-layer Tensors: \(e2bPerLayerTensors.count)")
|
||||
|
||||
// ===== 2. TEXT Performance Comparison =====
|
||||
print("\n2. TEXT Performance Comparison:")
|
||||
|
||||
// Warmup
|
||||
_ = try e4bModel.forwardOptimized(tokenId: 2, position: 0)
|
||||
_ = try e2bModel.forwardOptimized(tokenId: 2, position: 0)
|
||||
|
||||
// E4B TEXT Speed
|
||||
print("\n E4B TEXT Speed (10 tokens):")
|
||||
let e4bStart = Date()
|
||||
var token = 2
|
||||
for i in 0..<10 {
|
||||
let result = try e4bModel.forwardOptimized(tokenId: token, position: i)
|
||||
token = result.enumerated().max(by: { $0.element < $1.element })?.offset ?? 0
|
||||
}
|
||||
let e4bTime = Date().timeIntervalSince(e4bStart) * 1000 / 10.0
|
||||
print(" Latency: \(String(format: "%.1f", e4bTime))ms/token")
|
||||
print(" Throughput: \(String(format: "%.1f", 1000.0 / e4bTime)) tok/s")
|
||||
|
||||
// E2B TEXT Speed
|
||||
print("\n E2B TEXT Speed (10 tokens):")
|
||||
let e2bStart = Date()
|
||||
token = 2
|
||||
for i in 0..<10 {
|
||||
let result = try e2bModel.forwardOptimized(tokenId: token, position: i)
|
||||
token = result.enumerated().max(by: { $0.element < $1.element })?.offset ?? 0
|
||||
}
|
||||
let e2bTime = Date().timeIntervalSince(e2bStart) * 1000 / 10.0
|
||||
print(" Latency: \(String(format: "%.1f", e2bTime))ms/token")
|
||||
print(" Throughput: \(String(format: "%.1f", 1000.0 / e2bTime)) tok/s")
|
||||
|
||||
// ===== 3. NaN Stability Comparison =====
|
||||
print("\n3. NaN Stability Comparison (tokenIds 0-10):")
|
||||
|
||||
// E4B NaN Test
|
||||
var e4bNaNCount = 0
|
||||
for tokenId in 0..<10 {
|
||||
let result = try e4bModel.forwardOptimized(tokenId: tokenId, position: 0)
|
||||
e4bNaNCount += result.filter { $0.isNaN }.count
|
||||
}
|
||||
print(" E4B NaN total: \(e4bNaNCount)")
|
||||
|
||||
// E2B NaN Test
|
||||
var e2bNaNCount = 0
|
||||
for tokenId in 0..<10 {
|
||||
let result = try e2bModel.forwardOptimized(tokenId: tokenId, position: 0)
|
||||
e2bNaNCount += result.filter { $0.isNaN }.count
|
||||
}
|
||||
print(" E2B NaN total: \(e2bNaNCount)")
|
||||
|
||||
// ===== 4. Scales Quality Comparison =====
|
||||
print("\n4. Scales Quality Comparison:")
|
||||
|
||||
// E4B Scales
|
||||
let e4bScales = e4bTensors.first { $0.name.contains("embed_tokens.scales") }
|
||||
if let s = e4bScales {
|
||||
let data = try e4bReader.read(tensor: s)
|
||||
let scales = data.withUnsafeBytes { ptr in
|
||||
Array(ptr.assumingMemoryBound(to: Float.self).prefix(20))
|
||||
}
|
||||
let negCount = scales.filter { $0 < 0 }.count
|
||||
print(" E4B Scales shape: \(s.shape)")
|
||||
print(" E4B Negative scales: \(negCount)")
|
||||
}
|
||||
|
||||
// E2B Scales
|
||||
let e2bScales = e2bTensors.first { $0.name.contains("embed_tokens.scales") }
|
||||
if let s = e2bScales {
|
||||
let shardFile = e2bIndex.weightMap[s.name] ?? "model-00001-of-00002.safetensors"
|
||||
let reader = e2bReaders[shardFile]!
|
||||
let data = try reader.read(tensor: s)
|
||||
let scales = data.withUnsafeBytes { ptr in
|
||||
Array(ptr.assumingMemoryBound(to: Float.self).prefix(20))
|
||||
}
|
||||
let negCount = scales.filter { $0 < 0 }.count
|
||||
print(" E2B Scales shape: \(s.shape)")
|
||||
print(" E2B Negative scales: \(negCount)")
|
||||
}
|
||||
|
||||
// ===== 5. Long Context Comparison =====
|
||||
print("\n5. Long Context Performance (position 500-510):")
|
||||
|
||||
// E4B Long Context
|
||||
let e4bLongStart = Date()
|
||||
for i in 500..<510 {
|
||||
_ = try e4bModel.forwardOptimized(tokenId: 2, position: i)
|
||||
}
|
||||
let e4bLongTime = Date().timeIntervalSince(e4bLongStart) * 1000 / 10.0
|
||||
let e4bDegradation = ((e4bLongTime - e4bTime) / e4bTime) * 100.0
|
||||
print(" E4B Position 500: \(String(format: "%.1f", e4bLongTime))ms, degradation: \(String(format: "%.1f", e4bDegradation))%")
|
||||
|
||||
// E2B Long Context
|
||||
let e2bLongStart = Date()
|
||||
for i in 500..<510 {
|
||||
_ = try e2bModel.forwardOptimized(tokenId: 2, position: i)
|
||||
}
|
||||
let e2bLongTime = Date().timeIntervalSince(e2bLongStart) * 1000 / 10.0
|
||||
let e2bDegradation = ((e2bLongTime - e2bTime) / e2bTime) * 100.0
|
||||
print(" E2B Position 500: \(String(format: "%.1f", e2bLongTime))ms, degradation: \(String(format: "%.1f", e2bDegradation))%")
|
||||
|
||||
// ===== 6. Feature Comparison =====
|
||||
print("\n6. Feature Comparison:")
|
||||
print(" E4B Features:")
|
||||
print(" ✓ TEXT inference")
|
||||
print(" ✓ Audio processing (\(e4bAudioTensors.count) tensors)")
|
||||
print(" ✓ Vision processing (\(e4bVisionTensors.count) tensors)")
|
||||
print(" ✓ Multimodal generation")
|
||||
|
||||
print("\n E2B Features:")
|
||||
print(" ✓ TEXT inference")
|
||||
print(" ✓ Per-layer embeddings (\(e2bPerLayerTensors.count) tensors)")
|
||||
print(" ✗ No audio tower")
|
||||
print(" ✗ No vision tower")
|
||||
|
||||
// ===== 7. Summary =====
|
||||
print("\n═══════════════════════════════════════════════════════════════════")
|
||||
print(" Comparison Summary")
|
||||
print("═══════════════════════════════════════════════════════════════════\n")
|
||||
|
||||
print("TEXT Performance:")
|
||||
print(" E4B: \(String(format: "%.1f", e4bTime))ms, \(String(format: "%.1f", 1000.0/e4bTime)) tok/s")
|
||||
print(" E2B: \(String(format: "%.1f", e2bTime))ms, \(String(format: "%.1f", 1000.0/e2bTime)) tok/s")
|
||||
print(" Winner: \(e4bTime < e2bTime ? "E4B" : "E2B") (by \(String(format: "%.1f", abs(e4bTime - e2bTime)))ms)")
|
||||
|
||||
print("\nNaN Stability:")
|
||||
print(" E4B: \(e4bNaNCount) NaN")
|
||||
print(" E2B: \(e2bNaNCount) NaN")
|
||||
print(" Winner: \(e4bNaNCount == 0 && e2bNaNCount == 0 ? "Both perfect" : (e4bNaNCount < e2bNaNCount ? "E4B" : "E2B"))")
|
||||
|
||||
print("\nKV Cache Efficiency:")
|
||||
print(" E4B: \(String(format: "%.1f", e4bDegradation))% degradation")
|
||||
print(" E2B: \(String(format: "%.1f", e2bDegradation))% degradation")
|
||||
print(" Winner: \(abs(e4bDegradation) < abs(e2bDegradation) ? "E4B" : "E2B")")
|
||||
|
||||
print("\nMultimodal:")
|
||||
print(" E4B: ✓ Full Audio+Vision+Text")
|
||||
print(" E2B: ✗ TEXT only")
|
||||
print(" Winner: E4B (multimodal support)")
|
||||
|
||||
print("\nUse Case Recommendation:")
|
||||
print(" TEXT only: \(e2bTime < e4bTime ? "E2B" : "E4B")")
|
||||
print(" Multimodal: E4B")
|
||||
print(" Per-layer feature: E2B")
|
||||
|
||||
print("\n═══════════════════════════════════════════════════════════════════")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user