Initial commit: E4B-MarkBase model integration with passing tests

- E4B-MarkBase model (42 layers, 4.4GB) loaded successfully - All Phase 1-6 tests passed (model loading, forward pass, vision/audio towers, token generation, performance) - All stress tests passed (5/5 in 127.6s) - Concurrent inference - Memory stress (67.5 tok/s, 0 NaN) - Continuous generation - Batch processing - Long-running stability - Swift Metal inference engine with multimodal support
2026-06-23 18:12:35 +08:00
commit ac75faa0cc
301 changed files with 63426 additions and 0 deletions
@@ -0,0 +1,109 @@
+import Foundation
+
+// ─────────────────────────────────────────────────────────────
+// Tokenizer Protocol - Unified interface for all tokenizers
+// ─────────────────────────────────────────────────────────────
+
+/// Tokenizer protocol for text-to-token and token-to-text conversion
+public protocol Tokenizer: Sendable {
+    /// Encode text to token IDs
+    func encode(text: String) -> [Int]
+    
+    /// Decode token IDs to text
+    func decode(tokens: [Int]) -> String
+    
+    /// Vocabulary size
+    var vocabSize: Int { get }
+    
+    /// Special token IDs
+    var bosTokenId: Int { get }
+    var eosTokenId: Int { get }
+    var eosTokenIds: Set<Int> { get }
+    var padTokenId: Int { get }
+
+    /// Raw token string for a given ID (used by StreamingDecoder)
+    func rawToken(for id: Int) -> String?
+}
+
+// ─────────────────────────────────────────────────────────────
+// Special Tokens Configuration
+// ─────────────────────────────────────────────────────────────
+
+public struct SpecialTokens: Sendable {
+    public let bosToken: String
+    public let eosToken: String
+    public let padToken: String
+    public let unkToken: String
+    
+    public init(
+        bosToken: String = "<bos>",
+        eosToken: String = "<eos>",
+        padToken: String = "<pad>",
+        unkToken: String = "<unk>"
+    ) {
+        self.bosToken = bosToken
+        self.eosToken = eosToken
+        self.padToken = padToken
+        self.unkToken = unkToken
+    }
+    
+    // Gemma-4 specific tokens
+    public static let gemma4 = SpecialTokens(
+        bosToken: "<bos>",
+        eosToken: "<eos>",
+        padToken: "<pad>",
+        unkToken: "<unk>"
+    )
+}
+
+// ─────────────────────────────────────────────────────────────
+// Tokenizer Error
+// ─────────────────────────────────────────────────────────────
+
+public enum TokenizerError: Error, LocalizedError {
+    case modelNotFound(String)
+    case invalidModelFormat
+    case encodingFailed(String)
+    case decodingFailed([Int])
+    case vocabMissing(String)
+    
+    public var errorDescription: String? {
+        switch self {
+        case .modelNotFound(let path):
+            return "Tokenizer model not found at: \(path)"
+        case .invalidModelFormat:
+            return "Invalid tokenizer model format"
+        case .encodingFailed(let text):
+            return "Failed to encode text: \(text)"
+        case .decodingFailed(let tokens):
+            return "Failed to decode tokens: \(tokens)"
+        case .vocabMissing(let token):
+            return "Token not in vocabulary: \(token)"
+        }
+    }
+}
+
+// ─────────────────────────────────────────────────────────────
+// Tokenizer Factory
+// ─────────────────────────────────────────────────────────────
+
+public final class TokenizerFactory: @unchecked Sendable {
+    /// Load tokenizer from model directory
+    public static func load(modelDir: String) throws -> Tokenizer {
+        // Try tokenizer.json first (HuggingFace format)
+        let tokenizerJsonPath = modelDir + "/tokenizer.json"
+        if FileManager.default.fileExists(atPath: tokenizerJsonPath) {
+            return try BPETokenizer(jsonPath: tokenizerJsonPath)
+        }
+        
+        // Try .model file (SentencePiece format)
+        let modelPath = modelDir + "/tokenizer.model"
+        if FileManager.default.fileExists(atPath: modelPath) {
+            return try SentencePieceTokenizer(modelPath: modelPath)
+        }
+        
+        // Fallback to simple tokenizer (for testing)
+        print("Warning: No tokenizer found, using simple tokenizer")
+        return SimpleTokenizer()
+    }
+}