Initial commit: E4B-MarkBase model integration with passing tests

- E4B-MarkBase model (42 layers, 4.4GB) loaded successfully
- All Phase 1-6 tests passed (model loading, forward pass, vision/audio towers, token generation, performance)
- All stress tests passed (5/5 in 127.6s)
  - Concurrent inference
  - Memory stress (67.5 tok/s, 0 NaN)
  - Continuous generation
  - Batch processing
  - Long-running stability
- Swift Metal inference engine with multimodal support

Sources/MarkBase/Tokenizer/BPETokenizer.swift

@@ -0,0 +1,428 @@
+import Foundation
+
+// ─────────────────────────────────────────────────────────────
+// Complete BPE Tokenizer for HuggingFace tokenizer.json format
+// Supports Gemma, Llama, Qwen and other modern models
+// ─────────────────────────────────────────────────────────────
+
+public final class BPETokenizer: Tokenizer, @unchecked Sendable {
+    private let vocab: [String: Int]
+    private let reverseVocab: [Int: String]
+    private let mergeRanks: [String: Int]
+    private let addedTokens: [String: Int]
+    private let addedTokensReverse: [Int: String]
+    private let preTokenizer: PreTokenizer?
+    
+    public let vocabSize: Int
+    public let bosTokenId: Int
+    public let eosTokenId: Int
+    public let eosTokenIds: Set<Int>
+    public let padTokenId: Int
+    public let unkTokenId: Int
+    
+    public init(jsonPath: String) throws {
+        let data = try Data(contentsOf: URL(fileURLWithPath: jsonPath))
+        let json = try JSONSerialization.jsonObject(with: data) as? [String: Any]
+        
+        guard let model = json?["model"] as? [String: Any] else {
+            throw TokenizerError.invalidModelFormat
+        }
+        
+        // Load vocabulary
+        if let vocabDict = model["vocab"] as? [String: Int] {
+            self.vocab = vocabDict
+            self.reverseVocab = Dictionary(uniqueKeysWithValues: vocabDict.map { ($1, $0) })
+            self.vocabSize = vocabDict.count
+        } else {
+            throw TokenizerError.invalidModelFormat
+        }
+        
+        // Load BPE merges with rank
+        var mergeRanksDict: [String: Int] = [:]
+        if let mergePairs = model["merges"] as? [[String]] {
+            // Gemma-4 format: merges is array of pairs ["token1", "token2"]
+            for (index, pair) in mergePairs.enumerated() {
+                if pair.count == 2 {
+                    mergeRanksDict[pair[0] + pair[1]] = index
+                }
+            }
+        } else if let mergeList = model["merges"] as? [String] {
+            // Legacy format: merges is array of strings "token1 token2"
+            for (index, merge) in mergeList.enumerated() {
+                let parts = merge.split(separator: " ", maxSplits: 1)
+                if parts.count == 2 {
+                    mergeRanksDict[String(parts[0]) + String(parts[1])] = index
+                }
+            }
+        }
+        self.mergeRanks = mergeRanksDict
+        
+        // Load added tokens
+        self.addedTokens = (json?["added_tokens"] as? [[String: Any]])?.reduce(into: [String: Int]()) { dict, tokenInfo in
+            if let content = tokenInfo["content"] as? String, let id = tokenInfo["id"] as? Int {
+                dict[content] = id
+            }
+        } ?? [:]
+        self.addedTokensReverse = Dictionary(uniqueKeysWithValues: addedTokens.map { ($1, $0) })
+        
+        // Parse pre-tokenizer
+        if let preTokenizers = json?["pre_tokenizer"] as? [String: Any],
+           let type = preTokenizers["type"] as? String {
+            let behavior = preTokenizers["behavior"] as? String
+            self.preTokenizer = PreTokenizer(type: type, behavior: behavior)
+        } else {
+            self.preTokenizer = nil
+        }
+        
+        // Special tokens
+        self.bosTokenId = addedTokens["<bos>"] ?? addedTokens["<start_of_turn>"] ?? vocab["<bos>"] ?? 2
+        self.eosTokenId = addedTokens["<eos>"] ?? addedTokens["<end_of_turn>"] ?? vocab["<eos>"] ?? 1
+        var eosIds = Set<Int>([eosTokenId])
+        if let t = addedTokens["<turn|>"] ?? vocab["<turn|>"] { eosIds.insert(t) }
+        if let t = addedTokens["<|tool_response>"] ?? vocab["<|tool_response>"] { eosIds.insert(t) }
+        self.eosTokenIds = eosIds
+        self.padTokenId = addedTokens["<pad>"] ?? vocab["<pad>"] ?? 0
+        self.unkTokenId = vocab["<unk>"] ?? 3
+    }
+    
+    public func encode(text: String) -> [Int] {
+        var tokens: [Int] = [bosTokenId]
+        tokens.append(contentsOf: encodeBPE(text))
+        return tokens
+    }
+    
+    public func rawToken(for id: Int) -> String? {
+        addedTokensReverse[id] ?? reverseVocab[id]
+    }
+
+    public func decode(tokens: [Int]) -> String {
+        var text = ""
+        
+        for tokenId in tokens {
+            if tokenId == bosTokenId || eosTokenIds.contains(tokenId) || tokenId == padTokenId {
+                continue
+            }
+            
+            if let token = addedTokensReverse[tokenId] ?? reverseVocab[tokenId] {
+                text += token
+            }
+        }
+        
+        return cleanupText(text)
+    }
+    
+    // ─────────────────────────────────────────────────────────────
+    // BPE Encoding
+    // ─────────────────────────────────────────────────────────────
+    
+    private func encodeBPE(_ text: String) -> [Int] {
+        // Pre-tokenize
+        let words = preTokenizer?.pretokenize(text) ?? [text]
+        
+        var allTokens: [Int] = []
+        for word in words {
+            if word.isEmpty { continue }
+            
+            // Convert to initial tokens
+            var symbols = wordToTokens(word)
+            
+            // Apply BPE merges
+            symbols = applyMerges(symbols)
+            
+            // Convert to IDs
+            for symbol in symbols {
+                if let id = vocab[symbol] ?? addedTokens[symbol] {
+                    allTokens.append(id)
+                } else {
+                    allTokens.append(unkTokenId)
+                }
+            }
+        }
+        
+        return allTokens
+    }
+    
+    private func wordToTokens(_ word: String) -> [String] {
+        var tokens: [String] = []
+        
+        // Handle each character (not byte-by-byte)
+        for char in word {
+            if char == "▁" {
+                // Sentencepiece underscore: add as single token
+                tokens.append("▁")
+            } else {
+                let charValue = char.asciiValue ?? 0
+                if charValue >= 0x21 && charValue <= 0x7E && charValue != 0x5C && charValue != 0x60 {
+                    // Regular ASCII: add as single character
+                    tokens.append(String(char))
+                } else {
+                    // Non-ASCII or special: encode as bytes
+                    for byte in String(char).utf8 {
+                        tokens.append(String(format: "<0x%02X>", byte))
+                    }
+                }
+            }
+        }
+        
+        return tokens
+    }
+    
+    private func applyMerges(_ symbols: [String]) -> [String] {
+        var current = symbols
+        
+        while current.count > 1 {
+            var bestRank = Int.max
+            var bestPos = -1
+            var bestMerge = ""
+            
+            for i in 0..<(current.count - 1) {
+                let pair = current[i] + current[i + 1]
+                if let rank = mergeRanks[pair], rank < bestRank {
+                    bestRank = rank
+                    bestPos = i
+                    bestMerge = pair
+                }
+            }
+            
+            guard bestPos >= 0 else { break }
+            
+            current[bestPos] = bestMerge
+            current.remove(at: bestPos + 1)
+        }
+        
+        return current
+    }
+    
+    private func cleanupText(_ text: String) -> String {
+        var cleaned = text.replacingOccurrences(of: "▁", with: " ")
+        cleaned = decodeByteTokens(cleaned)
+        cleaned = cleaned.replacingOccurrences(of: "  +", with: " ", options: .regularExpression)
+        return cleaned.trimmingCharacters(in: .whitespaces)
+    }
+    
+    private func decodeByteTokens(_ text: String) -> String {
+        var result = ""
+        var i = text.startIndex
+        
+        while i < text.endIndex {
+            if text[i] == "<" {
+                let nextIndex = text.index(after: i)
+                if nextIndex < text.endIndex && text[nextIndex] == "0" {
+                    let after0 = text.index(after: nextIndex)
+                    if after0 < text.endIndex && text[after0] == "x" {
+                        let hexStart = text.index(after: after0)
+                        let hexEnd = text.index(hexStart, offsetBy: 2, limitedBy: text.endIndex) ?? text.endIndex
+                        let hexStr = String(text[hexStart..<hexEnd])
+                        
+                        if let byte = UInt8(hexStr, radix: 16) {
+                            result.append(Character(UnicodeScalar(byte)))
+                            let afterHex = text.index(after: hexEnd)
+                            if afterHex < text.endIndex && text[afterHex] == ">" {
+                                i = text.index(after: afterHex)
+                            } else {
+                                i = afterHex
+                            }
+                            continue
+                        }
+                    }
+                }
+            }
+            
+            result.append(text[i])
+            i = text.index(after: i)
+        }
+        
+        return result
+    }
+}
+
+// ─────────────────────────────────────────────────────────────
+// Streaming Decoder
+// ─────────────────────────────────────────────────────────────
+
+/// Streaming-friendly decoder that accumulates raw byte tokens
+/// and only emits text when complete UTF-8 sequences are formed.
+/// This solves the 缺字 issue where Chinese characters (3 byte tokens each)
+/// would be decoded as individual Latin-1 garbage characters.
+public struct StreamingDecoder {
+    private let tokenizer: any Tokenizer
+    private var byteBuffer: [UInt8] = []
+
+    public init(tokenizer: any Tokenizer) {
+        self.tokenizer = tokenizer
+    }
+
+    /// Consume one token, return any newly completed text.
+    public mutating func consume(tokenId: Int) -> String {
+        // Skip special tokens
+        if tokenizer.bosTokenId == tokenId || tokenizer.eosTokenIds.contains(tokenId) || tokenizer.padTokenId == tokenId {
+            return ""
+        }
+        guard let raw = tokenizer.rawToken(for: tokenId) else {
+            return ""
+        }
+        // Extract bytes from raw string (handles both <0xXX> and literal ASCII)
+        extractBytes(from: raw, into: &byteBuffer)
+        // Decode as many complete UTF-8 sequences as possible
+        return drainUTF8()
+    }
+
+    /// Flush any remaining buffered bytes as a lossy UTF-8 string.
+    public func flush() -> String {
+        if byteBuffer.isEmpty { return "" }
+        return String(decoding: byteBuffer, as: UTF8.self)
+    }
+
+    /// Reset the buffer (e.g., on generation complete).
+    public mutating func reset() {
+        byteBuffer.removeAll()
+    }
+
+    // ── Private helpers ──
+
+    private mutating func drainUTF8() -> String {
+        guard !byteBuffer.isEmpty else { return "" }
+        // Find the longest valid UTF-8 prefix
+        var validCount = 0
+        var i = 0
+        while i < byteBuffer.count {
+            let byte = byteBuffer[i]
+            if byte < 0x80 {
+                // Single byte
+                i += 1
+                validCount = i
+            } else if byte < 0xC0 {
+                // Unexpected continuation byte — stop here
+                break
+            } else if byte < 0xE0 {
+                // 2-byte sequence
+                guard i + 1 < byteBuffer.count, byteBuffer[i+1] & 0xC0 == 0x80 else { break }
+                i += 2
+                validCount = i
+            } else if byte < 0xF0 {
+                // 3-byte sequence (Chinese characters)
+                guard i + 2 < byteBuffer.count,
+                      byteBuffer[i+1] & 0xC0 == 0x80,
+                      byteBuffer[i+2] & 0xC0 == 0x80 else { break }
+                i += 3
+                validCount = i
+            } else {
+                // 4-byte sequence
+                guard i + 3 < byteBuffer.count,
+                      byteBuffer[i+1] & 0xC0 == 0x80,
+                      byteBuffer[i+2] & 0xC0 == 0x80,
+                      byteBuffer[i+3] & 0xC0 == 0x80 else { break }
+                i += 4
+                validCount = i
+            }
+        }
+        guard validCount > 0 else { return "" }
+        let data = Data(byteBuffer[0..<validCount])
+        byteBuffer.removeFirst(validCount)
+        return String(data: data, encoding: .utf8) ?? ""
+    }
+}
+
+private func extractBytes(from text: String, into buffer: inout [UInt8]) {
+    var i = text.startIndex
+    while i < text.endIndex {
+        if text[i] == "<" {
+            let nextIdx = text.index(after: i)
+            if nextIdx < text.endIndex, text[nextIdx] == "0" {
+                let a0 = text.index(after: nextIdx)
+                if a0 < text.endIndex, text[a0] == "x" {
+                    let hStart = text.index(after: a0)
+                    let hEnd = text.index(hStart, offsetBy: 2, limitedBy: text.endIndex) ?? text.endIndex
+                    let hex = String(text[hStart..<hEnd])
+                    if let byte = UInt8(hex, radix: 16) {
+                        buffer.append(byte)
+                        let after = text.index(after: hEnd)
+                        if after < text.endIndex, text[after] == ">" {
+                            i = text.index(after: after)
+                        } else {
+                            i = after
+                        }
+                        continue
+                    }
+                }
+            }
+        }
+        // Literal character — encode as UTF-8
+        let ch = text[i]
+        let utf8 = String(ch).utf8
+        buffer.append(contentsOf: utf8)
+        i = text.index(after: i)
+    }
+}
+
+// ─────────────────────────────────────────────────────────────
+// Pre-Tokenization
+// ─────────────────────────────────────────────────────────────
+
+final class PreTokenizer {
+    let type: String
+    let behavior: String?
+    
+    init(type: String, behavior: String? = nil) {
+        self.type = type
+        self.behavior = behavior
+    }
+    
+    func pretokenize(_ text: String) -> [String] {
+        switch type {
+        case "Split":
+            return splitPreTokenize(text)
+        case "ByteLevel":
+            return byteLevelPreTokenize(text)
+        default:
+            return [text]
+        }
+    }
+    
+    private func splitPreTokenize(_ text: String) -> [String] {
+        // Split on whitespace with "MergedWithPrevious" behavior
+        // This means spaces are attached to the previous word
+        // For sentencepiece, we convert spaces to ▁ prefix
+        
+        var words: [String] = []
+        var currentWord = ""
+        var i = 0
+        
+        while i < text.count {
+            let char = text[text.index(text.startIndex, offsetBy: i)]
+            
+            if char == " " {
+                // Space: merge with previous word, convert to ▁ prefix
+                if !currentWord.isEmpty {
+                    // Add ▁ prefix to current word
+                    currentWord = "▁" + currentWord
+                    words.append(currentWord)
+                    currentWord = ""
+                }
+            } else {
+                currentWord.append(char)
+            }
+            
+            i += 1
+        }
+        
+        // Add last word (without ▁ prefix if it's the first word)
+        if !currentWord.isEmpty {
+            if words.isEmpty {
+                // First word: no prefix
+                words.append(currentWord)
+            } else {
+                // Not first word: add prefix
+                words.append("▁" + currentWord)
+            }
+        }
+        
+        return words
+    }
+    
+    private func byteLevelPreTokenize(_ text: String) -> [String] {
+        // Replace space with ▁ and split
+        let modified = text.replacingOccurrences(of: " ", with: "▁")
+        return [modified]
+    }
+}

Sources/MarkBase/Tokenizer/HuggingFaceTokenizer.swift

@@ -0,0 +1,372 @@
+import Foundation
+
+// ─────────────────────────────────────────────────────────────
+// HuggingFace Tokenizer (tokenizer.json format)
+// Used by Gemma-4 and most modern models
+// ─────────────────────────────────────────────────────────────
+
+public final class HuggingFaceTokenizer: Tokenizer {
+    private let vocab: [String: Int]
+    private let reverseVocab: [Int: String]
+    private let mergeRanks: [String: Int] // BPE merge ranks (pair -> rank)
+    private let addedTokens: [String: Int]
+    private let addedTokensReverse: [Int: String]
+    
+    public let vocabSize: Int
+    public let bosTokenId: Int
+    public let eosTokenId: Int
+    public let eosTokenIds: Set<Int>
+    public let padTokenId: Int
+    public let unkTokenId: Int
+    
+    private let specialTokens: SpecialTokens
+    
+    public init(jsonPath: String) throws {
+        let data = try Data(contentsOf: URL(fileURLWithPath: jsonPath))
+        let json = try JSONSerialization.jsonObject(with: data) as? [String: Any]
+        
+        guard let model = json?["model"] as? [String: Any] else {
+            throw TokenizerError.invalidModelFormat
+        }
+        
+        // Load vocabulary
+        if let vocabDict = model["vocab"] as? [String: Int] {
+            self.vocab = vocabDict
+            self.reverseVocab = Dictionary(uniqueKeysWithValues: vocabDict.map { ($1, $0) })
+            self.vocabSize = vocabDict.count
+        } else {
+            throw TokenizerError.invalidModelFormat
+        }
+        
+        // Load BPE merges with rank (for proper merge ordering)
+        var mergeRanksDict: [String: Int] = [:]
+        if let mergeList = model["merges"] as? [String] {
+            for (index, merge) in mergeList.enumerated() {
+                let parts = merge.split(separator: " ", maxSplits: 1)
+                if parts.count == 2 {
+                    let p0 = String(parts[0])
+                    let p1 = String(parts[1])
+                    mergeRanksDict[p0 + p1] = index
+                }
+            }
+        }
+        self.mergeRanks = mergeRanksDict
+        
+        // Load added tokens (special tokens)
+        self.addedTokens = (json?["added_tokens"] as? [[String: Any]])?.reduce(into: [String: Int]()) { dict, tokenInfo in
+            if let content = tokenInfo["content"] as? String, let id = tokenInfo["id"] as? Int {
+                dict[content] = id
+            }
+        } ?? [:]
+        self.addedTokensReverse = Dictionary(uniqueKeysWithValues: addedTokens.map { ($1, $0) })
+        
+        // Debug: check mappings
+        print("\n=== Tokenizer Init Debug ===")
+        fflush(stdout)
+        print("vocabSize: \(vocabSize)")
+        fflush(stdout)
+        print("addedTokensReverse count: \(addedTokensReverse.count)")
+        fflush(stdout)
+        print("reverseVocab count: \(reverseVocab.count)")
+        fflush(stdout)
+        print("addedTokensReverse[6226]: \(addedTokensReverse[6226] ?? "nil")")
+        fflush(stdout)
+        print("reverseVocab[6226]: \(reverseVocab[6226] ?? "nil")")
+        fflush(stdout)
+        print("addedTokensReverse[262143]: \(addedTokensReverse[262143] ?? "nil")")
+        fflush(stdout)
+        print("reverseVocab[262143]: \(reverseVocab[262143] ?? "nil")")
+        fflush(stdout)
+        
+        // Check if reverseVocab is correct
+        if let vocab6226 = reverseVocab[6226] {
+            print("reverseVocab[6226] = '\(vocab6226)'")
+            fflush(stdout)
+        }
+        if let vocab262143 = reverseVocab[262143] {
+            print("reverseVocab[262143] = '\(vocab262143)'")
+            fflush(stdout)
+        }
+        // Check vocab dictionary
+        if let vocabToken = vocab["<unused6226>"] {
+            print("vocab['<unused6226>'] = \(vocabToken)")
+            fflush(stdout)
+        }
+        print("=== End Tokenizer Init Debug ===")
+        fflush(stdout)
+        
+        // Special tokens IDs
+        self.specialTokens = SpecialTokens.gemma4
+        self.bosTokenId = addedTokens[specialTokens.bosToken] ?? vocab[specialTokens.bosToken] ?? 2
+        self.eosTokenId = addedTokens[specialTokens.eosToken] ?? vocab[specialTokens.eosToken] ?? 1
+        var eosIds = Set<Int>([eosTokenId])
+        if let t = addedTokens["<turn|>"] ?? vocab["<turn|>"] { eosIds.insert(t) }
+        if let t = addedTokens["<|tool_response>"] ?? vocab["<|tool_response>"] { eosIds.insert(t) }
+        self.eosTokenIds = eosIds
+        self.padTokenId = addedTokens[specialTokens.padToken] ?? vocab[specialTokens.padToken] ?? 0
+        self.unkTokenId = vocab["<unk>"] ?? 3
+    }
+    
+    public func encode(text: String) -> [Int] {
+        var tokens: [Int] = []
+        
+        // Add BOS token
+        tokens.append(bosTokenId)
+        
+        // Apply BPE encoding
+        let bpeTokens = encodeBPE(text)
+        tokens.append(contentsOf: bpeTokens)
+        
+        // Add EOS token
+        tokens.append(eosTokenId)
+        
+        return tokens
+    }
+    
+    /// Full BPE encoding algorithm
+    private func encodeBPE(_ text: String) -> [Int] {
+        // Step 1: Pre-tokenize (split into words, handle special chars)
+        let words = pretokenize(text)
+        
+        var allTokens: [Int] = []
+        for word in words {
+            // Step 2: Convert word to byte-level tokens
+            var symbols = wordToSymbols(word)
+            
+            // Step 3: Apply BPE merges
+            symbols = applyMerges(symbols)
+            
+            // Step 4: Convert to token IDs
+            for symbol in symbols {
+                if let tokenId = vocab[symbol] {
+                    allTokens.append(tokenId)
+                } else if let tokenId = addedTokens[symbol] {
+                    allTokens.append(tokenId)
+                } else {
+                    allTokens.append(unkTokenId)
+                }
+            }
+        }
+        
+        return allTokens
+    }
+    
+    /// Pre-tokenization: split text into words and special tokens
+    private func pretokenize(_ text: String) -> [String] {
+        // Split on whitespace but keep the spaces
+        var words: [String] = []
+        var currentWord = ""
+        var inSpace = false
+        
+        for char in text {
+            if char.isWhitespace {
+                if !currentWord.isEmpty && !inSpace {
+                    words.append(currentWord)
+                    currentWord = ""
+                }
+                currentWord.append(char)
+                inSpace = true
+            } else {
+                if !currentWord.isEmpty && inSpace {
+                    words.append(currentWord)
+                    currentWord = ""
+                }
+                currentWord.append(char)
+                inSpace = false
+            }
+        }
+        
+        if !currentWord.isEmpty {
+            words.append(currentWord)
+        }
+        
+        return words
+    }
+    
+    /// Convert a word to initial byte-level tokens
+    private func wordToSymbols(_ word: String) -> [String] {
+        var symbols: [String] = []
+        let utf8 = word.utf8
+        
+        for byte in utf8 {
+            if byte >= 33 && byte <= 126 && byte != 92 && byte != 96 {
+                // Printable ASCII
+                symbols.append(String(UnicodeScalar(byte)))
+            } else {
+                // Encode as special token
+                symbols.append(String(format: "<0x%02X>", byte))
+            }
+        }
+        
+        return symbols
+    }
+    
+    /// Apply BPE merges to a list of tokens
+    private func applyMerges(_ symbols: [String]) -> [String] {
+        var current = symbols
+        
+        while current.count > 1 {
+            // Find best merge (lowest rank = highest priority)
+            var bestRank = Int.max
+            var bestPos = -1
+            var bestMerge = ""
+            
+            for i in 0..<(current.count - 1) {
+                let pair = current[i] + current[i + 1]
+                if let rank = mergeRanks[pair], rank < bestRank {
+                    bestRank = rank
+                    bestPos = i
+                    bestMerge = pair
+                }
+            }
+            
+            guard bestPos >= 0 else { break }
+            
+            // Apply merge
+            current[bestPos] = bestMerge
+            current.remove(at: bestPos + 1)
+        }
+        
+        return current
+    }
+    
+    public func rawToken(for id: Int) -> String? {
+        addedTokensReverse[id] ?? reverseVocab[id]
+    }
+    
+    public func decode(tokens: [Int]) -> String {
+        var text = ""
+        
+        for tokenId in tokens {
+            // Skip special tokens
+            if tokenId == bosTokenId || eosTokenIds.contains(tokenId) || tokenId == padTokenId {
+                continue
+            }
+            
+            // Look up in reverse vocab or added tokens
+            if let token = addedTokensReverse[tokenId] ?? reverseVocab[tokenId] {
+                text += token
+            }
+        }
+        
+        // Clean up tokenization artifacts
+        return cleanupText(text)
+    }
+    
+    private func cleanupText(_ text: String) -> String {
+        // Convert SentencePiece space marker back to space
+        var cleaned = text.replacingOccurrences(of: "▁", with: " ")
+        
+        // Decode byte-level tokens
+        cleaned = decodeByteTokens(cleaned)
+        
+        // Remove multiple spaces
+        cleaned = cleaned.replacingOccurrences(of: "  +", with: " ", options: .regularExpression)
+        
+        return cleaned.trimmingCharacters(in: .whitespaces)
+    }
+    
+    /// Decode <0xXX> byte tokens back to characters
+    private func decodeByteTokens(_ text: String) -> String {
+        var result = ""
+        var i = text.startIndex
+        
+        while i < text.endIndex {
+            // Check for <0xXX> pattern
+            if text[i] == "<" {
+                let nextIndex = text.index(after: i)
+                if nextIndex < text.endIndex && text[nextIndex] == "0" {
+                    let after0 = text.index(after: nextIndex)
+                    if after0 < text.endIndex && text[after0] == "x" {
+                        let hexStart = text.index(after: after0)
+                        let hexEnd = text.index(hexStart, offsetBy: 2, limitedBy: text.endIndex) ?? text.endIndex
+                        let hexStr = String(text[hexStart..<hexEnd])
+                        
+                        if let byte = UInt8(hexStr, radix: 16) {
+                            result.append(Character(UnicodeScalar(byte)))
+                            // Skip past the closing >
+                            let afterHex = text.index(after: hexEnd)
+                            if afterHex < text.endIndex && text[afterHex] == ">" {
+                                i = text.index(after: afterHex)
+                            } else {
+                                i = afterHex
+                            }
+                            continue
+                        }
+                    }
+                }
+            }
+            
+            result.append(text[i])
+            i = text.index(after: i)
+        }
+        
+        return result
+    }
+}
+
+// ─────────────────────────────────────────────────────────────
+// Simple Tokenizer (Fallback for testing)
+// ─────────────────────────────────────────────────────────────
+
+public final class SimpleTokenizer: Tokenizer {
+    private let vocab: [String: Int]
+    private let reverseVocab: [Int: String]
+    
+    public let vocabSize: Int
+    public let bosTokenId: Int = 0
+    public let eosTokenId: Int = 1
+    public let eosTokenIds: Set<Int> = [1]
+    public let padTokenId: Int = 2
+    
+    public init() {
+        // Create minimal vocab for testing
+        var vocabDict: [String: Int] = ["<bos>": 0, "<eos>": 1, "<pad>": 2, "<unk>": 3]
+        var reverseDict: [Int: String] = [0: "<bos>", 1: "<eos>", 2: "<pad>", 3: "<unk>"]
+        
+        // Add basic characters
+        var idx = 4
+        for char in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 " {
+            vocabDict[String(char)] = idx
+            reverseDict[idx] = String(char)
+            idx += 1
+        }
+        
+        self.vocab = vocabDict
+        self.reverseVocab = reverseDict
+        self.vocabSize = vocabDict.count
+    }
+    
+    public func encode(text: String) -> [Int] {
+        var tokens: [Int] = [bosTokenId]
+        
+        for char in text {
+            if let tokenId = vocab[String(char)] {
+                tokens.append(tokenId)
+            }
+        }
+        
+        tokens.append(eosTokenId)
+        return tokens
+    }
+    
+    public func rawToken(for id: Int) -> String? {
+        reverseVocab[id]
+    }
+    
+    public func decode(tokens: [Int]) -> String {
+        var text = ""
+        
+        for tokenId in tokens {
+            if tokenId == bosTokenId || eosTokenIds.contains(tokenId) || tokenId == padTokenId {
+                continue
+            }
+            
+            if let char = reverseVocab[tokenId] {
+                text += char
+            }
+        }
+        
+        return text
+    }
+}

Sources/MarkBase/Tokenizer/SentencePieceTokenizer.swift

@@ -0,0 +1,153 @@
+import Foundation
+
+// ─────────────────────────────────────────────────────────────
+// SentencePiece Tokenizer (tokenizer.model format)
+// Simplified implementation for Gemma models
+// ─────────────────────────────────────────────────────────────
+
+public final class SentencePieceTokenizer: Tokenizer {
+    private let vocab: [String: Int]
+    private let reverseVocab: [Int: String]
+    private let pieceToId: [String: Int]
+    
+    public let vocabSize: Int
+    public let bosTokenId: Int
+    public let eosTokenId: Int
+    public let eosTokenIds: Set<Int>
+    public let padTokenId: Int
+    
+    public init(modelPath: String) throws {
+        // Load SentencePiece model file
+        // Note: This is simplified implementation
+        // Full implementation requires protobuf parsing
+        
+        let data = try Data(contentsOf: URL(fileURLWithPath: modelPath))
+        
+        // Parse vocab from model (simplified)
+        // SentencePiece .model is protobuf format with vocab embedded
+        self.vocab = try Self.parseVocabFromModel(data)
+        self.reverseVocab = Dictionary(uniqueKeysWithValues: vocab.map { ($1, $0) })
+        self.vocabSize = vocab.count
+        
+        // Special tokens for Gemma
+        self.bosTokenId = vocab["<bos>"] ?? vocab["<start_of_turn>"] ?? 2
+        self.eosTokenId = vocab["<eos>"] ?? vocab["<end_of_turn>"] ?? 1
+        var eosIds = Set<Int>([eosTokenId])
+        if let t = vocab["<turn|>"] { eosIds.insert(t) }
+        if let t = vocab["<|tool_response>"] { eosIds.insert(t) }
+        self.eosTokenIds = eosIds
+        self.padTokenId = vocab["<pad>"] ?? 0
+        
+        self.pieceToId = vocab
+    }
+    
+    public func rawToken(for id: Int) -> String? {
+        reverseVocab[id]
+    }
+    
+    public func encode(text: String) -> [Int] {
+        var tokens: [Int] = [bosTokenId]
+        
+        // SentencePiece encoding algorithm (simplified)
+        // Full algorithm: find longest matching pieces
+        
+        var remaining = text
+        while !remaining.isEmpty {
+            // Find longest matching piece in vocab
+            var found = false
+            for length in stride(from: min(remaining.count, 20), through: 1, by: -1) {
+                let piece = String(remaining.prefix(length))
+                
+                // Check vocab (with SentencePiece space marker)
+                let spPiece = piece.hasPrefix(" ") ? "▁" + piece.dropFirst() : piece
+                
+                if let tokenId = vocab[spPiece] ?? vocab[piece] {
+                    tokens.append(tokenId)
+                    remaining = String(remaining.dropFirst(length))
+                    found = true
+                    break
+                }
+            }
+            
+            if !found {
+                // Unknown character
+                if let unkId = vocab["<unk>"] {
+                    tokens.append(unkId)
+                    remaining = String(remaining.dropFirst())
+                } else {
+                    // Skip unknown
+                    remaining = String(remaining.dropFirst())
+                }
+            }
+        }
+        
+        tokens.append(eosTokenId)
+        return tokens
+    }
+    
+    public func decode(tokens: [Int]) -> String {
+        var text = ""
+        
+        for tokenId in tokens {
+            // Skip special tokens
+            if tokenId == bosTokenId || tokenId == eosTokenId || tokenId == padTokenId {
+                continue
+            }
+            
+            // Look up piece
+            if let piece = reverseVocab[tokenId] {
+                // Convert SentencePiece space marker back to space
+                let decodedPiece = piece.replacingOccurrences(of: "▁", with: " ")
+                text += decodedPiece
+            }
+        }
+        
+        return text.trimmingCharacters(in: .whitespaces)
+    }
+    
+    // ─────────────────────────────────────────────────────────────
+    // Model Parsing (Simplified)
+    // ─────────────────────────────────────────────────────────────
+    
+    private static func parseVocabFromModel(_ data: Data) throws -> [String: Int] {
+        // Simplified vocab parsing
+        // Full implementation requires protobuf decoder
+        
+        // For prototype, try to extract vocab from text representation
+        // SentencePiece models sometimes have text vocab embedded
+        
+        var vocab: [String: Int] = [:]
+        
+        // Add common Gemma tokens
+        vocab["<bos>"] = 2
+        vocab["<eos>"] = 1
+        vocab["<pad>"] = 0
+        vocab["<unk>"] = 3
+        
+        // Try to parse vocab entries (simplified)
+        if let text = String(data: data, encoding: .utf8) {
+            let lines = text.split(separator: "\n")
+            for line in lines {
+                // Parse vocab entries: piece <space> id
+                let parts = line.split(separator: "\t")
+                if parts.count >= 2 {
+                    let piece = String(parts[0])
+                    let id = Int(parts[1]) ?? vocab.count
+                    vocab[piece] = id
+                }
+            }
+        }
+        
+        // Fallback: create character-level vocab if parsing failed
+        if vocab.count < 100 {
+            var idx = vocab.count
+            for char in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 ,.!?;:'\"-()[]{}" {
+                vocab[String(char)] = idx
+                vocab["▁" + String(char)] = idx + 100 // Space marker variant
+                idx += 1
+            }
+        }
+        
+        return vocab
+    }
+}

Sources/MarkBase/Tokenizer/Tokenizer.swift

@@ -0,0 +1,109 @@
+import Foundation
+
+// ─────────────────────────────────────────────────────────────
+// Tokenizer Protocol - Unified interface for all tokenizers
+// ─────────────────────────────────────────────────────────────
+
+/// Tokenizer protocol for text-to-token and token-to-text conversion
+public protocol Tokenizer: Sendable {
+    /// Encode text to token IDs
+    func encode(text: String) -> [Int]
+    
+    /// Decode token IDs to text
+    func decode(tokens: [Int]) -> String
+    
+    /// Vocabulary size
+    var vocabSize: Int { get }
+    
+    /// Special token IDs
+    var bosTokenId: Int { get }
+    var eosTokenId: Int { get }
+    var eosTokenIds: Set<Int> { get }
+    var padTokenId: Int { get }
+
+    /// Raw token string for a given ID (used by StreamingDecoder)
+    func rawToken(for id: Int) -> String?
+}
+
+// ─────────────────────────────────────────────────────────────
+// Special Tokens Configuration
+// ─────────────────────────────────────────────────────────────
+
+public struct SpecialTokens: Sendable {
+    public let bosToken: String
+    public let eosToken: String
+    public let padToken: String
+    public let unkToken: String
+    
+    public init(
+        bosToken: String = "<bos>",
+        eosToken: String = "<eos>",
+        padToken: String = "<pad>",
+        unkToken: String = "<unk>"
+    ) {
+        self.bosToken = bosToken
+        self.eosToken = eosToken
+        self.padToken = padToken
+        self.unkToken = unkToken
+    }
+    
+    // Gemma-4 specific tokens
+    public static let gemma4 = SpecialTokens(
+        bosToken: "<bos>",
+        eosToken: "<eos>",
+        padToken: "<pad>",
+        unkToken: "<unk>"
+    )
+}
+
+// ─────────────────────────────────────────────────────────────
+// Tokenizer Error
+// ─────────────────────────────────────────────────────────────
+
+public enum TokenizerError: Error, LocalizedError {
+    case modelNotFound(String)
+    case invalidModelFormat
+    case encodingFailed(String)
+    case decodingFailed([Int])
+    case vocabMissing(String)
+    
+    public var errorDescription: String? {
+        switch self {
+        case .modelNotFound(let path):
+            return "Tokenizer model not found at: \(path)"
+        case .invalidModelFormat:
+            return "Invalid tokenizer model format"
+        case .encodingFailed(let text):
+            return "Failed to encode text: \(text)"
+        case .decodingFailed(let tokens):
+            return "Failed to decode tokens: \(tokens)"
+        case .vocabMissing(let token):
+            return "Token not in vocabulary: \(token)"
+        }
+    }
+}
+
+// ─────────────────────────────────────────────────────────────
+// Tokenizer Factory
+// ─────────────────────────────────────────────────────────────
+
+public final class TokenizerFactory: @unchecked Sendable {
+    /// Load tokenizer from model directory
+    public static func load(modelDir: String) throws -> Tokenizer {
+        // Try tokenizer.json first (HuggingFace format)
+        let tokenizerJsonPath = modelDir + "/tokenizer.json"
+        if FileManager.default.fileExists(atPath: tokenizerJsonPath) {
+            return try BPETokenizer(jsonPath: tokenizerJsonPath)
+        }
+        
+        // Try .model file (SentencePiece format)
+        let modelPath = modelDir + "/tokenizer.model"
+        if FileManager.default.fileExists(atPath: modelPath) {
+            return try SentencePieceTokenizer(modelPath: modelPath)
+        }
+        
+        // Fallback to simple tokenizer (for testing)
+        print("Warning: No tokenizer found, using simple tokenizer")
+        return SimpleTokenizer()
+    }
+}