v2: Initial clean branch with unit tests + CI/CD pipeline

- Started from ac75faa (initial E4B-MarkBase integration) - Kept Sources/ (all engine code) + Package.swift + .gitignore - Removed all ad-hoc tests, documentation, scripts, Python files - Added Tests/00_Unit/ (MathTest, TokenizerTest, SamplerTest) - Added .gitea/workflows/ci.yaml (build + unit tests + lint) - Added Scripts/check_resources.sh (memory-aware test runner) - Added Tests/Manifest.json (resource requirements for all tests) - Focus: 4-bit quantized models only
2026-07-05 13:29:25 +08:00
commit 8a66b9086a
90 changed files with 22252 additions and 0 deletions
@@ -0,0 +1,109 @@
+import Foundation
+
+// ─────────────────────────────────────────────────────────────
+// Tokenizer Protocol - Unified interface for all tokenizers
+// ─────────────────────────────────────────────────────────────
+
+/// Tokenizer protocol for text-to-token and token-to-text conversion
+public protocol Tokenizer: Sendable {
+    /// Encode text to token IDs
+    func encode(text: String) -> [Int]
+    
+    /// Decode token IDs to text
+    func decode(tokens: [Int]) -> String
+    
+    /// Vocabulary size
+    var vocabSize: Int { get }
+    
+    /// Special token IDs
+    var bosTokenId: Int { get }
+    var eosTokenId: Int { get }
+    var eosTokenIds: Set<Int> { get }
+    var padTokenId: Int { get }
+
+    /// Raw token string for a given ID (used by StreamingDecoder)
+    func rawToken(for id: Int) -> String?
+}
+
+// ─────────────────────────────────────────────────────────────
+// Special Tokens Configuration
+// ─────────────────────────────────────────────────────────────
+
+public struct SpecialTokens: Sendable {
+    public let bosToken: String
+    public let eosToken: String
+    public let padToken: String
+    public let unkToken: String
+    
+    public init(
+        bosToken: String = "<bos>",
+        eosToken: String = "<eos>",
+        padToken: String = "<pad>",
+        unkToken: String = "<unk>"
+    ) {
+        self.bosToken = bosToken
+        self.eosToken = eosToken
+        self.padToken = padToken
+        self.unkToken = unkToken
+    }
+    
+    // Gemma-4 specific tokens
+    public static let gemma4 = SpecialTokens(
+        bosToken: "<bos>",
+        eosToken: "<eos>",
+        padToken: "<pad>",
+        unkToken: "<unk>"
+    )
+}
+
+// ─────────────────────────────────────────────────────────────
+// Tokenizer Error
+// ─────────────────────────────────────────────────────────────
+
+public enum TokenizerError: Error, LocalizedError {
+    case modelNotFound(String)
+    case invalidModelFormat
+    case encodingFailed(String)
+    case decodingFailed([Int])
+    case vocabMissing(String)
+    
+    public var errorDescription: String? {
+        switch self {
+        case .modelNotFound(let path):
+            return "Tokenizer model not found at: \(path)"
+        case .invalidModelFormat:
+            return "Invalid tokenizer model format"
+        case .encodingFailed(let text):
+            return "Failed to encode text: \(text)"
+        case .decodingFailed(let tokens):
+            return "Failed to decode tokens: \(tokens)"
+        case .vocabMissing(let token):
+            return "Token not in vocabulary: \(token)"
+        }
+    }
+}
+
+// ─────────────────────────────────────────────────────────────
+// Tokenizer Factory
+// ─────────────────────────────────────────────────────────────
+
+public final class TokenizerFactory: @unchecked Sendable {
+    /// Load tokenizer from model directory
+    public static func load(modelDir: String) throws -> Tokenizer {
+        // Try tokenizer.json first (HuggingFace format)
+        let tokenizerJsonPath = modelDir + "/tokenizer.json"
+        if FileManager.default.fileExists(atPath: tokenizerJsonPath) {
+            return try BPETokenizer(jsonPath: tokenizerJsonPath)
+        }
+        
+        // Try .model file (SentencePiece format)
+        let modelPath = modelDir + "/tokenizer.model"
+        if FileManager.default.fileExists(atPath: modelPath) {
+            return try SentencePieceTokenizer(modelPath: modelPath)
+        }
+        
+        // Fallback to simple tokenizer (for testing)
+        print("Warning: No tokenizer found, using simple tokenizer")
+        return SimpleTokenizer()
+    }
+}