v2: fix multi-language support - use SentencePieceTokenizer for GemmaTokenizer

Sources/MarkBase/Tokenizer/Tokenizer.swift

@@ -90,15 +90,35 @@ public enum TokenizerError: Error, LocalizedError {
 public final class TokenizerFactory: @unchecked Sendable {
     /// Load tokenizer from model directory
     public static func load(modelDir: String) throws -> Tokenizer {
+        // Check tokenizer_config.json for tokenizer_class
+        let configPath = modelDir + "/tokenizer_config.json"
+        var tokenizerClass: String? = nil
+        if FileManager.default.fileExists(atPath: configPath),
+           let configData = try? Data(contentsOf: URL(fileURLWithPath: configPath)),
+           let config = try? JSONSerialization.jsonObject(with: configData) as? [String: Any] {
+            tokenizerClass = config["tokenizer_class"] as? String
+        }
+        
+        // For GemmaTokenizer or SentencePiece-based tokenizers, prefer .model file
+        if tokenizerClass == "GemmaTokenizer" || tokenizerClass?.contains("SentencePiece") == true {
+            let modelPath = modelDir + "/tokenizer.model"
+            if FileManager.default.fileExists(atPath: modelPath) {
+                print("  Using SentencePieceTokenizer (tokenizer_class=\(tokenizerClass ?? "unknown"))")
+                return try SentencePieceTokenizer(modelPath: modelPath)
+            }
+        }
+        
         // Try tokenizer.json first (HuggingFace format)
         let tokenizerJsonPath = modelDir + "/tokenizer.json"
         if FileManager.default.fileExists(atPath: tokenizerJsonPath) {
+            print("  Using BPETokenizer (tokenizer.json)")
             return try BPETokenizer(jsonPath: tokenizerJsonPath)
         }
         
         // Try .model file (SentencePiece format)
         let modelPath = modelDir + "/tokenizer.model"
         if FileManager.default.fileExists(atPath: modelPath) {
+            print("  Using SentencePieceTokenizer (tokenizer.model)")
             return try SentencePieceTokenizer(modelPath: modelPath)
         }