v2: Apply tokenizer UTF-8 fix + Engine writeFloats helper
- Tokenizer fix: collect <0xXX> bytes and decode as UTF-8 (fixes Chinese/non-ASCII character decoding) - BPETokenizer + HuggingFaceTokenizer: both updated - Engine.swift: added writeFloats() utility method - FloatWeights struct added to Layer.swift (bf16 support) - attnQBits/KBits/VBits/OBits detection added to Model.swift - bf16 layer weight support from commit 48c0347 cherry-picked
This commit is contained in:
@@ -268,11 +268,11 @@ public final class HuggingFaceTokenizer: Tokenizer {
|
||||
|
||||
/// Decode <0xXX> byte tokens back to characters
|
||||
private func decodeByteTokens(_ text: String) -> String {
|
||||
var bytes: [UInt8] = []
|
||||
var result = ""
|
||||
var i = text.startIndex
|
||||
|
||||
while i < text.endIndex {
|
||||
// Check for <0xXX> pattern
|
||||
if text[i] == "<" {
|
||||
let nextIndex = text.index(after: i)
|
||||
if nextIndex < text.endIndex && text[nextIndex] == "0" {
|
||||
@@ -283,8 +283,7 @@ public final class HuggingFaceTokenizer: Tokenizer {
|
||||
let hexStr = String(text[hexStart..<hexEnd])
|
||||
|
||||
if let byte = UInt8(hexStr, radix: 16) {
|
||||
result.append(Character(UnicodeScalar(byte)))
|
||||
// Skip past the closing >
|
||||
bytes.append(byte)
|
||||
let afterHex = text.index(after: hexEnd)
|
||||
if afterHex < text.endIndex && text[afterHex] == ">" {
|
||||
i = text.index(after: afterHex)
|
||||
@@ -297,10 +296,18 @@ public final class HuggingFaceTokenizer: Tokenizer {
|
||||
}
|
||||
}
|
||||
|
||||
if !bytes.isEmpty {
|
||||
result += String(bytes: bytes, encoding: .utf8) ?? ""
|
||||
bytes.removeAll()
|
||||
}
|
||||
result.append(text[i])
|
||||
i = text.index(after: i)
|
||||
}
|
||||
|
||||
if !bytes.isEmpty {
|
||||
result += String(bytes: bytes, encoding: .utf8) ?? ""
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user