v2: Apply tokenizer UTF-8 fix + Engine writeFloats helper
- Tokenizer fix: collect <0xXX> bytes and decode as UTF-8 (fixes Chinese/non-ASCII character decoding) - BPETokenizer + HuggingFaceTokenizer: both updated - Engine.swift: added writeFloats() utility method - FloatWeights struct added to Layer.swift (bf16 support) - attnQBits/KBits/VBits/OBits detection added to Model.swift - bf16 layer weight support from commit 48c0347 cherry-picked
This commit is contained in:
@@ -201,6 +201,7 @@ public final class BPETokenizer: Tokenizer, @unchecked Sendable {
|
||||
}
|
||||
|
||||
private func decodeByteTokens(_ text: String) -> String {
|
||||
var bytes: [UInt8] = []
|
||||
var result = ""
|
||||
var i = text.startIndex
|
||||
|
||||
@@ -215,7 +216,7 @@ public final class BPETokenizer: Tokenizer, @unchecked Sendable {
|
||||
let hexStr = String(text[hexStart..<hexEnd])
|
||||
|
||||
if let byte = UInt8(hexStr, radix: 16) {
|
||||
result.append(Character(UnicodeScalar(byte)))
|
||||
bytes.append(byte)
|
||||
let afterHex = text.index(after: hexEnd)
|
||||
if afterHex < text.endIndex && text[afterHex] == ">" {
|
||||
i = text.index(after: afterHex)
|
||||
@@ -228,10 +229,18 @@ public final class BPETokenizer: Tokenizer, @unchecked Sendable {
|
||||
}
|
||||
}
|
||||
|
||||
if !bytes.isEmpty {
|
||||
result += String(bytes: bytes, encoding: .utf8) ?? ""
|
||||
bytes.removeAll()
|
||||
}
|
||||
result.append(text[i])
|
||||
i = text.index(after: i)
|
||||
}
|
||||
|
||||
if !bytes.isEmpty {
|
||||
result += String(bytes: bytes, encoding: .utf8) ?? ""
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user