v2: Apply tokenizer UTF-8 fix + Engine writeFloats helper
CI / build (push) Waiting to run
CI / unit-tests (push) Blocked by required conditions
CI / lint (push) Blocked by required conditions

- Tokenizer fix: collect <0xXX> bytes and decode as UTF-8
  (fixes Chinese/non-ASCII character decoding)
- BPETokenizer + HuggingFaceTokenizer: both updated
- Engine.swift: added writeFloats() utility method
- FloatWeights struct added to Layer.swift (bf16 support)
- attnQBits/KBits/VBits/OBits detection added to Model.swift
- bf16 layer weight support from commit 48c0347 cherry-picked
This commit is contained in:
MarkBase Admin
2026-07-05 13:41:48 +08:00
parent 5a94501f95
commit 31427770b1
5 changed files with 40 additions and 5 deletions
@@ -268,11 +268,11 @@ public final class HuggingFaceTokenizer: Tokenizer {
/// Decode <0xXX> byte tokens back to characters
private func decodeByteTokens(_ text: String) -> String {
var bytes: [UInt8] = []
var result = ""
var i = text.startIndex
while i < text.endIndex {
// Check for <0xXX> pattern
if text[i] == "<" {
let nextIndex = text.index(after: i)
if nextIndex < text.endIndex && text[nextIndex] == "0" {
@@ -283,8 +283,7 @@ public final class HuggingFaceTokenizer: Tokenizer {
let hexStr = String(text[hexStart..<hexEnd])
if let byte = UInt8(hexStr, radix: 16) {
result.append(Character(UnicodeScalar(byte)))
// Skip past the closing >
bytes.append(byte)
let afterHex = text.index(after: hexEnd)
if afterHex < text.endIndex && text[afterHex] == ">" {
i = text.index(after: afterHex)
@@ -297,10 +296,18 @@ public final class HuggingFaceTokenizer: Tokenizer {
}
}
if !bytes.isEmpty {
result += String(bytes: bytes, encoding: .utf8) ?? ""
bytes.removeAll()
}
result.append(text[i])
i = text.index(after: i)
}
if !bytes.isEmpty {
result += String(bytes: bytes, encoding: .utf8) ?? ""
}
return result
}
}