v2: Apply tokenizer UTF-8 fix + Engine writeFloats helper
CI / build (push) Has been cancelled
CI / unit-tests (push) Has been cancelled
CI / lint (push) Has been cancelled

- Tokenizer fix: collect <0xXX> bytes and decode as UTF-8
  (fixes Chinese/non-ASCII character decoding)
- BPETokenizer + HuggingFaceTokenizer: both updated
- Engine.swift: added writeFloats() utility method
- FloatWeights struct added to Layer.swift (bf16 support)
- attnQBits/KBits/VBits/OBits detection added to Model.swift
- bf16 layer weight support from commit 48c0347 cherry-picked
This commit is contained in:
MarkBase Admin
2026-07-05 13:41:48 +08:00
parent 5a94501f95
commit 31427770b1
5 changed files with 40 additions and 5 deletions
+10 -1
View File
@@ -201,6 +201,7 @@ public final class BPETokenizer: Tokenizer, @unchecked Sendable {
}
private func decodeByteTokens(_ text: String) -> String {
var bytes: [UInt8] = []
var result = ""
var i = text.startIndex
@@ -215,7 +216,7 @@ public final class BPETokenizer: Tokenizer, @unchecked Sendable {
let hexStr = String(text[hexStart..<hexEnd])
if let byte = UInt8(hexStr, radix: 16) {
result.append(Character(UnicodeScalar(byte)))
bytes.append(byte)
let afterHex = text.index(after: hexEnd)
if afterHex < text.endIndex && text[afterHex] == ">" {
i = text.index(after: afterHex)
@@ -228,10 +229,18 @@ public final class BPETokenizer: Tokenizer, @unchecked Sendable {
}
}
if !bytes.isEmpty {
result += String(bytes: bytes, encoding: .utf8) ?? ""
bytes.removeAll()
}
result.append(text[i])
i = text.index(after: i)
}
if !bytes.isEmpty {
result += String(bytes: bytes, encoding: .utf8) ?? ""
}
return result
}
}
@@ -268,11 +268,11 @@ public final class HuggingFaceTokenizer: Tokenizer {
/// Decode <0xXX> byte tokens back to characters
private func decodeByteTokens(_ text: String) -> String {
var bytes: [UInt8] = []
var result = ""
var i = text.startIndex
while i < text.endIndex {
// Check for <0xXX> pattern
if text[i] == "<" {
let nextIndex = text.index(after: i)
if nextIndex < text.endIndex && text[nextIndex] == "0" {
@@ -283,8 +283,7 @@ public final class HuggingFaceTokenizer: Tokenizer {
let hexStr = String(text[hexStart..<hexEnd])
if let byte = UInt8(hexStr, radix: 16) {
result.append(Character(UnicodeScalar(byte)))
// Skip past the closing >
bytes.append(byte)
let afterHex = text.index(after: hexEnd)
if afterHex < text.endIndex && text[afterHex] == ">" {
i = text.index(after: afterHex)
@@ -297,10 +296,18 @@ public final class HuggingFaceTokenizer: Tokenizer {
}
}
if !bytes.isEmpty {
result += String(bytes: bytes, encoding: .utf8) ?? ""
bytes.removeAll()
}
result.append(text[i])
i = text.index(after: i)
}
if !bytes.isEmpty {
result += String(bytes: bytes, encoding: .utf8) ?? ""
}
return result
}
}