v2: Apply tokenizer UTF-8 fix + Engine writeFloats helper

- Tokenizer fix: collect <0xXX> bytes and decode as UTF-8 (fixes Chinese/non-ASCII character decoding) - BPETokenizer + HuggingFaceTokenizer: both updated - Engine.swift: added writeFloats() utility method - FloatWeights struct added to Layer.swift (bf16 support) - attnQBits/KBits/VBits/OBits detection added to Model.swift - bf16 layer weight support from commit 48c0347 cherry-picked
2026-07-05 13:41:48 +08:00
parent 5a94501f95
commit 31427770b1
5 changed files with 40 additions and 5 deletions
@@ -286,4 +286,11 @@ public final class MarkBaseEngine: @unchecked Sendable {
        let ptr = buffer.contents().assumingMemoryBound(to: Float.self)
        return Array(UnsafeBufferPointer(start: ptr + offset, count: count))
    }
+
+    public func writeFloats(to buffer: MTLBuffer, values: [Float], offset: Int = 0) {
+        let ptr = buffer.contents().assumingMemoryBound(to: Float.self)
+        for i in 0..<values.count {
+            ptr[i + offset] = values[i]
+        }
+    }
 }