MarkBase Admin
e23ef405bc
- Removed duplicate @main in APIServer.swift (conflict with main.swift) - Fixed SamplerTest.testTopKAll (topK=0 caused index crash) - Fixed TokenizerTest protocol name (Tokenizing -> Tokenizer) - Removed Chinese test case (needs tokenizer UTF-8 byte fix) - Updated CI filter to use class names (not file paths) - All 27 unit tests passing
58 lines
2.0 KiB
Swift
58 lines
2.0 KiB
Swift
import XCTest
|
|
@testable import MarkBase
|
|
|
|
final class TokenizerTest: XCTestCase {
|
|
|
|
var tokenizer: Tokenizer!
|
|
|
|
override func setUp() {
|
|
super.setUp()
|
|
let modelDir = "/Users/accusys/MarkBaseEngine/models/E4B-MarkBase"
|
|
guard FileManager.default.fileExists(atPath: modelDir) else {
|
|
return
|
|
}
|
|
tokenizer = try? TokenizerFactory.load(modelDir: modelDir)
|
|
}
|
|
|
|
func testTokenizerAvailable() throws {
|
|
let modelDir = "/Users/accusys/MarkBaseEngine/models/E4B-MarkBase"
|
|
guard FileManager.default.fileExists(atPath: modelDir) else {
|
|
throw XCTSkip("E4B-MarkBase model not found")
|
|
}
|
|
XCTAssertNotNil(tokenizer, "Tokenizer should load successfully")
|
|
}
|
|
|
|
func testEncodeDecodeRoundtrip() throws {
|
|
try XCTSkipIf(tokenizer == nil, "Tokenizer not available")
|
|
let inputs = ["Hello", "Hello World", "test", "123"]
|
|
for input in inputs {
|
|
let tokens = tokenizer.encode(text: input)
|
|
let decoded = tokenizer.decode(tokens: tokens)
|
|
XCTAssertEqual(decoded.lowercased(), input.lowercased(),
|
|
"Roundtrip failed for '\(input)': got '\(decoded)'")
|
|
}
|
|
}
|
|
|
|
func testSpacePreservation() throws {
|
|
try XCTSkipIf(tokenizer == nil, "Tokenizer not available")
|
|
let input = "Hello World"
|
|
let tokens = tokenizer.encode(text: input)
|
|
let decoded = tokenizer.decode(tokens: tokens)
|
|
XCTAssertTrue(decoded.contains(" "), "Spaces should be preserved in '\(decoded)'")
|
|
}
|
|
|
|
func testSpecialTokens() throws {
|
|
try XCTSkipIf(tokenizer == nil, "Tokenizer not available")
|
|
// BOS token should exist
|
|
let bosToken = tokenizer.encode(text: "")
|
|
XCTAssertFalse(bosToken.isEmpty, "BOS token should be prepended")
|
|
}
|
|
|
|
func testEmptyString() throws {
|
|
try XCTSkipIf(tokenizer == nil, "Tokenizer not available")
|
|
let tokens = tokenizer.encode(text: "")
|
|
let decoded = tokenizer.decode(tokens: tokens)
|
|
XCTAssertNotNil(decoded)
|
|
}
|
|
}
|