MarkBase Admin
80a78ec554
CI / build-and-test (push) Has been cancelled
- Created test infrastructure for 240 tests (57 implemented) - Programming tests: Swift, Python, C++, JavaScript, Rust (40 tests) - Non-programming tests: Text, Math, Logic, Knowledge, Vision, Audio (17 tests) - Installed Rust compiler (rustc 1.96.0) - Test framework builds successfully - Sample test executed (generation quality needs improvement) - Identified issues: greedy sampling, position indexing, code syntax
83 lines
3.8 KiB
Swift
83 lines
3.8 KiB
Swift
import XCTest
|
|
@testable import MarkBase
|
|
|
|
class CodeGenerationTest: XCTestCase {
|
|
|
|
func testSwiftCodeGeneration() throws {
|
|
print("\n═════════════════════════════════════════════════════════════════════")
|
|
print(" Swift Code Generation Test")
|
|
print("═════════════════════════════════════════════════════════════════════\n")
|
|
|
|
let modelPath = "/Users/accusys/MarkBaseEngine/models/E4B-MarkBase"
|
|
let engine = try MarkBaseEngine(autoCompile: true)
|
|
let model = try E4BModel(modelDir: modelPath, engine: engine, maxContextLength: 512)
|
|
let tokenizer = try TokenizerFactory.load(modelDir: modelPath)
|
|
|
|
let prompt = swiftPrompts[0]
|
|
print("Prompt: \(prompt.prompt)")
|
|
|
|
let tokens = tokenizer.encode(text: prompt.prompt)
|
|
var generatedTokens: [Int] = []
|
|
|
|
for i in 0..<50 {
|
|
let pos = tokens.count + i - 1
|
|
let logits = try model.forwardOptimized(tokenId: tokens.last ?? 0, position: pos)
|
|
let nextToken = logits.enumerated().max(by: { $0.element < $1.element })?.offset ?? 0
|
|
generatedTokens.append(nextToken)
|
|
|
|
if tokenizer.eosTokenIds.contains(nextToken) { break }
|
|
}
|
|
|
|
let generatedCode = tokenizer.decode(tokens: generatedTokens)
|
|
print("Generated: \(generatedCode)")
|
|
|
|
let result = compileAndRunSwift(code: generatedCode, testInput: prompt.testInput)
|
|
|
|
if result.success {
|
|
print("✅ PASS")
|
|
} else {
|
|
print("⚠ FAIL: \(result.error)")
|
|
}
|
|
|
|
model.kvCaches.forEach { cache in cache.reset() }
|
|
}
|
|
|
|
func testPythonCodeGeneration() throws {
|
|
print("\n═════════════════════════════════════════════════════════════════════")
|
|
print(" Python Code Generation Test")
|
|
print("═════════════════════════════════════════════════════════════════════\n")
|
|
|
|
let modelPath = "/Users/accusys/MarkBaseEngine/models/E4B-MarkBase"
|
|
let engine = try MarkBaseEngine(autoCompile: true)
|
|
let model = try E4BModel(modelDir: modelPath, engine: engine, maxContextLength: 512)
|
|
let tokenizer = try TokenizerFactory.load(modelDir: modelPath)
|
|
|
|
let prompt = pythonPrompts[0]
|
|
print("Prompt: \(prompt.prompt)")
|
|
|
|
let tokens = tokenizer.encode(text: prompt.prompt)
|
|
var generatedTokens: [Int] = []
|
|
|
|
for i in 0..<50 {
|
|
let pos = tokens.count + i - 1
|
|
let logits = try model.forwardOptimized(tokenId: tokens.last ?? 0, position: pos)
|
|
let nextToken = logits.enumerated().max(by: { $0.element < $1.element })?.offset ?? 0
|
|
generatedTokens.append(nextToken)
|
|
|
|
if tokenizer.eosTokenIds.contains(nextToken) { break }
|
|
}
|
|
|
|
let generatedCode = tokenizer.decode(tokens: generatedTokens)
|
|
print("Generated: \(generatedCode)")
|
|
|
|
let result = compileAndRunPython(code: generatedCode, testInput: prompt.testInput)
|
|
|
|
if result.success {
|
|
print("✅ PASS")
|
|
} else {
|
|
print("⚠ FAIL: \(result.error)")
|
|
}
|
|
|
|
model.kvCaches.forEach { cache in cache.reset() }
|
|
}
|
|
} |