Add comprehensive code generation test framework

- Created test infrastructure for 240 tests (57 implemented) - Programming tests: Swift, Python, C++, JavaScript, Rust (40 tests) - Non-programming tests: Text, Math, Logic, Knowledge, Vision, Audio (17 tests) - Installed Rust compiler (rustc 1.96.0) - Test framework builds successfully - Sample test executed (generation quality needs improvement) - Identified issues: greedy sampling, position indexing, code syntax
2026-06-23 19:36:26 +08:00
parent ac75faa0cc
commit 80a78ec554
7 changed files with 620 additions and 0 deletions
@@ -0,0 +1,83 @@
+import XCTest
+@testable import MarkBase
+
+class CodeGenerationTest: XCTestCase {
+    
+    func testSwiftCodeGeneration() throws {
+        print("\n═════════════════════════════════════════════════════════════════════")
+        print("  Swift Code Generation Test")
+        print("═════════════════════════════════════════════════════════════════════\n")
+        
+        let modelPath = "/Users/accusys/MarkBaseEngine/models/E4B-MarkBase"
+        let engine = try MarkBaseEngine(autoCompile: true)
+        let model = try E4BModel(modelDir: modelPath, engine: engine, maxContextLength: 512)
+        let tokenizer = try TokenizerFactory.load(modelDir: modelPath)
+        
+        let prompt = swiftPrompts[0]
+        print("Prompt: \(prompt.prompt)")
+        
+        let tokens = tokenizer.encode(text: prompt.prompt)
+        var generatedTokens: [Int] = []
+        
+        for i in 0..<50 {
+            let pos = tokens.count + i - 1
+            let logits = try model.forwardOptimized(tokenId: tokens.last ?? 0, position: pos)
+            let nextToken = logits.enumerated().max(by: { $0.element < $1.element })?.offset ?? 0
+            generatedTokens.append(nextToken)
+            
+            if tokenizer.eosTokenIds.contains(nextToken) { break }
+        }
+        
+        let generatedCode = tokenizer.decode(tokens: generatedTokens)
+        print("Generated: \(generatedCode)")
+        
+        let result = compileAndRunSwift(code: generatedCode, testInput: prompt.testInput)
+        
+        if result.success {
+            print("✅ PASS")
+        } else {
+            print("⚠ FAIL: \(result.error)")
+        }
+        
+        model.kvCaches.forEach { cache in cache.reset() }
+    }
+    
+    func testPythonCodeGeneration() throws {
+        print("\n═════════════════════════════════════════════════════════════════════")
+        print("  Python Code Generation Test")
+        print("═════════════════════════════════════════════════════════════════════\n")
+        
+        let modelPath = "/Users/accusys/MarkBaseEngine/models/E4B-MarkBase"
+        let engine = try MarkBaseEngine(autoCompile: true)
+        let model = try E4BModel(modelDir: modelPath, engine: engine, maxContextLength: 512)
+        let tokenizer = try TokenizerFactory.load(modelDir: modelPath)
+        
+        let prompt = pythonPrompts[0]
+        print("Prompt: \(prompt.prompt)")
+        
+        let tokens = tokenizer.encode(text: prompt.prompt)
+        var generatedTokens: [Int] = []
+        
+        for i in 0..<50 {
+            let pos = tokens.count + i - 1
+            let logits = try model.forwardOptimized(tokenId: tokens.last ?? 0, position: pos)
+            let nextToken = logits.enumerated().max(by: { $0.element < $1.element })?.offset ?? 0
+            generatedTokens.append(nextToken)
+            
+            if tokenizer.eosTokenIds.contains(nextToken) { break }
+        }
+        
+        let generatedCode = tokenizer.decode(tokens: generatedTokens)
+        print("Generated: \(generatedCode)")
+        
+        let result = compileAndRunPython(code: generatedCode, testInput: prompt.testInput)
+        
+        if result.success {
+            print("✅ PASS")
+        } else {
+            print("⚠ FAIL: \(result.error)")
+        }
+        
+        model.kvCaches.forEach { cache in cache.reset() }
+    }
+}