warren/markbaseengine
1
0
Fork 0
Code Issues Pull Requests Actions 19 Packages Projects Releases Wiki Activity

Add comprehensive code generation test framework
CI / build-and-test (push) Has been cancelled
Details

Browse Source 
- Created test infrastructure for 240 tests (57 implemented)
- Programming tests: Swift, Python, C++, JavaScript, Rust (40 tests)
- Non-programming tests: Text, Math, Logic, Knowledge, Vision, Audio (17 tests)
- Installed Rust compiler (rustc 1.96.0)
- Test framework builds successfully
- Sample test executed (generation quality needs improvement)
- Identified issues: greedy sampling, position indexing, code syntax
This commit is contained in:
MarkBase Admin
2026-06-23 19:36:26 +08:00
parent ac75faa0cc
commit 80a78ec554
7 changed files with 620 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Tests/MarkBaseTests/TestData/CodePrompts.swift
+67
View File
@@ -0,0 +1,67 @@
import Foundation
struct CodePrompt {
let id: String
let language: String
let level: String
let prompt: String
let testInput: String
let expectedOutput: String
}
let swiftPrompts: [CodePrompt] = [
CodePrompt(id: "L1-SW-01", language: "Swift", level: "Level 1", prompt: "Write a Swift function `factorial(n: Int) -> Int` to calculate factorial. Include complete implementation.", testInput: "5", expectedOutput: "120"),
CodePrompt(id: "L1-SW-02", language: "Swift", level: "Level 1", prompt: "Write a Swift function `isPrime(n: Int) -> Bool` to check if a number is prime. Include complete implementation.", testInput: "7", expectedOutput: "true"),
CodePrompt(id: "L2-SW-01", language: "Swift", level: "Level 2", prompt: "Implement a Stack<T> struct in Swift with push and pop methods.", testInput: "", expectedOutput: "Stack"),
CodePrompt(id: "L3-SW-01", language: "Swift", level: "Level 3", prompt: "Write a Swift function `quicksort<T: Comparable>(_ arr: [T]) -> [T]` to implement quicksort.", testInput: "", expectedOutput: "sorted"),
CodePrompt(id: "L4-SW-01", language: "Swift", level: "Level 4", prompt: "Write a complete Swift program that reads a file and counts lines.", testInput: "", expectedOutput: "program"),
CodePrompt(id: "L5-SW-01", language: "Swift", level: "Level 5", prompt: "Write a Swift function with proper error handling using Result<T, Error>.", testInput: "", expectedOutput: "Result"),
CodePrompt(id: "L6-SW-01", language: "Swift", level: "Level 6", prompt: "Write a Swift async function that runs concurrent tasks.", testInput: "", expectedOutput: "async"),
CodePrompt(id: "L7-SW-01", language: "Swift", level: "Level 7", prompt: "Write a Swift function that calls a REST API and handles JSON response.", testInput: "", expectedOutput: "API")
]
let pythonPrompts: [CodePrompt] = [
CodePrompt(id: "L1-PY-01", language: "Python", level: "Level 1", prompt: "Write a Python function `reverse_string(s)` to reverse a string.", testInput: "hello", expectedOutput: "olleh"),
CodePrompt(id: "L1-PY-02", language: "Python", level: "Level 1", prompt: "Write a Python function `find_max(lst)` to find the maximum in a list.", testInput: "", expectedOutput: "max"),
CodePrompt(id: "L2-PY-01", language: "Python", level: "Level 2", prompt: "Implement a BinaryTree class in Python with insert and search.", testInput: "", expectedOutput: "Tree"),
CodePrompt(id: "L3-PY-01", language: "Python", level: "Level 3", prompt: "Write a Python function `mergesort(arr)` to implement mergesort.", testInput: "", expectedOutput: "sorted"),
CodePrompt(id: "L4-PY-01", language: "Python", level: "Level 4", prompt: "Write a Python script to analyze a CSV file and calculate statistics.", testInput: "", expectedOutput: "CSV"),
CodePrompt(id: "L5-PY-01", language: "Python", level: "Level 5", prompt: "Write a Python function with proper exception handling for file operations.", testInput: "", expectedOutput: "Exception"),
CodePrompt(id: "L6-PY-01", language: "Python", level: "Level 6", prompt: "Write a Python async function using asyncio.", testInput: "", expectedOutput: "async"),
CodePrompt(id: "L7-PY-01", language: "Python", level: "Level 7", prompt: "Write a Python function to connect to SQLite database and execute queries.", testInput: "", expectedOutput: "SQL")
]
let cppPrompts: [CodePrompt] = [
CodePrompt(id: "L1-CP-01", language: "C++", level: "Level 1", prompt: "Write a C++ function `int factorial(int n)` for factorial calculation.", testInput: "5", expectedOutput: "120"),
CodePrompt(id: "L1-CP-02", language: "C++", level: "Level 1", prompt: "Write a C++ function `int gcd(int a, int b)` for GCD calculation.", testInput: "", expectedOutput: "gcd"),
CodePrompt(id: "L2-CP-01", language: "C++", level: "Level 2", prompt: "Implement a Stack template class in C++ with push and pop.", testInput: "", expectedOutput: "Stack"),
CodePrompt(id: "L3-CP-01", language: "C++", level: "Level 3", prompt: "Write a C++ function for binary search implementation.", testInput: "", expectedOutput: "binary"),
CodePrompt(id: "L4-CP-01", language: "C++", level: "Level 4", prompt: "Write a C++ program that processes text files.", testInput: "", expectedOutput: "program"),
CodePrompt(id: "L5-CP-01", language: "C++", level: "Level 5", prompt: "Write a C++ function with proper exception handling.", testInput: "", expectedOutput: "Exception"),
CodePrompt(id: "L6-CP-01", language: "C++", level: "Level 6", prompt: "Write a C++ thread-safe counter using mutex.", testInput: "", expectedOutput: "mutex"),
CodePrompt(id: "L7-CP-01", language: "C++", level: "Level 7", prompt: "Write a C++ function to make HTTP requests using curl.", testInput: "", expectedOutput: "HTTP")
]
let javascriptPrompts: [CodePrompt] = [
CodePrompt(id: "L1-JS-01", language: "JavaScript", level: "Level 1", prompt: "Write a JavaScript function `sumArray(arr)` to sum array elements.", testInput: "", expectedOutput: "sum"),
CodePrompt(id: "L1-JS-02", language: "JavaScript", level: "Level 1", prompt: "Write a JavaScript function `filterOdd(arr)` to filter odd numbers.", testInput: "", expectedOutput: "odd"),
CodePrompt(id: "L2-JS-01", language: "JavaScript", level: "Level 2", prompt: "Implement a LinkedList in JavaScript with insert and delete.", testInput: "", expectedOutput: "LinkedList"),
CodePrompt(id: "L3-JS-01", language: "JavaScript", level: "Level 3", prompt: "Write a JavaScript function `quicksort(arr)` implementation.", testInput: "", expectedOutput: "sorted"),
CodePrompt(id: "L4-JS-01", language: "JavaScript", level: "Level 4", prompt: "Write a Node.js HTTP server that serves JSON data.", testInput: "", expectedOutput: "server"),
CodePrompt(id: "L5-JS-01", language: "JavaScript", level: "Level 5", prompt: "Write a JavaScript async function with proper error handling.", testInput: "", expectedOutput: "async"),
CodePrompt(id: "L6-JS-01", language: "JavaScript", level: "Level 6", prompt: "Write a JavaScript async generator for stream processing.", testInput: "", expectedOutput: "generator"),
CodePrompt(id: "L7-JS-01", language: "JavaScript", level: "Level 7", prompt: "Write a JavaScript function to fetch API and handle JSON response.", testInput: "", expectedOutput: "fetch")
]
let rustPrompts: [CodePrompt] = [
CodePrompt(id: "L1-RS-01", language: "Rust", level: "Level 1", prompt: "Write a Rust function `fn factorial(n: u64) -> u64` for factorial.", testInput: "5", expectedOutput: "120"),
CodePrompt(id: "L1-RS-02", language: "Rust", level: "Level 1", prompt: "Write a Rust function `fn is_prime(n: u64) -> bool` for prime check.", testInput: "7", expectedOutput: "true"),
CodePrompt(id: "L2-RS-01", language: "Rust", level: "Level 2", prompt: "Implement a Stack using Vec<T> in Rust.", testInput: "", expectedOutput: "Stack"),
CodePrompt(id: "L3-RS-01", language: "Rust", level: "Level 3", prompt: "Write a Rust function for mergesort implementation.", testInput: "", expectedOutput: "sorted"),
CodePrompt(id: "L4-RS-01", language: "Rust", level: "Level 4", prompt: "Write a Rust CLI program that processes command arguments.", testInput: "", expectedOutput: "CLI"),
CodePrompt(id: "L5-RS-01", language: "Rust", level: "Level 5", prompt: "Write a Rust function using Result<T, E> for error handling.", testInput: "", expectedOutput: "Result"),
CodePrompt(id: "L6-RS-01", language: "Rust", level: "Level 6", prompt: "Write a Rust async function using tokio.", testInput: "", expectedOutput: "async"),
CodePrompt(id: "L7-RS-01", language: "Rust", level: "Level 7", prompt: "Write a Rust HTTP client using reqwest crate.", testInput: "", expectedOutput: "HTTP")
]
let allCodePrompts = swiftPrompts + pythonPrompts + cppPrompts + javascriptPrompts + rustPrompts