MarkBase Admin
80a78ec554
CI / build-and-test (push) Has been cancelled
- Created test infrastructure for 240 tests (57 implemented) - Programming tests: Swift, Python, C++, JavaScript, Rust (40 tests) - Non-programming tests: Text, Math, Logic, Knowledge, Vision, Audio (17 tests) - Installed Rust compiler (rustc 1.96.0) - Test framework builds successfully - Sample test executed (generation quality needs improvement) - Identified issues: greedy sampling, position indexing, code syntax
44 lines
3.2 KiB
Swift
44 lines
3.2 KiB
Swift
import Foundation
|
|
|
|
struct NonProgrammingPrompt {
|
|
let id: String
|
|
let category: String
|
|
let prompt: String
|
|
let expectedKeywords: [String]
|
|
}
|
|
|
|
let textPrompts: [NonProgrammingPrompt] = [
|
|
NonProgrammingPrompt(id: "TEXT-01", category: "Text", prompt: "Summarize this article in 3 sentences: 'Swift is a powerful programming language developed by Apple. It combines modern syntax with performance optimizations. Swift is used for iOS, macOS, and server-side development.'", expectedKeywords: ["Swift", "Apple", "programming"]),
|
|
NonProgrammingPrompt(id: "TEXT-02", category: "Text", prompt: "Translate this English text to Chinese: 'Hello, world'", expectedKeywords: ["你好", "世界"]),
|
|
NonProgrammingPrompt(id: "TEXT-03", category: "Text", prompt: "Rewrite this sentence in formal tone: 'Hey, what's up?'", expectedKeywords: ["formal", "greeting"])
|
|
]
|
|
|
|
let mathPrompts: [NonProgrammingPrompt] = [
|
|
NonProgrammingPrompt(id: "MATH-01", category: "Math", prompt: "Calculate: 234 + 567 - 123", expectedKeywords: ["678"]),
|
|
NonProgrammingPrompt(id: "MATH-02", category: "Math", prompt: "Solve for x: 2x + 5 = 15", expectedKeywords: ["5", "x=5"]),
|
|
NonProgrammingPrompt(id: "MATH-03", category: "Math", prompt: "Calculate the area of a triangle with base 5 and height 8", expectedKeywords: ["20", "area"])
|
|
]
|
|
|
|
let logicPrompts: [NonProgrammingPrompt] = [
|
|
NonProgrammingPrompt(id: "LOGIC-01", category: "Logic", prompt: "If A implies B, and B is false, what can we conclude about A?", expectedKeywords: ["false", "A is false"]),
|
|
NonProgrammingPrompt(id: "LOGIC-02", category: "Logic", prompt: "All humans are mortal. Socrates is human. What follows?", expectedKeywords: ["mortal", "Socrates"]),
|
|
NonProgrammingPrompt(id: "LOGIC-03", category: "Logic", prompt: "Prove that the sum of two even numbers is even", expectedKeywords: ["even", "2", "sum"])
|
|
]
|
|
|
|
let knowledgePrompts: [NonProgrammingPrompt] = [
|
|
NonProgrammingPrompt(id: "KNOW-01", category: "Knowledge", prompt: "What is the speed of light in vacuum?", expectedKeywords: ["299792458", "3e8", "300000"]),
|
|
NonProgrammingPrompt(id: "KNOW-02", category: "Knowledge", prompt: "When did World War II end?", expectedKeywords: ["1945", "September"]),
|
|
NonProgrammingPrompt(id: "KNOW-03", category: "Knowledge", prompt: "What is the capital of Australia?", expectedKeywords: ["Canberra"])
|
|
]
|
|
|
|
let multimodalPrompts: [NonProgrammingPrompt] = [
|
|
NonProgrammingPrompt(id: "VIS-01", category: "Vision", prompt: "Describe what you see in a typical office scene", expectedKeywords: ["desk", "computer", "chair"]),
|
|
NonProgrammingPrompt(id: "VIS-02", category: "Vision", prompt: "What features would you expect in a sunset image?", expectedKeywords: ["sun", "colors", "horizon"])
|
|
]
|
|
|
|
let audioPrompts: [NonProgrammingPrompt] = [
|
|
NonProgrammingPrompt(id: "AUD-01", category: "Audio", prompt: "What are common characteristics of speech audio?", expectedKeywords: ["voice", "words", "frequency"]),
|
|
NonProgrammingPrompt(id: "AUD-02", category: "Audio", prompt: "Describe the difference between music and noise", expectedKeywords: ["harmony", "random", "pattern"])
|
|
]
|
|
|
|
let allNonProgrammingPrompts = textPrompts + mathPrompts + logicPrompts + knowledgePrompts + multimodalPrompts + audioPrompts |