Initial commit: E4B-MarkBase model integration with passing tests
CI / build-and-test (push) Has been cancelled
CI / build-and-test (push) Has been cancelled
- E4B-MarkBase model (42 layers, 4.4GB) loaded successfully - All Phase 1-6 tests passed (model loading, forward pass, vision/audio towers, token generation, performance) - All stress tests passed (5/5 in 127.6s) - Concurrent inference - Memory stress (67.5 tok/s, 0 NaN) - Continuous generation - Batch processing - Long-running stability - Swift Metal inference engine with multimodal support
This commit is contained in:
MarkBase Admin
@@ -0,0 +1,267 @@
|
||||
import Foundation
|
||||
|
||||
// ─────────────────────────────────────────────────────────────
|
||||
// Multimodal API Models
|
||||
// OpenAI Compatible Format
|
||||
// ─────────────────────────────────────────────────────────────
|
||||
|
||||
/// Multimodal message content part
|
||||
public enum ContentPart: Codable {
|
||||
case text(String)
|
||||
case imageUrl(ImageUrl)
|
||||
case audioUrl(AudioUrl)
|
||||
case videoUrl(VideoUrl)
|
||||
|
||||
public struct ImageUrl: Codable {
|
||||
public let url: String
|
||||
|
||||
public init(url: String) {
|
||||
self.url = url
|
||||
}
|
||||
}
|
||||
|
||||
public struct AudioUrl: Codable {
|
||||
public let url: String
|
||||
|
||||
public init(url: String) {
|
||||
self.url = url
|
||||
}
|
||||
}
|
||||
|
||||
public struct VideoUrl: Codable {
|
||||
public let url: String
|
||||
|
||||
public init(url: String) {
|
||||
self.url = url
|
||||
}
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case type
|
||||
case text
|
||||
case imageUrl = "image_url"
|
||||
case audioUrl = "audio_url"
|
||||
case videoUrl = "video_url"
|
||||
}
|
||||
|
||||
public init(from decoder: Decoder) throws {
|
||||
let container = try decoder.container(keyedBy: CodingKeys.self)
|
||||
let type = try container.decode(String.self, forKey: .type)
|
||||
|
||||
switch type {
|
||||
case "text":
|
||||
let text = try container.decode(String.self, forKey: .text)
|
||||
self = .text(text)
|
||||
case "image_url":
|
||||
let imageUrl = try container.decode(ImageUrl.self, forKey: .imageUrl)
|
||||
self = .imageUrl(imageUrl)
|
||||
case "audio_url":
|
||||
let audioUrl = try container.decode(AudioUrl.self, forKey: .audioUrl)
|
||||
self = .audioUrl(audioUrl)
|
||||
case "video_url":
|
||||
let videoUrl = try container.decode(VideoUrl.self, forKey: .videoUrl)
|
||||
self = .videoUrl(videoUrl)
|
||||
default:
|
||||
throw DecodingError.dataCorruptedError(
|
||||
forKey: .type,
|
||||
in: container,
|
||||
debugDescription: "Unknown content type: \(type)"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
public func encode(to encoder: Encoder) throws {
|
||||
var container = encoder.container(keyedBy: CodingKeys.self)
|
||||
|
||||
switch self {
|
||||
case .text(let text):
|
||||
try container.encode("text", forKey: .type)
|
||||
try container.encode(text, forKey: .text)
|
||||
case .imageUrl(let imageUrl):
|
||||
try container.encode("image_url", forKey: .type)
|
||||
try container.encode(imageUrl, forKey: .imageUrl)
|
||||
case .audioUrl(let audioUrl):
|
||||
try container.encode("audio_url", forKey: .type)
|
||||
try container.encode(audioUrl, forKey: .audioUrl)
|
||||
case .videoUrl(let videoUrl):
|
||||
try container.encode("video_url", forKey: .type)
|
||||
try container.encode(videoUrl, forKey: .videoUrl)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Multimodal message
|
||||
public struct MultimodalMessage: Codable {
|
||||
public let role: String
|
||||
public let content: [ContentPart]
|
||||
|
||||
public init(role: String, content: [ContentPart]) {
|
||||
self.role = role
|
||||
self.content = content
|
||||
}
|
||||
|
||||
/// Extract text content
|
||||
public var textContent: String {
|
||||
content.compactMap { part -> String? in
|
||||
if case .text(let text) = part { return text }
|
||||
return nil
|
||||
}.joined(separator: "\n")
|
||||
}
|
||||
|
||||
/// Extract image URLs
|
||||
public var imageUrls: [ContentPart.ImageUrl] {
|
||||
content.compactMap { part -> ContentPart.ImageUrl? in
|
||||
if case .imageUrl(let url) = part { return url }
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract audio URLs
|
||||
public var audioUrls: [ContentPart.AudioUrl] {
|
||||
content.compactMap { part -> ContentPart.AudioUrl? in
|
||||
if case .audioUrl(let url) = part { return url }
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract video URLs
|
||||
public var videoUrls: [ContentPart.VideoUrl] {
|
||||
content.compactMap { part -> ContentPart.VideoUrl? in
|
||||
if case .videoUrl(let url) = part { return url }
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Multimodal chat completion request
|
||||
public struct MultimodalChatRequest: Codable {
|
||||
public let messages: [MultimodalMessage]
|
||||
public let max_tokens: Int?
|
||||
public let temperature: Float?
|
||||
public let top_p: Float?
|
||||
public let top_k: Int?
|
||||
public let stream: Bool?
|
||||
public let tools: [Tool]?
|
||||
public let response_format: ResponseFormat?
|
||||
|
||||
public init(
|
||||
messages: [MultimodalMessage],
|
||||
max_tokens: Int? = nil,
|
||||
temperature: Float? = nil,
|
||||
top_p: Float? = nil,
|
||||
top_k: Int? = nil,
|
||||
stream: Bool? = nil,
|
||||
tools: [Tool]? = nil,
|
||||
response_format: ResponseFormat? = nil
|
||||
) {
|
||||
self.messages = messages
|
||||
self.max_tokens = max_tokens
|
||||
self.temperature = temperature
|
||||
self.top_p = top_p
|
||||
self.top_k = top_k
|
||||
self.stream = stream
|
||||
self.tools = tools
|
||||
self.response_format = response_format
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────
|
||||
// Image/Audio Processing Helpers
|
||||
// ─────────────────────────────────────────────────────────────
|
||||
|
||||
public enum MediaProcessor {
|
||||
/// Parse data URI to base64 and mime type
|
||||
public static func parseDataURI(_ uri: String) throws -> (mimeType: String, data: Data) {
|
||||
guard uri.hasPrefix("data:") else {
|
||||
throw MarkBaseError.invalidParameter(
|
||||
parameter: "url",
|
||||
message: "Expected data URI format"
|
||||
)
|
||||
}
|
||||
|
||||
let parts = uri.split(separator: ",", maxSplits: 1)
|
||||
guard parts.count == 2 else {
|
||||
throw MarkBaseError.invalidParameter(
|
||||
parameter: "url",
|
||||
message: "Invalid data URI format"
|
||||
)
|
||||
}
|
||||
|
||||
let header = String(parts[0])
|
||||
let base64 = String(parts[1])
|
||||
|
||||
// Extract mime type
|
||||
let mimeType = header.dropFirst(5).split(separator: ";").first.map(String.init) ?? "application/octet-stream"
|
||||
|
||||
// Decode base64
|
||||
guard let data = Data(base64Encoded: base64) else {
|
||||
throw MarkBaseError.invalidParameter(
|
||||
parameter: "url",
|
||||
message: "Invalid base64 data"
|
||||
)
|
||||
}
|
||||
|
||||
return (mimeType, data)
|
||||
}
|
||||
|
||||
/// Load image from URL or data URI
|
||||
public static func loadImage(from url: String) throws -> Data {
|
||||
if url.hasPrefix("data:") {
|
||||
let (_, data) = try parseDataURI(url)
|
||||
return data
|
||||
} else if url.hasPrefix("http://") || url.hasPrefix("https://") {
|
||||
throw MarkBaseError.invalidParameter(
|
||||
parameter: "url",
|
||||
message: "HTTP URLs not yet supported, use base64 data URI"
|
||||
)
|
||||
} else {
|
||||
// Local file path
|
||||
let filePath = url
|
||||
guard FileManager.default.fileExists(atPath: filePath) else {
|
||||
throw MarkBaseError.invalidParameter(
|
||||
parameter: "url",
|
||||
message: "File not found: \(filePath)"
|
||||
)
|
||||
}
|
||||
return try Data(contentsOf: URL(fileURLWithPath: filePath))
|
||||
}
|
||||
}
|
||||
|
||||
/// Load audio from URL or data URI
|
||||
public static func loadAudio(from url: String) throws -> Data {
|
||||
if url.hasPrefix("data:") {
|
||||
let (_, data) = try parseDataURI(url)
|
||||
return data
|
||||
} else {
|
||||
throw MarkBaseError.invalidParameter(
|
||||
parameter: "url",
|
||||
message: "Only base64 data URI supported for audio"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Load video from file path or data URI
|
||||
public static func loadVideo(from url: String) throws -> URL {
|
||||
if url.hasPrefix("data:") {
|
||||
let (mimeType, data) = try parseDataURI(url)
|
||||
let ext: String
|
||||
if mimeType.contains("mp4") { ext = "mp4" }
|
||||
else if mimeType.contains("quicktime") { ext = "mov" }
|
||||
else { ext = "mp4" }
|
||||
let tempURL = FileManager.default.temporaryDirectory
|
||||
.appendingPathComponent(UUID().uuidString)
|
||||
.appendingPathExtension(ext)
|
||||
try data.write(to: tempURL)
|
||||
return tempURL
|
||||
} else {
|
||||
let fileURL = URL(fileURLWithPath: url)
|
||||
guard FileManager.default.fileExists(atPath: fileURL.path) else {
|
||||
throw MarkBaseError.invalidParameter(
|
||||
parameter: "url",
|
||||
message: "Video file not found: \(url)"
|
||||
)
|
||||
}
|
||||
return fileURL
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user