import Foundation // ───────────────────────────────────────────────────────────── // Multimodal API Models // OpenAI Compatible Format // ───────────────────────────────────────────────────────────── /// Multimodal message content part public enum ContentPart: Codable { case text(String) case imageUrl(ImageUrl) case audioUrl(AudioUrl) case videoUrl(VideoUrl) public struct ImageUrl: Codable { public let url: String public init(url: String) { self.url = url } } public struct AudioUrl: Codable { public let url: String public init(url: String) { self.url = url } } public struct VideoUrl: Codable { public let url: String public init(url: String) { self.url = url } } private enum CodingKeys: String, CodingKey { case type case text case imageUrl = "image_url" case audioUrl = "audio_url" case videoUrl = "video_url" } public init(from decoder: Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) let type = try container.decode(String.self, forKey: .type) switch type { case "text": let text = try container.decode(String.self, forKey: .text) self = .text(text) case "image_url": let imageUrl = try container.decode(ImageUrl.self, forKey: .imageUrl) self = .imageUrl(imageUrl) case "audio_url": let audioUrl = try container.decode(AudioUrl.self, forKey: .audioUrl) self = .audioUrl(audioUrl) case "video_url": let videoUrl = try container.decode(VideoUrl.self, forKey: .videoUrl) self = .videoUrl(videoUrl) default: throw DecodingError.dataCorruptedError( forKey: .type, in: container, debugDescription: "Unknown content type: \(type)" ) } } public func encode(to encoder: Encoder) throws { var container = encoder.container(keyedBy: CodingKeys.self) switch self { case .text(let text): try container.encode("text", forKey: .type) try container.encode(text, forKey: .text) case .imageUrl(let imageUrl): try container.encode("image_url", forKey: .type) try container.encode(imageUrl, forKey: .imageUrl) case .audioUrl(let audioUrl): try container.encode("audio_url", forKey: .type) try container.encode(audioUrl, forKey: .audioUrl) case .videoUrl(let videoUrl): try container.encode("video_url", forKey: .type) try container.encode(videoUrl, forKey: .videoUrl) } } } /// Multimodal message public struct MultimodalMessage: Codable { public let role: String public let content: [ContentPart] public init(role: String, content: [ContentPart]) { self.role = role self.content = content } /// Extract text content public var textContent: String { content.compactMap { part -> String? in if case .text(let text) = part { return text } return nil }.joined(separator: "\n") } /// Extract image URLs public var imageUrls: [ContentPart.ImageUrl] { content.compactMap { part -> ContentPart.ImageUrl? in if case .imageUrl(let url) = part { return url } return nil } } /// Extract audio URLs public var audioUrls: [ContentPart.AudioUrl] { content.compactMap { part -> ContentPart.AudioUrl? in if case .audioUrl(let url) = part { return url } return nil } } /// Extract video URLs public var videoUrls: [ContentPart.VideoUrl] { content.compactMap { part -> ContentPart.VideoUrl? in if case .videoUrl(let url) = part { return url } return nil } } } /// Multimodal chat completion request public struct MultimodalChatRequest: Codable { public let messages: [MultimodalMessage] public let max_tokens: Int? public let temperature: Float? public let top_p: Float? public let top_k: Int? public let stream: Bool? public let tools: [Tool]? public let response_format: ResponseFormat? public init( messages: [MultimodalMessage], max_tokens: Int? = nil, temperature: Float? = nil, top_p: Float? = nil, top_k: Int? = nil, stream: Bool? = nil, tools: [Tool]? = nil, response_format: ResponseFormat? = nil ) { self.messages = messages self.max_tokens = max_tokens self.temperature = temperature self.top_p = top_p self.top_k = top_k self.stream = stream self.tools = tools self.response_format = response_format } } // ───────────────────────────────────────────────────────────── // Image/Audio Processing Helpers // ───────────────────────────────────────────────────────────── public enum MediaProcessor { /// Parse data URI to base64 and mime type public static func parseDataURI(_ uri: String) throws -> (mimeType: String, data: Data) { guard uri.hasPrefix("data:") else { throw MarkBaseError.invalidParameter( parameter: "url", message: "Expected data URI format" ) } let parts = uri.split(separator: ",", maxSplits: 1) guard parts.count == 2 else { throw MarkBaseError.invalidParameter( parameter: "url", message: "Invalid data URI format" ) } let header = String(parts[0]) let base64 = String(parts[1]) // Extract mime type let mimeType = header.dropFirst(5).split(separator: ";").first.map(String.init) ?? "application/octet-stream" // Decode base64 guard let data = Data(base64Encoded: base64) else { throw MarkBaseError.invalidParameter( parameter: "url", message: "Invalid base64 data" ) } return (mimeType, data) } /// Load image from URL or data URI public static func loadImage(from url: String) throws -> Data { if url.hasPrefix("data:") { let (_, data) = try parseDataURI(url) return data } else if url.hasPrefix("http://") || url.hasPrefix("https://") { throw MarkBaseError.invalidParameter( parameter: "url", message: "HTTP URLs not yet supported, use base64 data URI" ) } else { // Local file path let filePath = url guard FileManager.default.fileExists(atPath: filePath) else { throw MarkBaseError.invalidParameter( parameter: "url", message: "File not found: \(filePath)" ) } return try Data(contentsOf: URL(fileURLWithPath: filePath)) } } /// Load audio from URL or data URI public static func loadAudio(from url: String) throws -> Data { if url.hasPrefix("data:") { let (_, data) = try parseDataURI(url) return data } else { throw MarkBaseError.invalidParameter( parameter: "url", message: "Only base64 data URI supported for audio" ) } } /// Load video from file path or data URI public static func loadVideo(from url: String) throws -> URL { if url.hasPrefix("data:") { let (mimeType, data) = try parseDataURI(url) let ext: String if mimeType.contains("mp4") { ext = "mp4" } else if mimeType.contains("quicktime") { ext = "mov" } else { ext = "mp4" } let tempURL = FileManager.default.temporaryDirectory .appendingPathComponent(UUID().uuidString) .appendingPathExtension(ext) try data.write(to: tempURL) return tempURL } else { let fileURL = URL(fileURLWithPath: url) guard FileManager.default.fileExists(atPath: fileURL.path) else { throw MarkBaseError.invalidParameter( parameter: "url", message: "Video file not found: \(url)" ) } return fileURL } } }