import CryptoKit import Foundation public enum ImportEngineError: LocalizedError, Sendable { case sourceNotFound(String) case unsupportedSource(path: String, expected: String) case unreadableSource(path: String, reason: String) case invalidFormat(path: String, reason: String) public var errorDescription: String? { switch self { case let .sourceNotFound(path): "Die Importquelle wurde nicht gefunden: \(path)" case let .unsupportedSource(path, expected): "Die Importquelle \(path) passt nicht zum erwarteten Format (\(expected))." case let .unreadableSource(path, reason): "Die Importquelle \(path) konnte nicht gelesen werden: \(reason)" case let .invalidFormat(path, reason): "Die Importquelle \(path) hat ein ungültiges Format: \(reason)" } } } struct ImportParseResult: Sendable { let candidates: [ImportCandidate] let statistics: ImportStatistics } protocol ImportParser: Sendable { func parse(sourceURL: URL, dedupIndex: DedupIndex) throws -> ImportParseResult } struct ImportAccumulator { private(set) var candidates: [ImportCandidate] = [] private(set) var statistics = ImportStatistics() private let minimumLength: Int private let maximumLength: Int private let dedupIndex: DedupIndex private let fallbackPromptedAt: String private var previewHashes: Set = [] init(minimumLength: Int, maximumLength: Int, dedupIndex: DedupIndex) { self.minimumLength = minimumLength self.maximumLength = maximumLength self.dedupIndex = dedupIndex fallbackPromptedAt = ImportTimestamp.now() } mutating func accept( _ rawContent: String, sourcePrefix: String, tags: String?, model: String? = nil, promptedAt: String? = nil ) { statistics.scanned += 1 let content = ImportText.trimmed(rawContent) let length = ImportText.pythonCharacterCount(content) guard length >= minimumLength else { statistics.skippedShort += 1 return } guard length <= maximumLength else { statistics.skippedLong += 1 return } let hash = ImportText.contentHash(content) guard !dedupIndex.contains(hash), !previewHashes.contains(hash) else { statistics.skippedDuplicate += 1 return } previewHashes.insert(hash) candidates.append(ImportCandidate( title: ImportText.title(for: content), content: content, source: "\(sourcePrefix):\(hash)", tags: tags, model: model, promptedAt: promptedAt ?? fallbackPromptedAt )) statistics.imported += 1 } mutating func skipAgent() { statistics.scanned += 1 statistics.skippedAgent += 1 } mutating func skipSystem() { statistics.scanned += 1 statistics.skippedSystem += 1 } mutating func skipMalformed(scanned: Bool = false) { if scanned { statistics.scanned += 1 } statistics.skippedMalformed += 1 } func result() -> ImportParseResult { ImportParseResult(candidates: candidates, statistics: statistics) } } enum ImportTimestamp { static func normalize(_ value: Any?) -> String? { if let number = value as? NSNumber { return fromUnixTimestamp(number.doubleValue) } guard let string = value as? String else { return nil } let trimmed = ImportText.trimmed(string) guard !trimmed.isEmpty else { return nil } if let numericValue = Double(trimmed) { return fromUnixTimestamp(numericValue) } let fractionalFormatter = ISO8601DateFormatter() fractionalFormatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] if let date = fractionalFormatter.date(from: trimmed) { return format(date) } let internetFormatter = ISO8601DateFormatter() internetFormatter.formatOptions = [.withInternetDateTime] if let date = internetFormatter.date(from: trimmed) { return format(date) } let dateOnlyFormatter = DateFormatter() dateOnlyFormatter.calendar = Calendar(identifier: .gregorian) dateOnlyFormatter.locale = Locale(identifier: "en_US_POSIX") dateOnlyFormatter.timeZone = TimeZone(secondsFromGMT: 0) dateOnlyFormatter.dateFormat = "yyyy-MM-dd" dateOnlyFormatter.isLenient = false if let date = dateOnlyFormatter.date(from: trimmed) { return format(date) } return nil } static func now() -> String { format(Date()) } private static func fromUnixTimestamp(_ value: Double) -> String? { guard value.isFinite else { return nil } let seconds = abs(value) >= 10_000_000_000 ? value / 1_000 : value return format(Date(timeIntervalSince1970: seconds)) } private static func format(_ date: Date) -> String { let formatter = ISO8601DateFormatter() formatter.timeZone = TimeZone(secondsFromGMT: 0) formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] return formatter.string(from: date) } } enum ImportText { static func trimmed(_ text: String) -> String { text.trimmingCharacters(in: .whitespacesAndNewlines) } /// Python's `len(str)` counts Unicode code points. Unicode scalars are the /// matching representation for valid Swift strings decoded from UTF-8/JSON. static func pythonCharacterCount(_ text: String) -> Int { text.unicodeScalars.count } static func contentHash(_ text: String) -> String { let digest = Insecure.MD5.hash(data: Data(text.utf8)) return digest.prefix(6).map { String(format: "%02x", $0) }.joined() } static func title(for content: String) -> String { let prefix = unicodeScalarPrefix(content, count: 80) .replacingOccurrences(of: "\n", with: " ") .trimmingCharacters(in: .whitespacesAndNewlines) return pythonCharacterCount(content) > 80 ? prefix + "..." : prefix } private static func unicodeScalarPrefix(_ text: String, count: Int) -> String { var result = "" for scalar in text.unicodeScalars.prefix(count) { result.unicodeScalars.append(scalar) } return result } } enum ImportFiles { static func isDirectory(_ url: URL) -> Bool { var isDirectory = ObjCBool(false) return FileManager.default.fileExists(atPath: url.path, isDirectory: &isDirectory) && isDirectory.boolValue } static func requireExisting(_ url: URL) throws { guard FileManager.default.fileExists(atPath: url.path) else { throw ImportEngineError.sourceNotFound(url.path) } } static func immediateFiles( at sourceURL: URL, expected: String, directFileMatches: (URL) -> Bool, directoryFileMatches: (URL) -> Bool ) throws -> [URL] { try requireExisting(sourceURL) if !isDirectory(sourceURL) { guard directFileMatches(sourceURL) else { throw ImportEngineError.unsupportedSource(path: sourceURL.path, expected: expected) } return [sourceURL] } let files: [URL] do { files = try FileManager.default.contentsOfDirectory( at: sourceURL, includingPropertiesForKeys: [.isRegularFileKey], options: [.skipsHiddenFiles] ) } catch { throw ImportEngineError.unreadableSource( path: sourceURL.path, reason: error.localizedDescription ) } let matches = files .filter(directoryFileMatches) .filter(isRegularFile) .sorted { $0.path < $1.path } guard !matches.isEmpty else { throw ImportEngineError.unsupportedSource(path: sourceURL.path, expected: expected) } return matches } static func recursiveJSONLFiles(at sourceURL: URL, excluding component: String) throws -> [URL] { try requireExisting(sourceURL) if !isDirectory(sourceURL) { guard sourceURL.pathExtension.lowercased() == "jsonl" else { throw ImportEngineError.unsupportedSource( path: sourceURL.path, expected: "eine .jsonl-Datei oder ein Claude-Projektordner" ) } if sourceURL.pathComponents.contains(where: { $0.caseInsensitiveCompare(component) == .orderedSame }) { return [] } return [sourceURL] } guard let enumerator = FileManager.default.enumerator( at: sourceURL, includingPropertiesForKeys: [.isRegularFileKey], options: [], errorHandler: nil ) else { throw ImportEngineError.unreadableSource( path: sourceURL.path, reason: "Der Ordner konnte nicht aufgelistet werden." ) } var files: [URL] = [] for case let fileURL as URL in enumerator { if fileURL.pathComponents.contains(where: { $0.caseInsensitiveCompare(component) == .orderedSame }) { if isDirectory(fileURL) { enumerator.skipDescendants() } continue } guard fileURL.pathExtension.lowercased() == "jsonl", isRegularFile(fileURL) else { continue } files.append(fileURL) } return files.sorted { $0.path < $1.path } } static func data(from url: URL) throws -> Data { do { return try Data(contentsOf: url, options: [.mappedIfSafe]) } catch { throw ImportEngineError.unreadableSource(path: url.path, reason: error.localizedDescription) } } static func text(from url: URL) throws -> String { let fileData = try data(from: url) guard let text = String(data: fileData, encoding: .utf8) else { throw ImportEngineError.unreadableSource( path: url.path, reason: "Die Datei ist nicht als UTF-8 codiert." ) } return text } static func lines(from url: URL) throws -> [String] { let contents = try text(from: url) var lines: [String] = [] contents.enumerateLines { line, _ in lines.append(line) } return lines } private static func isRegularFile(_ url: URL) -> Bool { (try? url.resourceValues(forKeys: [.isRegularFileKey]).isRegularFile) == true } } enum ImportJSON { static func object(from url: URL) throws -> Any { let fileData = try ImportFiles.data(from: url) return try JSONSerialization.jsonObject(with: fileData) } static func object(fromLine line: String) throws -> Any { try JSONSerialization.jsonObject(with: Data(line.utf8)) } }