| import CryptoKit |
| import Foundation |
|
|
| public enum ImportEngineError: LocalizedError, Sendable { |
| case sourceNotFound(String) |
| case unsupportedSource(path: String, expected: String) |
| case unreadableSource(path: String, reason: String) |
| case invalidFormat(path: String, reason: String) |
|
|
| public var errorDescription: String? { |
| switch self { |
| case let .sourceNotFound(path): |
| "Die Importquelle wurde nicht gefunden: \(path)" |
| case let .unsupportedSource(path, expected): |
| "Die Importquelle \(path) passt nicht zum erwarteten Format (\(expected))." |
| case let .unreadableSource(path, reason): |
| "Die Importquelle \(path) konnte nicht gelesen werden: \(reason)" |
| case let .invalidFormat(path, reason): |
| "Die Importquelle \(path) hat ein ungültiges Format: \(reason)" |
| } |
| } |
| } |
|
|
| struct ImportParseResult: Sendable { |
| let candidates: [ImportCandidate] |
| let statistics: ImportStatistics |
| } |
|
|
| protocol ImportParser: Sendable { |
| func parse(sourceURL: URL, dedupIndex: DedupIndex) throws -> ImportParseResult |
| } |
|
|
| struct ImportAccumulator { |
| private(set) var candidates: [ImportCandidate] = [] |
| private(set) var statistics = ImportStatistics() |
|
|
| private let minimumLength: Int |
| private let maximumLength: Int |
| private let dedupIndex: DedupIndex |
| private let fallbackPromptedAt: String |
| private var previewHashes: Set<String> = [] |
|
|
| init(minimumLength: Int, maximumLength: Int, dedupIndex: DedupIndex) { |
| self.minimumLength = minimumLength |
| self.maximumLength = maximumLength |
| self.dedupIndex = dedupIndex |
| fallbackPromptedAt = ImportTimestamp.now() |
| } |
|
|
| mutating func accept( |
| _ rawContent: String, |
| sourcePrefix: String, |
| tags: String?, |
| model: String? = nil, |
| promptedAt: String? = nil |
| ) { |
| statistics.scanned += 1 |
|
|
| let content = ImportText.trimmed(rawContent) |
| let length = ImportText.pythonCharacterCount(content) |
| guard length >= minimumLength else { |
| statistics.skippedShort += 1 |
| return |
| } |
| guard length <= maximumLength else { |
| statistics.skippedLong += 1 |
| return |
| } |
|
|
| let hash = ImportText.contentHash(content) |
| guard !dedupIndex.contains(hash), !previewHashes.contains(hash) else { |
| statistics.skippedDuplicate += 1 |
| return |
| } |
|
|
| previewHashes.insert(hash) |
| candidates.append(ImportCandidate( |
| title: ImportText.title(for: content), |
| content: content, |
| source: "\(sourcePrefix):\(hash)", |
| tags: tags, |
| model: model, |
| promptedAt: promptedAt ?? fallbackPromptedAt |
| )) |
| statistics.imported += 1 |
| } |
|
|
| mutating func skipAgent() { |
| statistics.scanned += 1 |
| statistics.skippedAgent += 1 |
| } |
|
|
| mutating func skipSystem() { |
| statistics.scanned += 1 |
| statistics.skippedSystem += 1 |
| } |
|
|
| mutating func skipMalformed(scanned: Bool = false) { |
| if scanned { |
| statistics.scanned += 1 |
| } |
| statistics.skippedMalformed += 1 |
| } |
|
|
| func result() -> ImportParseResult { |
| ImportParseResult(candidates: candidates, statistics: statistics) |
| } |
| } |
|
|
| enum ImportTimestamp { |
| static func normalize(_ value: Any?) -> String? { |
| if let number = value as? NSNumber { |
| return fromUnixTimestamp(number.doubleValue) |
| } |
|
|
| guard let string = value as? String else { return nil } |
| let trimmed = ImportText.trimmed(string) |
| guard !trimmed.isEmpty else { return nil } |
|
|
| if let numericValue = Double(trimmed) { |
| return fromUnixTimestamp(numericValue) |
| } |
|
|
| let fractionalFormatter = ISO8601DateFormatter() |
| fractionalFormatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] |
| if let date = fractionalFormatter.date(from: trimmed) { |
| return format(date) |
| } |
|
|
| let internetFormatter = ISO8601DateFormatter() |
| internetFormatter.formatOptions = [.withInternetDateTime] |
| if let date = internetFormatter.date(from: trimmed) { |
| return format(date) |
| } |
|
|
| let dateOnlyFormatter = DateFormatter() |
| dateOnlyFormatter.calendar = Calendar(identifier: .gregorian) |
| dateOnlyFormatter.locale = Locale(identifier: "en_US_POSIX") |
| dateOnlyFormatter.timeZone = TimeZone(secondsFromGMT: 0) |
| dateOnlyFormatter.dateFormat = "yyyy-MM-dd" |
| dateOnlyFormatter.isLenient = false |
| if let date = dateOnlyFormatter.date(from: trimmed) { |
| return format(date) |
| } |
|
|
| return nil |
| } |
|
|
| static func now() -> String { |
| format(Date()) |
| } |
|
|
| private static func fromUnixTimestamp(_ value: Double) -> String? { |
| guard value.isFinite else { return nil } |
| let seconds = abs(value) >= 10_000_000_000 ? value / 1_000 : value |
| return format(Date(timeIntervalSince1970: seconds)) |
| } |
|
|
| private static func format(_ date: Date) -> String { |
| let formatter = ISO8601DateFormatter() |
| formatter.timeZone = TimeZone(secondsFromGMT: 0) |
| formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] |
| return formatter.string(from: date) |
| } |
| } |
|
|
| enum ImportText { |
| static func trimmed(_ text: String) -> String { |
| text.trimmingCharacters(in: .whitespacesAndNewlines) |
| } |
|
|
| |
| |
| static func pythonCharacterCount(_ text: String) -> Int { |
| text.unicodeScalars.count |
| } |
|
|
| static func contentHash(_ text: String) -> String { |
| let digest = Insecure.MD5.hash(data: Data(text.utf8)) |
| return digest.prefix(6).map { String(format: "%02x", $0) }.joined() |
| } |
|
|
| static func title(for content: String) -> String { |
| let prefix = unicodeScalarPrefix(content, count: 80) |
| .replacingOccurrences(of: "\n", with: " ") |
| .trimmingCharacters(in: .whitespacesAndNewlines) |
| return pythonCharacterCount(content) > 80 ? prefix + "..." : prefix |
| } |
|
|
| private static func unicodeScalarPrefix(_ text: String, count: Int) -> String { |
| var result = "" |
| for scalar in text.unicodeScalars.prefix(count) { |
| result.unicodeScalars.append(scalar) |
| } |
| return result |
| } |
| } |
|
|
| enum ImportFiles { |
| static func isDirectory(_ url: URL) -> Bool { |
| var isDirectory = ObjCBool(false) |
| return FileManager.default.fileExists(atPath: url.path, isDirectory: &isDirectory) |
| && isDirectory.boolValue |
| } |
|
|
| static func requireExisting(_ url: URL) throws { |
| guard FileManager.default.fileExists(atPath: url.path) else { |
| throw ImportEngineError.sourceNotFound(url.path) |
| } |
| } |
|
|
| static func immediateFiles( |
| at sourceURL: URL, |
| expected: String, |
| directFileMatches: (URL) -> Bool, |
| directoryFileMatches: (URL) -> Bool |
| ) throws -> [URL] { |
| try requireExisting(sourceURL) |
|
|
| if !isDirectory(sourceURL) { |
| guard directFileMatches(sourceURL) else { |
| throw ImportEngineError.unsupportedSource(path: sourceURL.path, expected: expected) |
| } |
| return [sourceURL] |
| } |
|
|
| let files: [URL] |
| do { |
| files = try FileManager.default.contentsOfDirectory( |
| at: sourceURL, |
| includingPropertiesForKeys: [.isRegularFileKey], |
| options: [.skipsHiddenFiles] |
| ) |
| } catch { |
| throw ImportEngineError.unreadableSource( |
| path: sourceURL.path, |
| reason: error.localizedDescription |
| ) |
| } |
|
|
| let matches = files |
| .filter(directoryFileMatches) |
| .filter(isRegularFile) |
| .sorted { $0.path < $1.path } |
| guard !matches.isEmpty else { |
| throw ImportEngineError.unsupportedSource(path: sourceURL.path, expected: expected) |
| } |
| return matches |
| } |
|
|
| static func recursiveJSONLFiles(at sourceURL: URL, excluding component: String) throws -> [URL] { |
| try requireExisting(sourceURL) |
|
|
| if !isDirectory(sourceURL) { |
| guard sourceURL.pathExtension.lowercased() == "jsonl" else { |
| throw ImportEngineError.unsupportedSource( |
| path: sourceURL.path, |
| expected: "eine .jsonl-Datei oder ein Claude-Projektordner" |
| ) |
| } |
| if sourceURL.pathComponents.contains(where: { $0.caseInsensitiveCompare(component) == .orderedSame }) { |
| return [] |
| } |
| return [sourceURL] |
| } |
|
|
| guard let enumerator = FileManager.default.enumerator( |
| at: sourceURL, |
| includingPropertiesForKeys: [.isRegularFileKey], |
| options: [], |
| errorHandler: nil |
| ) else { |
| throw ImportEngineError.unreadableSource( |
| path: sourceURL.path, |
| reason: "Der Ordner konnte nicht aufgelistet werden." |
| ) |
| } |
|
|
| var files: [URL] = [] |
| for case let fileURL as URL in enumerator { |
| if fileURL.pathComponents.contains(where: { |
| $0.caseInsensitiveCompare(component) == .orderedSame |
| }) { |
| if isDirectory(fileURL) { |
| enumerator.skipDescendants() |
| } |
| continue |
| } |
| guard fileURL.pathExtension.lowercased() == "jsonl", isRegularFile(fileURL) else { |
| continue |
| } |
| files.append(fileURL) |
| } |
| return files.sorted { $0.path < $1.path } |
| } |
|
|
| static func data(from url: URL) throws -> Data { |
| do { |
| return try Data(contentsOf: url, options: [.mappedIfSafe]) |
| } catch { |
| throw ImportEngineError.unreadableSource(path: url.path, reason: error.localizedDescription) |
| } |
| } |
|
|
| static func text(from url: URL) throws -> String { |
| let fileData = try data(from: url) |
| guard let text = String(data: fileData, encoding: .utf8) else { |
| throw ImportEngineError.unreadableSource( |
| path: url.path, |
| reason: "Die Datei ist nicht als UTF-8 codiert." |
| ) |
| } |
| return text |
| } |
|
|
| static func lines(from url: URL) throws -> [String] { |
| let contents = try text(from: url) |
| var lines: [String] = [] |
| contents.enumerateLines { line, _ in |
| lines.append(line) |
| } |
| return lines |
| } |
|
|
| private static func isRegularFile(_ url: URL) -> Bool { |
| (try? url.resourceValues(forKeys: [.isRegularFileKey]).isRegularFile) == true |
| } |
| } |
|
|
| enum ImportJSON { |
| static func object(from url: URL) throws -> Any { |
| let fileData = try ImportFiles.data(from: url) |
| return try JSONSerialization.jsonObject(with: fileData) |
| } |
|
|
| static func object(fromLine line: String) throws -> Any { |
| try JSONSerialization.jsonObject(with: Data(line.utf8)) |
| } |
| } |
|
|
