File size: 2,386 Bytes
4977db2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | import Foundation
struct GeminiJSONLParser: ImportParser {
func parse(sourceURL: URL, dedupIndex: DedupIndex) throws -> ImportParseResult {
let files = try ImportFiles.immediateFiles(
at: sourceURL,
expected: "eine Gemini-.jsonl-Datei oder einen Ordner mit .jsonl-Dateien",
directFileMatches: { $0.pathExtension.lowercased() == "jsonl" },
directoryFileMatches: { $0.pathExtension.lowercased() == "jsonl" }
)
var accumulator = ImportAccumulator(
minimumLength: 20,
maximumLength: 10_000,
dedupIndex: dedupIndex
)
for fileURL in files {
let notebook = fileURL.deletingPathExtension().lastPathComponent
for line in try ImportFiles.lines(from: fileURL) {
let root: Any
do {
root = try ImportJSON.object(fromLine: line)
} catch {
accumulator.skipMalformed()
continue
}
guard let object = root as? [String: Any] else {
accumulator.skipMalformed()
continue
}
guard object["role"] as? String == "user" else { continue }
let text: String
if object["content"] == nil {
// Python's obj.get("content", "") treats a missing value
// as an empty prompt, which is scanned and skipped_short.
text = ""
} else if let content = object["content"] as? String {
text = content
} else {
accumulator.skipMalformed()
continue
}
accumulator.accept(
text,
sourcePrefix: "gemini-raw",
tags: "gemini,\(notebook.lowercased())",
promptedAt: firstTimestamp(in: object)
)
}
}
return accumulator.result()
}
private func firstTimestamp(in object: [String: Any]) -> String? {
for key in ["timestamp", "date", "created_at", "create_time"] {
if let timestamp = ImportTimestamp.normalize(object[key]) {
return timestamp
}
}
return nil
}
}
|