File size: 1,529 Bytes
4977db2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | import Foundation
struct CodexHistoryParser: ImportParser {
func parse(sourceURL: URL, dedupIndex: DedupIndex) throws -> ImportParseResult {
let files = try ImportFiles.immediateFiles(
at: sourceURL,
expected: "eine Codex history.jsonl oder den zugehörigen Konfigurationsordner",
directFileMatches: { $0.pathExtension.lowercased() == "jsonl" },
directoryFileMatches: { $0.lastPathComponent == "history.jsonl" }
)
var accumulator = ImportAccumulator(
minimumLength: 20,
maximumLength: 10_000,
dedupIndex: dedupIndex
)
for fileURL in files {
for line in try ImportFiles.lines(from: fileURL) {
let root: Any
do {
root = try ImportJSON.object(fromLine: line)
} catch {
accumulator.skipMalformed()
continue
}
guard
let object = root as? [String: Any],
let text = object["text"] as? String
else {
accumulator.skipMalformed()
continue
}
accumulator.accept(
text,
sourcePrefix: "codex-raw",
tags: "codex",
promptedAt: ImportTimestamp.normalize(object["ts"])
)
}
}
return accumulator.result()
}
}
|