Download repair_cache_jsonl.py from Tinghong-Ye22/script: direct link, hf CLI and curl.
- Browser
- Download file 4.61 kB
-
https://huggingface.co/Tinghong-Ye22/script/resolve/main/repair_cache_jsonl.py
- Command line
-
hf download hf://Tinghong-Ye22/script/repair_cache_jsonl.py
-
curl -L -o repair_cache_jsonl.py https://huggingface.co/Tinghong-Ye22/script/resolve/main/repair_cache_jsonl.py
4.61 kB
| #!/usr/bin/env python3 | |
| """Repair cached confidence/reason fields in JSONL files in place.""" | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| from collections import Counter | |
| from pathlib import Path | |
| from tempfile import NamedTemporaryFile | |
| from typing import Dict, Iterable, Iterator, List, Tuple | |
| ROOT = Path(__file__).resolve().parent.parent | |
| if str(ROOT) not in sys.path: | |
| sys.path.insert(0, str(ROOT)) | |
| from answer_utils import repair_confidence_reason_record_fields | |
| def parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser( | |
| description="Repair cache JSONL files by backfilling conf/reason from raw output." | |
| ) | |
| parser.add_argument( | |
| "paths", | |
| nargs="*", | |
| default=["cache/benchmarks"], | |
| help="Files or directories to scan. Defaults to cache/benchmarks.", | |
| ) | |
| parser.add_argument( | |
| "--dry-run", | |
| action="store_true", | |
| help="Scan and report changes without rewriting files.", | |
| ) | |
| parser.add_argument( | |
| "--verbose", | |
| action="store_true", | |
| help="Print per-file repair counts.", | |
| ) | |
| return parser.parse_args() | |
| def iter_jsonl_files(paths: Iterable[str]) -> Iterator[Path]: | |
| seen = set() | |
| for raw_path in paths: | |
| path = Path(raw_path) | |
| if path.is_file() and path.suffix == ".jsonl": | |
| resolved = path.resolve() | |
| if resolved not in seen: | |
| seen.add(resolved) | |
| yield path | |
| continue | |
| if path.is_dir(): | |
| for item in sorted(path.rglob("*.jsonl")): | |
| resolved = item.resolve() | |
| if resolved in seen: | |
| continue | |
| seen.add(resolved) | |
| yield item | |
| def changed_field_counts(before: Dict, after: Dict) -> Counter: | |
| counts: Counter = Counter() | |
| keys = set(before.keys()) | set(after.keys()) | |
| for key in keys: | |
| if before.get(key) == after.get(key): | |
| continue | |
| if key == "conf" or key.endswith("_conf"): | |
| counts["conf_fields"] += 1 | |
| elif key == "reason" or key.endswith("_reason"): | |
| counts["reason_fields"] += 1 | |
| else: | |
| counts["other_fields"] += 1 | |
| return counts | |
| def repair_file(path: Path, dry_run: bool = False) -> Tuple[bool, Counter]: | |
| file_counts: Counter = Counter() | |
| repaired_lines: List[str] = [] | |
| with path.open("r", encoding="utf-8") as handle: | |
| for line_no, raw_line in enumerate(handle, start=1): | |
| stripped = raw_line.strip() | |
| if not stripped: | |
| repaired_lines.append(raw_line) | |
| continue | |
| try: | |
| record = json.loads(stripped) | |
| except json.JSONDecodeError as exc: | |
| raise ValueError(f"{path}:{line_no}: invalid json: {exc}") from exc | |
| repaired = repair_confidence_reason_record_fields(record) | |
| if repaired != record: | |
| file_counts["records_changed"] += 1 | |
| file_counts.update(changed_field_counts(record, repaired)) | |
| repaired_lines.append(json.dumps(repaired, ensure_ascii=False) + "\n") | |
| changed = bool(file_counts["records_changed"]) | |
| if changed and not dry_run: | |
| with NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as tmp: | |
| tmp.writelines(repaired_lines) | |
| tmp_path = Path(tmp.name) | |
| tmp_path.replace(path) | |
| return changed, file_counts | |
| def main() -> int: | |
| args = parse_args() | |
| totals: Counter = Counter() | |
| files = list(iter_jsonl_files(args.paths)) | |
| if not files: | |
| print("No .jsonl files found.") | |
| return 0 | |
| for path in files: | |
| changed, file_counts = repair_file(path, dry_run=args.dry_run) | |
| totals["files_scanned"] += 1 | |
| if changed: | |
| totals["files_changed"] += 1 | |
| totals.update(file_counts) | |
| if args.verbose and changed: | |
| rel = path.as_posix() | |
| print( | |
| f"{rel}: records={file_counts['records_changed']} " | |
| f"conf={file_counts['conf_fields']} reason={file_counts['reason_fields']}" | |
| ) | |
| mode = "dry-run" if args.dry_run else "rewritten" | |
| print( | |
| f"[{mode}] files_scanned={totals['files_scanned']} " | |
| f"files_changed={totals['files_changed']} " | |
| f"records_changed={totals['records_changed']} " | |
| f"conf_fields={totals['conf_fields']} " | |
| f"reason_fields={totals['reason_fields']}" | |
| ) | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |