| """Combine k-fold runs (each scored only on its held-out fold) into one run over the full dev set. |
| |
| python -m eval.combine_cv --runs results/exp_devcv_lora_f* --name exp_devcv_lora_all |
| |
| Writes results/<name>/{metrics.json, report.md, predictions.jsonl} in the usual layout, so eval.summarize can put it |
| next to the zero-shot baseline. Fails if a question is missing or scored twice. |
| """ |
| from __future__ import annotations |
|
|
| import argparse |
| import glob |
| import json |
| from pathlib import Path |
|
|
| from .common import ROOT, read_jsonl, write_jsonl |
| from .metrics import markdown_report |
| from .run_eval import metrics_by_dataset |
|
|
|
|
| def main() -> None: |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--runs", nargs="+", required=True) |
| ap.add_argument("--name", required=True) |
| ap.add_argument("--data", default="data/processed/sinhalammlu_dev.jsonl") |
| args = ap.parse_args() |
| dirs = sorted(Path(d) for pat in args.runs for d in glob.glob(pat)) |
| records, first = [], None |
| for d in dirs: |
| records += read_jsonl(d / "predictions.jsonl") |
| first = first or json.loads((d / "metrics.json").read_text()) |
| items = {r["uid"]: r for r in read_jsonl(ROOT / args.data)} |
| scored = [r["uid"] for r in records if r["perm_k"] == 0] |
| assert len(scored) == len(set(scored)), "a question was scored by more than one fold" |
| assert set(scored) == set(items), f"{len(set(items) - set(scored))} dev questions were not scored" |
| out = ROOT / "results" / args.name |
| out.mkdir(parents=True, exist_ok=True) |
| metrics = {"by_dataset": metrics_by_dataset(items, records), |
| "run": first["run"] | {"run_name": args.name, "combined_from": [d.name for d in dirs]}} |
| write_jsonl(out / "predictions.jsonl", records) |
| (out / "metrics.json").write_text(json.dumps(metrics, indent=2, ensure_ascii=False)) |
| (out / "report.md").write_text("\n".join(markdown_report(f"{args.name} / {k}", m) |
| for k, m in metrics["by_dataset"].items())) |
| o = metrics["by_dataset"]["sinhalammlu"]["overall"] |
| print(f"{args.name}: dev acc={o['acc']:.4f} vote={o['acc_vote']:.4f} over {o['n']} questions from {len(dirs)} folds") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
