| |
| """Export flat CSV versions of the dataset and the summary tables.""" |
| from __future__ import annotations |
|
|
| import csv |
| import json |
| import pathlib |
|
|
| ROOT = pathlib.Path(__file__).resolve().parent.parent |
| DATASET = ROOT / "dataset" |
| RESULTS = ROOT / "results" |
|
|
|
|
| def export_dataset() -> None: |
| items = [json.loads(l) for l in (DATASET / "msq49.jsonl").open(encoding="utf-8")] |
| path = DATASET / "msq49.csv" |
| with path.open("w", newline="", encoding="utf-8") as fh: |
| w = csv.writer(fh) |
| w.writerow(["id", "cls", "domain", "question", "stem", |
| "n_positions", "position_texts", "position_schools", |
| "majority_schools", "minority_school", "sources", "notes"]) |
| for it in items: |
| w.writerow([ |
| it["id"], it["cls"], it["domain"], it["question"], it["stem"], |
| len(it["positions"]), |
| " | ".join(p["txt"] for p in it["positions"]), |
| " | ".join("/".join(p["schools"]) for p in it["positions"]), |
| "/".join(it["majority_schools"]), it["minority_school"] or "", |
| " ; ".join(it["sources"]), it.get("notes", ""), |
| ]) |
| print(f"wrote {path} ({len(items)} rows)") |
|
|
|
|
| def export_tables() -> None: |
| p = RESULTS / "metrics.json" |
| if not p.exists(): |
| print("no metrics.json yet") |
| return |
| metrics = json.loads(p.read_text()) |
| rows = [] |
| for model, rep in metrics.items(): |
| h = rep.get("headline", {}) |
| n = rep.get("mcq", {}).get("mcq_neutral", {}) |
| s = rep.get("mcq", {}).get("mcq_specified", {}) |
| ft = rep.get("free_text", {}) |
| rows.append(dict( |
| model=model, |
| PRR=h.get("PRR"), PRR_A=h.get("PRR_A"), PRR_B=h.get("PRR_B"), |
| PRR_C=h.get("PRR_C"), PRR_mass=h.get("PRR_mass"), |
| MSQ=h.get("MSQ"), blind_spot_gap=h.get("blind_spot_gap"), |
| acc_control_D=h.get("acc_D"), |
| conf_contested=h.get("conf_contested"), |
| conf_control=h.get("conf_control"), |
| disclosure_default=ft.get("ft_default", {}).get("ALL", {}).get("disclosure_rate"), |
| disclosure_technical=ft.get("ft_technical", {}).get("ALL", {}).get("disclosure_rate"), |
| disclosure_two_schools=ft.get("ft_two_schools", {}).get("ALL", {}).get("disclosure_rate"), |
| attr_two_schools=ft.get("ft_two_schools", {}).get("ALL", {}).get("plural_attribution_rate"), |
| clarification_default=ft.get("ft_default", {}).get("ALL", {}).get("clarification_rate"), |
| answer_changed_rate=rep.get("answer_change", {}).get("changed_rate"), |
| mean_max_prob=n.get("mean_max_prob"), |
| mean_entropy=n.get("mean_entropy"), |
| )) |
| path = RESULTS / "summary_tables.csv" |
| with path.open("w", newline="", encoding="utf-8") as fh: |
| w = csv.DictWriter(fh, fieldnames=list(rows[0].keys())) |
| w.writeheader() |
| w.writerows(rows) |
| print(f"wrote {path} ({len(rows)} rows)") |
|
|
|
|
| if __name__ == "__main__": |
| export_dataset() |
| export_tables() |
|
|
