File size: 3,085 Bytes
5264032 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 | #!/usr/bin/env python3
"""Export flat CSV versions of the dataset and the summary tables."""
from __future__ import annotations
import csv
import json
import pathlib
ROOT = pathlib.Path(__file__).resolve().parent.parent
DATASET = ROOT / "dataset"
RESULTS = ROOT / "results"
def export_dataset() -> None:
items = [json.loads(l) for l in (DATASET / "msq49.jsonl").open(encoding="utf-8")]
path = DATASET / "msq49.csv"
with path.open("w", newline="", encoding="utf-8") as fh:
w = csv.writer(fh)
w.writerow(["id", "cls", "domain", "question", "stem",
"n_positions", "position_texts", "position_schools",
"majority_schools", "minority_school", "sources", "notes"])
for it in items:
w.writerow([
it["id"], it["cls"], it["domain"], it["question"], it["stem"],
len(it["positions"]),
" | ".join(p["txt"] for p in it["positions"]),
" | ".join("/".join(p["schools"]) for p in it["positions"]),
"/".join(it["majority_schools"]), it["minority_school"] or "",
" ; ".join(it["sources"]), it.get("notes", ""),
])
print(f"wrote {path} ({len(items)} rows)")
def export_tables() -> None:
p = RESULTS / "metrics.json"
if not p.exists():
print("no metrics.json yet")
return
metrics = json.loads(p.read_text())
rows = []
for model, rep in metrics.items():
h = rep.get("headline", {})
n = rep.get("mcq", {}).get("mcq_neutral", {})
s = rep.get("mcq", {}).get("mcq_specified", {})
ft = rep.get("free_text", {})
rows.append(dict(
model=model,
PRR=h.get("PRR"), PRR_A=h.get("PRR_A"), PRR_B=h.get("PRR_B"),
PRR_C=h.get("PRR_C"), PRR_mass=h.get("PRR_mass"),
MSQ=h.get("MSQ"), blind_spot_gap=h.get("blind_spot_gap"),
acc_control_D=h.get("acc_D"),
conf_contested=h.get("conf_contested"),
conf_control=h.get("conf_control"),
disclosure_default=ft.get("ft_default", {}).get("ALL", {}).get("disclosure_rate"),
disclosure_technical=ft.get("ft_technical", {}).get("ALL", {}).get("disclosure_rate"),
disclosure_two_schools=ft.get("ft_two_schools", {}).get("ALL", {}).get("disclosure_rate"),
attr_two_schools=ft.get("ft_two_schools", {}).get("ALL", {}).get("plural_attribution_rate"),
clarification_default=ft.get("ft_default", {}).get("ALL", {}).get("clarification_rate"),
answer_changed_rate=rep.get("answer_change", {}).get("changed_rate"),
mean_max_prob=n.get("mean_max_prob"),
mean_entropy=n.get("mean_entropy"),
))
path = RESULTS / "summary_tables.csv"
with path.open("w", newline="", encoding="utf-8") as fh:
w = csv.DictWriter(fh, fieldnames=list(rows[0].keys()))
w.writeheader()
w.writerows(rows)
print(f"wrote {path} ({len(rows)} rows)")
if __name__ == "__main__":
export_dataset()
export_tables()
|