File size: 2,575 Bytes
e46c127 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | #!/usr/bin/env python
"""Merge results/*.json into one markdown table (model x task). Usage: python summarize.py > results/summary.md"""
import glob, json, collections
NAMES = {
"ckpt/qwen3.5-0.8b-nli": "Qwen3.5-0.8B full FT",
"ckpt/qwen3.5-2b-nli": "Qwen3.5-2B full FT",
"ckpt/qwen3.5-4b-nli": "Qwen3.5-4B full FT",
"ckpt/qwen3.5-2b-nli-headonly": "Qwen3.5-2B head-only",
"dleemiller/ModernCE-large-nli": "ModernCE-large-nli (ref)",
}
MC = ["gpqa", "mmlu", "arc_easy", "arc_challenge", "winogrande", "chess"]
res = collections.defaultdict(dict)
for f in sorted(glob.glob("results/*.json")):
if "train_" in f or "smoke" in f or "fewshot" in f: # few-shot runs use an MMLU subsample; see README
continue
for m, r in json.load(open(f)).items():
res[m].update(r)
order = [m for m in NAMES if m in res] + [m for m in res if m not in NAMES]
f3 = lambda x: f"{x:.3f}"
print("### NLI sanity (accuracy)\n")
print("| model | MNLI-m | MNLI-mm |\n|---|---|---|")
for m in order:
mn = res[m].get("mnli")
if mn:
print(f"| {NAMES.get(m, m)} | {f3(mn['validation_matched']['acc'])} | {f3(mn['validation_mismatched']['acc'])} |")
print("\n### Multiple choice, rerank without reference (blog #3): premise = question, pick argmax P(entailment)\n")
print("| model | " + " | ".join(MC) + " |\n|---|" + "---|" * len(MC))
rb = {t: next((res[m][t]["random_baseline"] for m in order if t in res[m] and "random_baseline" in res[m][t]), None) for t in MC}
print("| random | " + " | ".join(f3(rb[t]) if rb[t] is not None else "-" for t in MC) + " |")
for m in order:
print(f"| {NAMES.get(m, m)} | " + " | ".join(f3(res[m][t]["rerank_acc"]) if t in res[m] else "-" for t in MC) + " |")
print("\n### Multiple choice, grading with reference (blog #6): premise = question + gold, entailment <=> option is gold (acc / F1)\n")
print("| model | " + " | ".join(MC) + " |\n|---|" + "---|" * len(MC))
for m in order:
print(f"| {NAMES.get(m, m)} | " + " | ".join(f"{f3(res[m][t]['grade_acc'])} / {f3(res[m][t]['grade_f1'])}" if t in res[m] else "-" for t in MC) + " |")
print("\n### GSM8K (200 test questions, candidates from Qwen3.5-4B: greedy + 4 samples)\n")
print("| model | greedy | pass@1 | maj@4 | NLI rerank@4 | oracle@4 | grade acc / F1 |\n|---|---|---|---|---|---|---|")
for m in order:
g = res[m].get("gsm8k")
if g:
print(f"| {NAMES.get(m, m)} | {f3(g['greedy_acc'])} | {f3(g['sample_pass1'])} | {f3(g['maj@4'])} | {f3(g['nli_rerank@4'])} | {f3(g['oracle@4'])} | {f3(g['grade_acc'])} / {f3(g['grade_f1'])} |")
|