File size: 4,522 Bytes
b5ee25d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 | """
sanity_dense.py -- REF (dense) gate: does our pipeline reproduce the OFFICIAL
LLaDA-8B-Base EVAL.md numbers? (README section 0: compare to LLaDA official,
NOT the Sink-Aware Dense row, which is not self-consistent under one protocol.)
Reads the per-item converter summaries written by analysis/lmeval_to_items.py
(field acc_primary = the pre-registered primary metric per task) and diffs them
against official EVAL.md. If a task is off beyond sampling error, investigate the
pipeline BEFORE running any compression eval.
Official LLaDA-8B-Base (evaluation/EVAL.md, lm-eval):
arc_challenge 47.9 (acc, w/ CFG) | hellaswag 72.5 (acc_norm, w/ CFG)
winogrande 74.8 (acc, 5-shot) | piqa 74.4 (acc_norm, w/ CFG)
mmlu 65.9 (acc, w/o CFG, 5-shot) | gsm8k 70.0 (exact_match, 256/256/256)
Usage:
python sanity_dense.py # reads results/eval/ref/*_ref_*_summary.json
"""
import os
import sys
import glob
import math
import argparse
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import common as C
TARGET = { # official EVAL.md, primary metric per task -- LLaDA-8B-BASE
"arc_challenge": 47.9,
"hellaswag": 72.5,
"winogrande": 74.8,
"piqa": 74.4,
"mmlu": 65.9,
"gsm8k": 70.0,
# arc_easy: not tabulated in EVAL.md -> no target
}
# LLaDA-8B-INSTRUCT, evaluation/EVAL.md Instruct table, "Internal toolkit" row --
# that is the column the paper's Tab.1/Tab.2 report. (The OpenCompass row of the same
# table runs 0.2-2.4pp lower; we gate against the reported numbers, not the reproduction.)
# Only the four tasks our Instruct table shares with that table get a target.
# bbh_llada and ifeval are absent from it entirely -> no target, no gate.
TARGET_INSTRUCT = {
"mmlu_generative": 65.5,
"gsm8k": 69.4,
"humaneval_instruct": 49.4,
"mbpp_instruct": 41.0,
}
TARGETS = {"base": TARGET, "instruct": TARGET_INSTRUCT}
def wilson_hw(p, n, z=1.96):
return z * math.sqrt(max(p * (1 - p), 1e-12) / n) if n else float("nan")
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--glob", default=None)
ap.add_argument("--tol_pp", type=float, default=3.0)
ap.add_argument("--family", default="base", choices=list(TARGETS))
ap.add_argument("--out_root", default=None,
help="eval results root (default results/eval, i.e. the Base table)")
args = ap.parse_args()
root = os.path.dirname(os.path.abspath(__file__))
target = TARGETS[args.family]
out_root = args.out_root or os.path.join(root, "results", "eval")
pat = args.glob or os.path.join(out_root, "ref", "*_ref_*_summary.json")
files = sorted(glob.glob(pat))
if not files:
sys.exit(f"no REF summaries at {pat}; run eval/run_lmeval.py + "
"analysis/lmeval_to_items.py (arm=ref) first")
rows = {}
for fp in files:
s = C.load_json(fp)
t = s.get("task")
if not t or "acc_primary" not in s:
continue
if t not in rows or (s.get("n") or 0) > (rows[t].get("n") or 0):
rows[t] = s
print(f"{'task':<15} {'n':>6} {'metric':>10} {'ours':>7} {'EVAL.md':>8} "
f"{'delta':>7} {'95%CI':>7} flag")
print("-" * 72)
flags = []
for t in sorted(rows):
s = rows[t]
n = s["n"]
acc = 100.0 * (s.get("acc_primary") or 0)
ci = 100.0 * wilson_hw(s.get("acc_primary") or 0, n)
tgt = target.get(t)
if tgt is None:
print(f"{t:<15} {n:>6} {s['primary_metric']:>10} {acc:>7.2f} "
f"{'--':>8} {'--':>7} {ci:>7.2f} (no target)")
continue
d = abs(acc - tgt)
bad = d > (args.tol_pp + ci)
flags.append((t, round(d, 2))) if bad else None
print(f"{t:<15} {n:>6} {s['primary_metric']:>10} {acc:>7.2f} {tgt:>8.2f} "
f"{d:>7.2f} {ci:>7.2f} {'FLAG' if bad else 'ok'}")
out = {"family": args.family, "target_evalmd": target,
"ours": {t: {"n": rows[t]["n"], "acc_primary": rows[t].get("acc_primary"),
"metric": rows[t].get("primary_metric")} for t in rows},
"flags": flags, "git_hash": C.git_hash()}
C.dump_json(out, os.path.join(out_root,
f"{C.git_hash()}_sanity_dense_{args.family}.json"))
if flags:
print(f"\n!! FLAGGED {flags} -- investigate pipeline before compression eval.")
else:
print("\nAll tasks within tolerance of official EVAL.md -> pipeline sane.")
if __name__ == "__main__":
main()
|