File size: 6,259 Bytes
e06e57f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 | #!/usr/bin/env python3
"""
compare_nll_vs_generative.py -- put the NLL proxy and the generative benchmark
side by side and judge three PRE-REGISTERED predictions against numeric
thresholds fixed before the numbers were seen.
usage: compare_nll_vs_generative.py <longeval_out_dir> <nll_ref_dir> [> EVAL_COMPARISON.md]
Why thresholds up front: at 100 prompts per (model, depth) cell the 95% Wilson
half-width is ~+-10 points, so "arm C beats arm B" is only meaningful relative to
that resolution. Judging after the fact would be post-hoc.
The two metrics are NOT on a common scale -- one is nats of NLL margin, the other
a hit rate -- so only ORDERINGS are compared, never absolute values.
"""
import csv
import math
import os
import sys
# NLL probe dir -> generative checkpoint key
NLL_MODEL = {
"probe_run7_yarn4": "base",
"probe_arm_a": "arm_a",
"probe_arm_b": "arm_b",
"probe_arm_c": "arm_c",
}
PFRAC_DEPTH = {"p0": 0.0, "p0.25": 0.25, "p0.5": 0.5, "p0.75": 0.75, "p1": 1.0}
def wilson(h, n):
if not n:
return float("nan")
z, p = 1.96, h / n
return 100.0 * z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / (1 + z * z / n)
def load_nll(ref):
"""{model: {depth: margin_nats}} margin = nll_null - nll (higher = better)."""
out = {}
for d, key in NLL_MODEL.items():
p = os.path.join(ref, "logs", d, "reach.csv")
if not os.path.exists(p):
continue
out[key] = {}
for r in csv.DictReader(open(p)):
dep = PFRAC_DEPTH.get(r["p_frac"])
if dep is None:
continue
out[key][dep] = float(r["nll_null"]) - float(r["nll"])
return out
def load_gen(out_dir):
"""{model: {depth: (hits, n)}} from depth_summary.csv."""
p = os.path.join(out_dir, "depth_summary.csv")
if not os.path.exists(p):
sys.exit(f"missing {p} -- run run_longeval.sh first")
g = {}
for r in csv.DictReader(open(p)):
g.setdefault(r["model"], {})[float(r["depth"])] = (int(r["hits"]), int(r["n"]))
return g
def main():
if len(sys.argv) < 3:
sys.exit(__doc__)
out_dir, ref = sys.argv[1], sys.argv[2]
nll, gen = load_nll(ref), load_gen(out_dir)
models = [m for m in ("base", "arm_a", "arm_b", "arm_c") if m in gen]
depths = sorted({d for m in gen.values() for d in m})
pct = lambda hn: 100.0 * hn[0] / hn[1] if hn[1] else float("nan")
G = lambda m, d: pct(gen[m][d]) if m in gen and d in gen[m] else float("nan")
print("# NLL proxy vs generative benchmark\n")
print("Two different measurements: NLL margin in nats (teacher-forced, answer")
print("already in the input) vs hit rate on generated text. Only the ORDERINGS")
print("are comparable.\n")
print("## Generative accuracy %, pooled by depth across tasks (+-95% Wilson)\n")
print("| depth | " + " | ".join(models) + " |")
print("|---" * (len(models) + 1) + "|")
for d in depths:
cells = []
for m in models:
if m in gen and d in gen[m]:
h, n = gen[m][d]
cells.append(f"{pct((h,n)):.1f} ±{wilson(h,n):.1f} (n={n})")
else:
cells.append("-")
print(f"| {d:.2f} | " + " | ".join(cells) + " |")
print("\n## NLL margin (nats), same models and depths\n")
print("| depth | " + " | ".join(models) + " |")
print("|---" * (len(models) + 1) + "|")
for d in depths:
print(f"| {d:.2f} | " + " | ".join(
(f"{nll[m][d]:.2f}" if m in nll and d in nll[m] else "-") for m in models) + " |")
# ---- pre-registered predictions -------------------------------------------
print("\n## Pre-registered predictions\n")
verdicts = []
# 1. depth 0.0 ~ 0% everywhere; conditional on the positive control
d0 = {m: G(m, 0.0) for m in models}
worst = max((v for v in d0.values() if v == v), default=float("nan"))
ctrl = {m: G(m, 1.0) for m in models}
ctrl_ok = any(v > 0 for v in ctrl.values() if v == v)
v1 = ("CONFIRMED" if worst <= 5 else "REFUTED" if worst >= 15 else "INCONCLUSIVE")
if not ctrl_ok:
v1 = "UNINTERPRETABLE (positive control failed)"
verdicts.append(("1. depth 0.0 ~= 0% for every model", v1,
f"max over models = {worst:.1f}%; positive control (any model >0% at depth 1.0) = "
f"{'PASS' if ctrl_ok else 'FAIL'}"))
# 2. arm C >> arm B at depth 0.25
if "arm_c" in gen and "arm_b" in gen:
diff2 = G("arm_c", 0.25) - G("arm_b", 0.25)
v2 = ("CONFIRMED" if diff2 >= 20 else "REFUTED" if abs(diff2) <= 5 else "INCONCLUSIVE")
verdicts.append(("2. arm C >> arm B at depth 0.25", v2,
f"armC {G('arm_c',0.25):.1f}% - armB {G('arm_b',0.25):.1f}% = {diff2:+.1f}pp "
f"(NLL said 2.80 vs 0.53 nats, ~9.5 sigma)"))
# 3. arm C non-monotonicity: depth 0.25 > depth 0.5
if "arm_c" in gen:
diff3 = G("arm_c", 0.25) - G("arm_c", 0.5)
v3 = ("CONFIRMED" if diff3 >= 20 else "REFUTED" if abs(diff3) <= 5 else "INCONCLUSIVE")
verdicts.append(("3. arm C depth 0.25 > depth 0.5", v3,
f"{G('arm_c',0.25):.1f}% - {G('arm_c',0.5):.1f}% = {diff3:+.1f}pp "
f"(NLL said 1.0865 nats at 6.87 sigma)"))
print("| prediction | verdict | evidence |")
print("|---|---|---|")
for name, v, ev in verdicts:
print(f"| {name} | **{v}** | {ev} |")
# ---- ordering agreement ----------------------------------------------------
print("\n## Ordering agreement (the only comparable thing)\n")
for m in models:
if m not in nll:
continue
gd = [(d, G(m, d)) for d in depths if G(m, d) == G(m, d)]
nd = [(d, nll[m][d]) for d in depths if d in nll[m]]
go = [d for d, _ in sorted(gd, key=lambda x: -x[1])]
no = [d for d, _ in sorted(nd, key=lambda x: -x[1])]
same = "AGREE" if go == no else "DIFFER"
print(f"- **{m}**: generative best->worst {go}; NLL best->worst {no} -> {same}")
if all(v == 0.0 for _, v in gd):
print(f" (all-zero generative: ordering is undefined, not disagreeing)")
if __name__ == "__main__":
main()
|