Buckets:
| """Report the statistics that actually rank this leaderboard. | |
| Measured across 19 submitted models on 2026-08-12: | |
| corr( mean L2 , sd L2 ) = -0.913 | |
| corr( mean L2 , mean|L2| ) = -0.926 | |
| corr( mean L2 , P(L2<-0.05) ) = -0.963 | |
| The field is sorted, monotonically, by how much its thoughts move the teacher. | |
| That matters for screening cost: the duel margin needs n~2080 to resolve +0.003, | |
| but these three are absolute values and proportions with much smaller spread, so | |
| a few hundred turns already separates arms that the margin cannot. | |
| Use them as a leading indicator, never as the verdict -- the crown is decided on | |
| the paired margin and nothing else. An arm that improves influence but not the | |
| margin has not won anything. | |
| python3 harness/influence.py --l2 out/l2_king.json out/l2_arm1.json | |
| python3 harness/influence.py --l2 out/*.json --paired-to out/l2_king.json | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import math | |
| import pathlib | |
| import statistics as st | |
| def load(path: pathlib.Path) -> dict[str, float]: | |
| d = json.load(open(path)) | |
| pt = d.get("per_turn") | |
| if not pt: | |
| raise SystemExit(f"{path} has no `per_turn` map — rescore with current screen_l2.py") | |
| out = {} | |
| for turn, v in pt.items(): | |
| x = v.get("l2") if isinstance(v, dict) else v | |
| if x is not None: | |
| out[turn] = float(x) | |
| return out | |
| def describe(name: str, v: dict[str, float]) -> tuple: | |
| xs = list(v.values()) | |
| n = len(xs) | |
| mean = st.mean(xs) | |
| sd = st.stdev(xs) if n > 1 else float("nan") | |
| return (name, n, mean, sd / math.sqrt(n) if n > 1 else float("nan"), sd, | |
| st.mean(abs(x) for x in xs), sum(1 for x in xs if x < -0.05) / n, | |
| st.median(xs)) | |
| def main() -> None: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--l2", nargs="+", required=True) | |
| ap.add_argument("--paired-to", help="reference arm for a turn-matched comparison") | |
| args = ap.parse_args() | |
| arms = {} | |
| for p in args.l2: | |
| path = pathlib.Path(p) | |
| arms[path.stem.replace("l2_", "")] = load(path) | |
| print(f"{'arm':<18} {'n':>6} {'mean L2':>10} {'se':>9} {'sd':>8} " | |
| f"{'mean|L2|':>9} {'P(<-.05)':>9} {'median':>9}") | |
| for name, v in arms.items(): | |
| _, n, m, se, sd, ma, pb, md = describe(name, v) | |
| print(f"{name:<18} {n:>6} {m:>+10.5f} {se:>9.5f} {sd:>8.4f} " | |
| f"{ma:>9.5f} {pb:>9.2%} {md:>+9.5f}") | |
| if not args.paired_to: | |
| return | |
| ref_name = pathlib.Path(args.paired_to).stem.replace("l2_", "") | |
| ref = arms.get(ref_name) | |
| if ref is None: | |
| raise SystemExit(f"--paired-to {ref_name} not among the loaded arms") | |
| print(f"\nturn-matched vs {ref_name}:") | |
| print(f"{'arm':<18} {'n':>6} {'d mean':>10} {'se':>9} {'z':>7} " | |
| f"{'d mean|L2|':>11} {'d P(<-.05)':>11}") | |
| for name, v in arms.items(): | |
| if name == ref_name: | |
| continue | |
| shared = sorted(set(v) & set(ref)) | |
| if len(shared) < 2: | |
| print(f"{name:<18} {len(shared):>6} (no turn overlap)") | |
| continue | |
| d = [v[t] - ref[t] for t in shared] | |
| m = st.mean(d) | |
| se = st.stdev(d) / math.sqrt(len(d)) | |
| dma = (st.mean(abs(v[t]) for t in shared) | |
| - st.mean(abs(ref[t]) for t in shared)) | |
| dpb = (sum(1 for t in shared if v[t] < -0.05) | |
| - sum(1 for t in shared if ref[t] < -0.05)) / len(shared) | |
| print(f"{name:<18} {len(shared):>6} {m:>+10.5f} {se:>9.5f} {m/se:>7.2f} " | |
| f"{dma:>+11.5f} {dpb:>+11.2%}") | |
| print("\n negative d mean|L2| and d P(<-.05) = the thought moves the teacher " | |
| "less\n the crown is still decided by `d mean` alone.") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 3.76 kB
- Xet hash:
- 7c229da4a269286da2fb69f1224c5be79b815ddd8f02de604f7f555c06cba24e
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.