| """The headline comparison, paired over the same 64 mixtures. |
| |
| Comparing +0.668 against the published +0.630 as a constant is weaker than it |
| needs to be: EAT's own proxy probe score exists for these exact 64 arms, so the |
| two proxies can be tested against the same target on the same mixtures with a |
| paired bootstrap over mixtures. That also lets us ask the question the campaign |
| was really for -- whether the two objectives agree with each other. |
| """ |
| import csv |
| import json |
| from pathlib import Path |
|
|
| import numpy as np |
| from scipy import stats |
|
|
| spec = json.loads(Path("/workspace/analysis/canonical64.json").read_text()) |
| curve = json.loads(Path("/workspace/analysis/transfer_curve.json").read_text()) |
|
|
| |
| eat_proxy = {} |
| for r in csv.DictReader(open("/workspace/analysis/eat-map-regmix/tables/runs.csv")): |
| |
| |
| if r.get("campaign") != "regmix256" or r.get("dist_id") in (None, ""): |
| continue |
| try: |
| if float(r["pretrain_seed"]) != 0.0: |
| continue |
| eat_proxy[int(float(r["dist_id"]))] = float(r["as20k_map"]) |
| except (ValueError, KeyError): |
| pass |
|
|
| dids = sorted(int(v["dist_id"]) for v in spec.values() if int(v["dist_id"]) in eat_proxy) |
| print(f"arms with both EAT proxy and MAE proxy: {len(dids)}") |
|
|
| base = np.array([spec[str(d)]["lora_mean"] for d in dids]) |
| eatp = np.array([eat_proxy[d] for d in dids]) |
|
|
| mae_by_step = {} |
| for row in curve: |
| mae_by_step[row["step"]] = row |
| final = max(mae_by_step) |
|
|
| |
| import glob |
| maep = [] |
| for d in dids: |
| p = glob.glob(f"/workspace/runs/mae-64/d{d}-s0/exports/step_{final:08d}/probe.json") |
| maep.append(json.loads(Path(p[0]).read_text())["probe/map"]) |
| maep = np.array(maep) |
|
|
|
|
| def rho(a, b): |
| return stats.spearmanr(a, b).statistic |
|
|
|
|
| r_eat, r_mae, r_cross = rho(eatp, base), rho(maep, base), rho(eatp, maep) |
| print(f"\nrho(EAT proxy -> EAT base) = {r_eat:+.3f} (published +0.630)") |
| print(f"rho(MAE proxy -> EAT base) = {r_mae:+.3f}") |
| print(f"rho(EAT proxy <-> MAE proxy) = {r_cross:+.3f} <- do the objectives agree?") |
|
|
| rng = np.random.default_rng(0) |
| d_transfer, d_cross = [], [] |
| for _ in range(4000): |
| i = rng.integers(0, len(dids), len(dids)) |
| d_transfer.append(rho(maep[i], base[i]) - rho(eatp[i], base[i])) |
| d_transfer = np.array(d_transfer) |
| print(f"\nDelta rho (MAE - EAT) as proxy for the EAT target:") |
| print(f" point {r_mae - r_eat:+.3f} 95% CI [{np.percentile(d_transfer,2.5):+.3f}, " |
| f"{np.percentile(d_transfer,97.5):+.3f}] P(MAE better) = {(d_transfer>0).mean():.3f}") |
|
|
| |
| cross = [r for r in curve if r["rho_base"] >= r_eat] |
| if cross: |
| c = min(cross, key=lambda r: r["clips"]) |
| print(f"\nMAE reaches EAT's converged rho ({r_eat:+.3f}) at {c['clips']:,} clips " |
| f"= {c['clips']/511872:.1f}x EAT's 512k budget") |
|
|
| print(f"\nnoise budget at each proxy's own endpoint:") |
| print(f" EAT 15M/512k : between-arm sd 0.00339, sigma_seed 0.00159, reliability 0.780") |
| f = mae_by_step[final] |
| print(f" MAE 15M/1.9M : between-arm sd {f['sd']:.5f}, sigma_seed {f['sigma']:.5f}, " |
| f"reliability {f['rel']:.3f}") |
| print(f"\ndisattenuated: EAT +0.718 (published) vs MAE {f['rho_dis']:+.3f}") |
|
|
| json.dump({"rho_eat_proxy": r_eat, "rho_mae_proxy": r_mae, "rho_cross": r_cross, |
| "delta": r_mae - r_eat, |
| "delta_ci": [float(np.percentile(d_transfer, 2.5)), |
| float(np.percentile(d_transfer, 97.5))], |
| "p_mae_better": float((d_transfer > 0).mean()), "n": len(dids)}, |
| open("/workspace/analysis/headline.json", "w"), indent=1) |
| print("\nwrote /workspace/analysis/headline.json") |
|
|
