Buckets:
| """Would the stage-1 screen have made the right call on the duels we already know? | |
| Leave-one-out: for each published duel, fit the ΔL1lift -> margin predictor on | |
| every OTHER duel, predict the held-out one from its ΔL1lift alone, and compare | |
| the screen's decision to what the validator actually ruled. | |
| This is the honest test of the screen. It needs no GPU: ΔL1lift is recomputed | |
| from the stored pair components exactly as screen.py would measure it live. | |
| """ | |
| from __future__ import annotations | |
| import statistics as st | |
| from analyze import load | |
| def ols(xs, ys): | |
| mx, my = st.mean(xs), st.mean(ys) | |
| sxx = sum((x - mx) ** 2 for x in xs) | |
| slope = sum((x - mx) * (y - my) for x, y in zip(xs, ys)) / sxx | |
| return slope, my - slope * mx | |
| def main() -> None: | |
| rows = [r for r in load() if r["se"] is not None] | |
| xs = [r["c"]["l1_clipped"] - r["k"]["l1_clipped"] for r in rows] | |
| ys = [r["margin"] for r in rows] | |
| n = len(rows) | |
| crown = 3 * st.median(r["se"] for r in rows) | |
| SUBMIT = 0.035 | |
| preds = [] | |
| for i in range(n): | |
| sx = xs[:i] + xs[i + 1:] | |
| sy = ys[:i] + ys[i + 1:] | |
| m, b = ols(sx, sy) | |
| preds.append(b + m * xs[i]) | |
| err = [p - y for p, y in zip(preds, ys)] | |
| print(f"leave-one-out over {n} scored duels") | |
| print(f" mean |error| = {st.mean(abs(e) for e in err):.5f} " | |
| f"sd = {st.stdev(err):.5f} (typical duel se = {crown/3:.5f})") | |
| # Decision quality: screen says PROMOTE (>=SUBMIT) / MARGINAL / REJECT, | |
| # truth is whether the validator crowned it. | |
| print(f"\n screen gate: PROMOTE if margin_hat >= {SUBMIT:.3f}, " | |
| f"MARGINAL if >= {crown:.4f}, else REJECT") | |
| print(f"\n {'bucket':<12}{'n':>5}{'crowns':>8}{'true margin >= crown':>23}") | |
| buckets = {"PROMOTE": [], "MARGINAL": [], "REJECT": []} | |
| for r, p in zip(rows, preds): | |
| k = "PROMOTE" if p >= SUBMIT else "MARGINAL" if p >= crown else "REJECT" | |
| buckets[k].append((r, p)) | |
| for k in ("PROMOTE", "MARGINAL", "REJECT"): | |
| g = buckets[k] | |
| if not g: | |
| print(f" {k:<12}{0:>5}") | |
| continue | |
| w = sum(1 for r, _ in g if r["wins"]) | |
| above = sum(1 for r, _ in g if r["margin"] >= crown) | |
| print(f" {k:<12}{len(g):>5}{w:>8}{above:>23}") | |
| missed = [(r, p) for r, p in zip(rows, preds) | |
| if r["wins"] and p < crown] | |
| print(f"\n crowns the screen would have REJECTED: {len(missed)}") | |
| for r, p in missed: | |
| print(f" {r['id']} margin_hat={p:+.4f} true={r['margin']:+.4f} {r['repo'][:40]}") | |
| wasted = [(r, p) for r, p in zip(rows, preds) | |
| if p >= SUBMIT and r["margin"] < crown] | |
| print(f" duels the screen would have wasted (PROMOTE but truly below crown): " | |
| f"{len(wasted)}") | |
| for r, p in wasted[:6]: | |
| print(f" {r['id']} margin_hat={p:+.4f} true={r['margin']:+.4f} {r['repo'][:40]}") | |
| kept = sum(len(buckets[k]) for k in ("PROMOTE", "MARGINAL")) | |
| print(f"\n full duels required: {kept}/{n} " | |
| f"({kept/n*100:.0f}%) — the screen removes {n-kept} of {n} " | |
| f"({(n-kept)/n*100:.0f}%) before any 8-GPU time is spent") | |
| # Stage 1 exists to remove obvious losers cheaply, so it must be tuned for | |
| # RECALL: a missed crown costs a reign, a false promote costs one duel. | |
| # Sweep the gate and show the trade-off explicitly. | |
| pos = [(r, p) for r, p in zip(rows, preds) if r["margin"] >= crown] | |
| print(f"\n=== gate sweep (positives = true margin >= {crown:.4f}, n={len(pos)}) ===") | |
| print(f" {'gate':>8}{'promoted':>10}{'% removed':>11}{'crowns kept':>13}" | |
| f"{'cost/candidate':>16}") | |
| for gate in (0.035, 0.030, 0.025, 0.020, 0.015, 0.010, 0.005, 0.000): | |
| keep = [(r, p) for r, p in zip(rows, preds) if p >= gate] | |
| got = sum(1 for r, p in keep if r["margin"] >= crown) | |
| # $5 screen on every candidate + $33 full duel on the promoted share | |
| cost = 5 + 33 * len(keep) / n | |
| print(f" {gate:>8.3f}{len(keep):>10}{(n-len(keep))/n*100:>10.0f}%" | |
| f"{got:>8}/{len(pos):<4}{cost:>15.2f}") | |
| print("\n A missed crown costs a reign (~34 TAO/day); a false promote costs " | |
| "one $33 duel.\n Tune for recall, not precision.") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 4.28 kB
- Xet hash:
- c3ae138f196f0998446f32a028f2e41666ef40befcce4e86f66728f727c43bdf
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.