Buckets:
| """Is there anything for a model merge to gain? Measure per-turn complementarity. | |
| Weight merging produces a model BETWEEN its parents. Blending the king with a | |
| weaker model therefore lands below the king — unless the two are complementary: | |
| strong on different turns, so a blend can inherit both strengths. | |
| Duel slices differ per duel (seeded by the reveal block hash), but turns recur | |
| across duels, so models can be compared on the turns they share. For every pair | |
| of models with enough overlap this reports | |
| * Spearman correlation of per-turn rank_term — high means the models agree on | |
| which turns are easy, so there is nothing to recombine; | |
| * the ORACLE gain: what per-turn max(A, B) would score over the better parent. | |
| Oracle selection is far stronger than any weight merge, so it is an UPPER | |
| BOUND on the opportunity. If the oracle gain is small, merging is dead. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import gzip | |
| import json | |
| import statistics as st | |
| from itertools import combinations | |
| import score | |
| from analyze import load as load_duels | |
| from replay import LEGACY_CONTRACT | |
| from screen import EVALS | |
| def per_turn_scores() -> dict[str, dict[str, float]]: | |
| """model repo -> {turn_id: mean rank_term over that turn's pairs}.""" | |
| out: dict[str, dict[str, float]] = {} | |
| for path in sorted(EVALS.glob("*.json.gz")): | |
| with gzip.open(path) as fh: | |
| d = json.load(fh) | |
| g = d["verdict"].get("gates", {}) | |
| clip = g.get("l1_clip", LEGACY_CONTRACT["l1_clip"]) | |
| w = g.get("l1_weight", 1.0) | |
| for side, repo in (("challenger_rows", d["request"]["challenger_repo"]), | |
| ("king_rows", d["request"]["king_repo"])): | |
| for row in d.get(side) or []: | |
| if not row.get("valid") or not row.get("pairs"): | |
| continue | |
| v = st.mean(score.lambda2(p) + w * score.clipped_l1_lift(p, clip) | |
| for p in row["pairs"]) | |
| # A model may appear on the same turn in several duels; average. | |
| cur = out.setdefault(repo, {}) | |
| cur[row["turn_id"]] = (cur[row["turn_id"]] + v) / 2 if row["turn_id"] in cur else v | |
| return out | |
| def spearman(xs, ys) -> float: | |
| def rank(v): | |
| order = sorted(range(len(v)), key=lambda i: v[i]) | |
| r = [0.0] * len(v) | |
| i = 0 | |
| while i < len(order): | |
| j = i | |
| while j + 1 < len(order) and v[order[j + 1]] == v[order[i]]: | |
| j += 1 | |
| for k in range(i, j + 1): | |
| r[order[k]] = (i + j) / 2.0 | |
| i = j + 1 | |
| return r | |
| rx, ry = rank(xs), rank(ys) | |
| mx, my = st.mean(rx), st.mean(ry) | |
| num = sum((a - mx) * (b - my) for a, b in zip(rx, ry)) | |
| den = (sum((a - mx) ** 2 for a in rx) * sum((b - my) ** 2 for b in ry)) ** 0.5 | |
| return num / den if den else 0.0 | |
| def main() -> None: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--min-overlap", type=int, default=30) | |
| ap.add_argument("--top", type=int, default=14) | |
| args = ap.parse_args() | |
| scores = per_turn_scores() | |
| means = {m: st.mean(v.values()) for m, v in scores.items() if len(v) >= 20} | |
| ranked = sorted(means, key=lambda m: -means[m]) | |
| print(f"models with >=20 scored turns: {len(ranked)}\n") | |
| print(f" {'mean S':>9}{'turns':>7} model") | |
| for m in ranked[: args.top]: | |
| print(f" {means[m]:>9.4f}{len(scores[m]):>7} {m[:58]}") | |
| # Crown threshold, for judging whether any gain matters at all. | |
| duels = [d for d in load_duels() if d["se"] is not None] | |
| crown = 3 * st.median(d["se"] for d in duels) | |
| print(f"\n=== pairwise, overlap >= {args.min_overlap} turns " | |
| f"(crown needs +{crown:.4f} over the king) ===") | |
| print(f" {'ovl':>5}{'rho':>7}{'A mean':>9}{'B mean':>9}" | |
| f"{'oracle':>9}{'gain':>9} pair") | |
| rows = [] | |
| for a, b in combinations(ranked, 2): | |
| shared = sorted(set(scores[a]) & set(scores[b])) | |
| if len(shared) < args.min_overlap: | |
| continue | |
| xs = [scores[a][t] for t in shared] | |
| ys = [scores[b][t] for t in shared] | |
| ma, mb = st.mean(xs), st.mean(ys) | |
| oracle = st.mean(max(x, y) for x, y in zip(xs, ys)) | |
| rows.append((oracle - max(ma, mb), len(shared), spearman(xs, ys), | |
| ma, mb, oracle, a, b)) | |
| rows.sort(key=lambda r: -r[0]) | |
| for gain, n, rho, ma, mb, orc, a, b in rows[: args.top]: | |
| flag = " <== beats crown threshold" if gain >= crown else "" | |
| print(f" {n:>5}{rho:>7.2f}{ma:>9.4f}{mb:>9.4f}{orc:>9.4f}{gain:>9.4f}" | |
| f" {a.split('/')[-1][:24]} + {b.split('/')[-1][:24]}{flag}") | |
| if rows: | |
| gains = [r[0] for r in rows] | |
| rhos = [r[2] for r in rows] | |
| print(f"\n pairs compared: {len(rows)}") | |
| print(f" median Spearman between models : {st.median(rhos):+.3f}") | |
| print(f" median ORACLE gain over better parent: {st.median(gains):+.4f}") | |
| print(f" best ORACLE gain : {max(gains):+.4f}") | |
| print(f" crown threshold (3*se) : {crown:+.4f}") | |
| beat = sum(1 for g in gains if g >= crown) | |
| print(f" pairs whose ORACLE clears the crown : {beat}/{len(rows)}") | |
| print("\n Oracle = per-turn max, an UPPER BOUND far above any weight\n" | |
| " merge. If the oracle barely clears the threshold, merging cannot.") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 5.46 kB
- Xet hash:
- 951fa178df6842d9c05c227dfdb77f6f930b4ff80365646664124fedc28db54e
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.