Buckets:
| """The Λ2 semantic microscope: controlled thought groups from stored duels. | |
| The whole research programme rests on one comparison — two thoughts that face | |
| an IDENTICAL prefix and an IDENTICAL target answer, scoring differently. Only | |
| then is the difference attributable to the thought itself. | |
| Getting the grouping key right is the entire correctness burden here: | |
| * NOT `turn_id` alone. A row holds 4 independently sampled thoughts, each | |
| paired with its OWN independently sampled teacher reference, so two pairs | |
| on the same turn usually face DIFFERENT y_C. | |
| * NOT `(turn_id, pair_index)` across dumps. The teacher re-samples its | |
| reference every duel: measured over the corpus, **82.3% of (turn, index) | |
| slots carry a different y_C in a different dump**. Grouping across dumps | |
| silently compares thoughts aimed at different answers. | |
| * `(dump, turn_id, y_C)` — the key used here. Keying on the ANSWER TEXT | |
| rather than the pair index is strictly better than the (dump, turn, index) | |
| key used by the earlier lever analysis: references frequently coincide by | |
| chance, so those thoughts belong in one group and that key split them. | |
| Within a group, `lpC(y_C|∅)` is a function of prefix and answer alone, so it | |
| must be constant. That is checked, not assumed — it is the cheapest available | |
| test that the grouping is really holding the target fixed. | |
| Two population are emitted, and the distinction matters: | |
| * `cross` — thoughts from different models (challenger vs king). A ranker | |
| trained here can cheat by learning MODEL IDENTITY instead of thought | |
| quality. | |
| * `same` — different samples from the SAME model on the same target. No | |
| model-identity signal exists to learn, so this is the honest population | |
| for the §7 milestone. | |
| python harness/microscope.py --out microscope.jsonl | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import collections | |
| import gzip | |
| import json | |
| import pathlib | |
| import statistics as st | |
| from screen import EVALS | |
| ROOT = pathlib.Path(__file__).resolve().parent.parent | |
| def lambda2(p: dict) -> float: | |
| return p["lpC_yc_za"] - p["lpC_yc_e"] | |
| def build(paths: list[pathlib.Path]) -> list[dict]: | |
| """One record per controlled group.""" | |
| groups: list[dict] = [] | |
| for path in paths: | |
| with gzip.open(path) as fh: | |
| d = json.load(fh) | |
| dump = path.name.split(".")[0] | |
| req = d.get("request") or {} | |
| refs = d.get("teacher_refs") or {} | |
| repos = {"challenger_rows": req.get("challenger_repo"), | |
| "king_rows": req.get("king_repo")} | |
| # (turn, y_C) -> members, pooled across BOTH sides of this one duel. | |
| bucket: dict[tuple, list[dict]] = collections.defaultdict(list) | |
| for side, repo in repos.items(): | |
| for row in d.get(side) or []: | |
| if not row.get("valid") or not row.get("pairs"): | |
| continue | |
| turn = row["turn_id"] | |
| tref = refs.get(turn) or [] | |
| for i, p in enumerate(row["pairs"]): | |
| if i >= len(tref): | |
| continue # no reference to identify the target | |
| z, y = p.get("z_a"), tref[i].get("y") | |
| if not z or not y: | |
| continue | |
| bucket[(turn, y)].append({ | |
| "thought": z, "action": p.get("y_a"), | |
| "l2": lambda2(p), "lp_empty": p["lpC_yc_e"], | |
| "model": repo, "side": side, "pair_index": i, | |
| }) | |
| for (turn, y), members in bucket.items(): | |
| # Deduplicate identical thoughts: a repeated sample carries no new | |
| # information and would inflate every count downstream. | |
| seen, uniq = set(), [] | |
| for m in members: | |
| if m["thought"] in seen: | |
| continue | |
| seen.add(m["thought"]) | |
| uniq.append(m) | |
| if len(uniq) < 2: | |
| continue | |
| mean_l2 = st.mean(m["l2"] for m in uniq) | |
| for m in uniq: | |
| m["d_l2"] = m["l2"] - mean_l2 | |
| groups.append({ | |
| "dump": dump, "turn": turn, "target": y, | |
| "n": len(uniq), "spread": max(m["l2"] for m in uniq) - min(m["l2"] for m in uniq), | |
| "lp_empty_spread": (max(m["lp_empty"] for m in uniq) | |
| - min(m["lp_empty"] for m in uniq)), | |
| "models": sorted({m["model"] for m in uniq if m["model"]}), | |
| "members": uniq, | |
| }) | |
| return groups | |
| def report(groups: list[dict]) -> None: | |
| n_same = sum(1 for g in groups if len(g["models"]) == 1) | |
| print(f"\ncontrolled groups: {len(groups)} " | |
| f"(same-model {n_same}, cross-model {len(groups) - n_same})") | |
| sizes = [g["n"] for g in groups] | |
| print(f" thoughts/group: mean {st.mean(sizes):.2f} median {st.median(sizes)} " | |
| f"max {max(sizes)}") | |
| print(f" total thoughts: {sum(sizes)}") | |
| # Integrity: lpC(y_C|empty) depends only on prefix and answer, so within a | |
| # group it must not move. A large spread means the key is not holding the | |
| # target fixed and every downstream number is suspect. | |
| sp = sorted(g["lp_empty_spread"] for g in groups) | |
| bad = sum(1 for v in sp if v > 1e-6) | |
| print(f"\n INTEGRITY lpC(y_C|null) spread within group:") | |
| print(f" median {sp[len(sp)//2]:.2e} p99 {sp[int(len(sp)*0.99)]:.2e} " | |
| f"max {sp[-1]:.2e}") | |
| print(f" groups with spread > 1e-6: {bad} ({100*bad/len(sp):.2f}%)") | |
| for label, sel in (("SAME-model", lambda g: len(g["models"]) == 1), | |
| ("CROSS-model", lambda g: len(g["models"]) > 1)): | |
| sub = [g for g in groups if sel(g)] | |
| if not sub: | |
| continue | |
| spread = sorted(g["spread"] for g in sub) | |
| pairs = sum(g["n"] * (g["n"] - 1) // 2 for g in sub) | |
| print(f"\n {label}: {len(sub)} groups, {pairs} orderable pairs") | |
| print(f" within-group Λ2 spread: median {spread[len(spread)//2]:.4f} " | |
| f"mean {st.mean(spread):.4f} p90 {spread[int(len(spread)*0.9)]:.4f}") | |
| def main() -> None: | |
| ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) | |
| ap.add_argument("--out", default=str(ROOT / "microscope.jsonl")) | |
| args = ap.parse_args() | |
| paths = sorted(EVALS.glob("*.json.gz")) | |
| print(f"reading {len(paths)} duel dumps from {EVALS}") | |
| groups = build(paths) | |
| report(groups) | |
| out = pathlib.Path(args.out) | |
| with out.open("w") as fh: | |
| for g in groups: | |
| fh.write(json.dumps(g) + "\n") | |
| print(f"\nwrote {len(groups)} groups -> {out} ({out.stat().st_size/1e6:.1f} MB)") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 6.8 kB
- Xet hash:
- 4366a891ab4cc1b9e2a1fe90bef84b80eccef5d287a9825ee5bdb6bfaf20ed7b
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.