Jordine's picture
Upload code_extra/xsize/summarize.py with huggingface_hub
2354a9e verified
Raw History Blame Contribute Delete
4.75 kB
"""Build results/xsize/lineage/SUMMARY_tables.md from the per-pair jsons (A_*.json, B_*.json, Bmatch_*.json, W_*.json).
python3 -m xsize.summarize
"""
import glob, json, os
import numpy as np
from xsize.lineage import OUT, PAIRS_A, slug
KIND = {f"{slug(s)}__{slug(l)}": k for s, l, k in PAIRS_A}
def verdict(r):
if "frac_mutual_nn" not in r["real"]: # scanned before the MNN metric existed: fall back to max-corr above rot null
v = r["real"].get("frac_maxcorr_above_rotnull_q999", float("nan"))
return ("**SHARED basis**" if v >= 0.5 else "partial?" if v >= 0.05 else "none detected") + " (old metric)"
mnn = r["real"]["frac_mutual_nn_above_rotnull_q999"]
same = r["real"]["frac_argmax_same_index"]
if mnn >= 0.5:
return "**SHARED basis** (" + ("same dim index" if same >= 0.5 else "permuted dims") + ")"
if mnn >= 0.05:
return "partial alignment"
return "none detected"
def fmt(d, k):
return f"{d[k]:.3f}" if k in d else "n/a"
def table_A():
rows = []
for f in sorted(glob.glob(os.path.join(OUT, "A_*.json"))):
r = json.load(open(f))
rows.append(r)
order = {f"{slug(s)}__{slug(l)}": i for i, (s, l, _) in enumerate(PAIRS_A)}
rows.sort(key=lambda r: order.get(r["tag"], 999))
out = ["| pair (small -> large) | prior | dims | diag abs corr | mean max abs corr | frac same-index | frac MNN > rot-null q99.9 | nulls: shuffle max / rot max / rot MNN | verdict | MB |",
"|---|---|---|---|---|---|---|---|---|---|"]
for r in rows:
R, S, Q = r["real"], r["null_shuffled_tokens"], r["null_random_rotation"]
out.append(f"| {slug(r['small'])} -> {slug(r['large'])} | {KIND.get(r['tag'], '')} | {r['d_small']}->{r['d_large']} | "
f"{R['diag_abs_mean']:.3f} | {R['maxcorr_mean']:.3f} | {R['frac_argmax_same_index']:.3f} | "
f"{fmt(R, 'frac_mutual_nn_above_rotnull_q999')} | {S['maxcorr_mean']:.3f} / {Q['maxcorr_mean']:.3f} / "
f"{fmt(Q, 'frac_mutual_nn_above_rotnull_q999')} | {verdict(r)} | {r['bytes']/1e6:.0f} |")
negs = [r["real"]["frac_mutual_nn_above_rotnull_q999"] for r in rows
if not verdict(r).startswith("**") and "frac_mutual_nn" in r["real"]]
return "\n".join(out), rows, negs
def segments(m):
"""compress a layer map [s_0, s_1, ...] into 'large a-b <- small c-d' runs."""
segs, a = [], 0
for i in range(1, len(m) + 1):
if i == len(m) or m[i] != m[i - 1] + 1:
segs.append((a, i - 1, m[a], m[i - 1])); a = i
return segs
def table_B():
out = ["| pair | layers | layer map (large <- small, by argmax mean cos) | mean cos on map (min-max) | max off-map cos | shuffled-neuron null max abs | MB |",
"|---|---|---|---|---|---|---|"]
for f in sorted(glob.glob(os.path.join(OUT, "B_*.json"))):
r = json.load(open(f))
c = np.array(r["cos"]); m = r["best_small_layer_for_each_large_layer"]
on = np.array([c[m[l], l] for l in range(len(m))])
mask = np.ones_like(c, bool)
for l in range(len(m)):
mask[m[l], l] = False
sig = on > 10 * max(r["null_abs_max"], 1e-3)
segs = []
for a, b, x, y in segments([m[l] if sig[l] else -10**6 - l for l in range(len(m))]):
if sig[a]:
segs.append(f"{a}-{b}<-{x}-{y}" if a != b else f"{a}<-{x}")
mp = ", ".join(segs) if sig.any() else "no structure (all cells ~ null)"
if 0 < sig.mean() < 1:
nl = np.where(~sig)[0]
mp += f" [large L{nl.min()}-{nl.max()} ({len(nl)} layers) at null level]" if nl.max() - nl.min() + 1 == len(nl) else f" [null-level large layers: {nl.tolist()}]"
out.append(f"| {slug(r['small'])} -> {slug(r['large'])} | {r['layers_small']}x{r['layers_large']} | {mp} | "
f"{on[sig].mean() if sig.any() else on.mean():.3f} ({on[sig].min() if sig.any() else on.min():.3f}-{on.max():.3f}) | "
f"{c[mask].max():.3f} | {r['null_abs_max']:.3f} | {r['bytes']/1e6:.0f} |")
return "\n".join(out)
if __name__ == "__main__":
tA, rows, negs = table_A()
tB = table_B()
tot = sum(r["bytes"] for r in rows)
for pat in ["B_*.json", "Bmatch_*.json", "W_*.json", "Q35_*.json"]:
tot += sum(json.load(open(f)).get("bytes", 0) for f in glob.glob(os.path.join(OUT, pat)))
s = f"## Test A (embedding dim alignment)\n\n{tA}\n\nmax MNN fraction over all pairs without a shared-basis verdict: {max(negs) if negs else float('nan'):.3f}\n\n" \
f"## Test B (same-index up_proj rows, layer x layer)\n\n{tB}\n\nbytes in saved result files: {tot/1e9:.2f} GB\n"
open(os.path.join(OUT, "SUMMARY_tables.md"), "w").write(s)
print(s)