v3_scale: code banc v4/gen/v3, rapports validés, corpus_v3 (sha256 vérifié), README
519e2ef verified | #!/usr/bin/env python3 | |
| """Checker — chevauchement 64 o des fenêtres d'éval (définitions eval_ref_v3) avec old/broad train.bin, par position d'éval. | |
| Position t (cible = patch t+1) « contaminée » si un octet du patch cible est couvert par une fenêtre de 64 o présente dans train. | |
| Recalcule fused_acc_invocab des réfs (scale/hits) sur positions propres. Usage : audit_overlap.py CORPUS (old_local_v1|broad_local_v1)""" | |
| import sys, os, glob, json, numpy as np | |
| sys.path.insert(0, "/workspace/SPIRA/python"); sys.path.insert(0, "/workspace/spira_shared/checker/fc2"); sys.path.insert(0, "/workspace/spira_shared/checker") | |
| from rollhash import hashes | |
| from spira.model import SPIRA | |
| from windows_v2 import freshcode_v2 | |
| from build_corpus_local import strip_gutenberg_header | |
| c = sys.argv[1]; C = f"/workspace/spira_shared/broad/{c}"; T = "/workspace/repos/toroidal-fractal-intelligence/data"; rd = lambda p: open(p, "rb").read() | |
| tr = rd(f"{C}/train.bin"); th = np.unique(np.concatenate([hashes(tr[s:s+4_000_063]) for s in range(0, len(tr), 4_000_000)])) | |
| W = {"fresh_en": (rd(f"{C}/oneshot.bin"), 8000), "fresh_fr": (strip_gutenberg_header(rd(f"{T}/contes_perrault.txt")), 8000), | |
| "neither_code": (b"\n\n".join(rd(p) for p in sorted(glob.glob("/workspace/kahnn/data/code/*"))), 8000), | |
| "test20k": (rd(f"{C}/test.bin"), 20000), "queries": (rd(f"{T}/chat_seed_fr_en.txt"), 8000), "freshcode_v2": (freshcode_v2(), 8000)} | |
| m = SPIRA.load(f"/workspace/spira_shared/checker/ckpt/ref_fixed2cal_{c}.npz") | |
| D = "/workspace/spira_shared/checker/scale/hits"; out = {} | |
| for k, (b, mp) in W.items(): | |
| h = hashes(b); bad = np.isin(h, th); cov = np.zeros(len(b) + 1, np.int32) | |
| for i in np.nonzero(bad)[0]: cov[i] += 1; cov[i + 64] -= 1 | |
| covered = np.cumsum(cov)[:len(b)] > 0 | |
| pats = m.ept.segment(b); off = np.cumsum([0] + [len(p) for p in pats]) | |
| n = min(len(pats) - 1, mp); cont = np.array([covered[off[t+1]:off[t+2]].any() for t in range(n)]) | |
| r = {"windows64": int(len(h)), "windows64_overlap": int(bad.sum()), "bytes_covered": int(covered.sum()), "positions": n, "positions_contaminated": int(cont.sum())} | |
| hp = f"{D}/ref_fixed2cal_{c}__{c}__{k}.npz" | |
| if os.path.exists(hp): | |
| z = np.load(hp); fh, iv = z["fused_hit"][:n], z["invocab"][:n]; cl = ~cont | |
| r["ref_fused_invocab_all"] = 100 * float(fh[iv].mean()); r["ref_fused_invocab_clean"] = 100 * float(fh[iv & cl].mean()) if (iv & cl).any() else None | |
| r["ref_fused_invocab_contaminated"] = 100 * float(fh[iv & cont].mean()) if (iv & cont).any() else None | |
| np.save(f"{os.path.dirname(os.path.abspath(__file__))}/contam_{c}__{k}.npy", cont); out[k] = r; print(k, json.dumps(r), flush=True) | |
| json.dump(out, open(f"{os.path.dirname(os.path.abspath(__file__))}/audit_overlap_{c}.json", "w"), indent=1) | |