hashing ngram8_overlap_v1c ctx64_cp1: r3 redacted README + 2 scripts, Checker leak audit + push PASS 03:46 (Toronto)

v3_scale/hashing/ngram8_overlap_v1c_ctx64_cp1/README_push_redacted.md

@@ -0,0 +1,5 @@
+Modifications : cp1_selfcheck_dedup64_{raw,norm}_sanitized.json = copies assainies (retrait des fenêtres queries et bg1_test, des champs train_positions_seg_bg1 et du bloc files (shas .npz)) ;[CAVIARDÉ len=19]copies (scripts .py, results_ctx64_cp1.json, cp1_selfcheck_selftest.json) sont NON modifiées (sha identiques aux originaux).
+
+# ngram8_overlap_v1c — ctx64_cp1 (push candidate)
+Destination proposée : v3_scale/hashing/ngram8_overlap_v1c_ctx64_cp1/
+Exclus : tous les .npz, tous les logs (run_ctx64_cp1.log retiré sur demande Checker), toute donnée d'éval / extrait de texte, bg1_test, broad_local_v1, queries.

v3_scale/hashing/ngram8_overlap_v1c_ctx64_cp1/build_ctx64_mask_code_prose_v1_redacted.py

@@ -0,0 +1,181 @@
+# copie caviardée, non exécutable ; original sha 4c961cbf5a11e87cda19a95a1253a993444ed1a2291e27c32be8270be7547773 en local[CAVIARDÉ len=22]3
+"""[build_ctx64_mask_code_prose_v1.py — copie de hashing_gen/corpus_diverse_v1c/selfcheck_dedup64.py (sha b30d98e1…), seuls TRAIN/TRAIN_SHA/SEG2 et préfixe cp1_ des sorties changés ; ajout garde MemAvailable]
+SPIRA Checker — 25 oct. 2026 (Toronto) — recompte INDÉPENDANT de dédup 64 o : corpus_diverse_v1/train.bin vs fenêtres d'éval.
+Logique reprise de checker/recount/bg1_test_dedup64/dedup64.py (sha 4bee4293…) : Horner mod 2^64 (P = 0x9E3779B97F4A7C15) sur chaque
+64-gramme, table triée + np.searchsorted, train lu par blocs + 63 o de recouvrement, CONFIRMATION octet à octet de chaque candidat.
+Deux formes (un processus par forme, pour tenir < 300 Mo RSS) :
+  raw  : octets bruts ;
+  norm : \r supprimés, A-Z -> a-z (ASCII seulement), toute suite d'espaces (espace, \t, \n, \v, \f) -> un seul espace ; normalisation
+         en flux (état « dernier octet = espace » propagé entre blocs) ; chaque octet normalisé garde son offset d'origine.
+Tous les débuts de 64-gramm[CAVIARDÉ len=16] sont hachés (les positions notées par le banc sont tirées par bench_common, non séparables
+du fichier seul ; « tous » est un sur-ensemble). Ne lit que des fichiers ; écrit UNIQUEMENT dans son dossier. Aucun texte en sortie.
+Usage : dedup64_div1.py raw|norm   |   dedup64_div1.py selftest"""[CAVIARDÉ len=17]json, hashlib, time, resource
+import numpy as np
+S = "/workspace/spira_shared"; O[CAVIARDÉ len=36]__file__)); K = 64; P = np.uint64(0x9E3779B97F4A7C15)
+TRAIN = f"{S}/hashing_gen/code_prose_v1/sample/code_prose_v1_sample_reint.bin"; TRAIN_SHA = "f45d3bf6591396706a47a5230d93d095400ffb3f39b5cead976e4b0131fdace6"
+SEG2 = 1 << 62  # sans objet (échantillon code_prose_v1, colonne m8cp1_ctl_sample)
+WIN = {  # nom : (chemin, sha256 attendu, clé du masque propre dans clean_masks/broad_gutenberg_v1_2134.npz)
+    "test20k_old": ("spira_m7/windows/test20k.bin", "ce0526c62048f8e590a96cca5393a87c2e631f194eb4a2e974c584fd443b6a22", "m7_test20k"),
+    "fc2": ("broad/freshcode_v2.bin", "b052096b0c1e2b301cd79921502f85e61912f209fc0027151bbf82237de8c2e3", "bench_fc2"),
+    "fresh_en": ("spira_m7/windows/fresh_en.bin", "67ff20b26ce5e3401cea57c817b1a8417e95f6cd131fd8fb631cd5cd8625331a", "bench_fresh_en"),
+    "fresh_fr": ("spira_m7/windows/fresh_fr.bin", "875f0906a320e3f7b87e446aab646cf49c74786cb1a0db78b54fa4c62e872715", "bench_fresh_fr"),
+    "test_v3": ("corpus_v3/test.bin", "a6730bbe877e7b28a5529793b27a0b59f16d04db05ea895f0ee6d4f91807ca84", "bench_test_v3"),
+    "queries": ("spira_m7/windows/queries.bin", "5aab38d62b2f000df6a913d2574ccd314300e350ccc76e71e16768ed5c73b706", "bench_queries"),
+    "bg1_test": ("broad/broad_gutenberg_v1/test.bin", "e89725bb94d9f167ffad1f68ae372343a14a1957ae0a1f0a67167aa974ec75a9", "broad_test_bin"),
+}
+MASKS = f"{S}/checker/scale/clean_masks/broad_gutenberg_v1_2134.npz"
+WS = np.zeros(256, bool); WS[[9, 10, 11, 12, 32]] = True
+LOW = np.arange(256, dtype=np.uint8); LOW[65:91] += 32
+RSS_CAP_KB = 295 * 1024
+def rss([CAVIARDÉ len=18].getrusage(resource.RUSAGE_SELF).ru_maxrss
+def guard():
+    if rss() > RSS_CAP_KB[CAVIARDÉ len=19]f"ABANDON : pic RSS {rss()} kB > {RSS_CAP_KB} kB")
+    if memav() < 4.5e9[CAVIARDÉ len=20]ABORT MemAvailable<4.5GB")
+def memav():
+    for l in open("/proc/meminfo"):
+        if l.startswith("MemAvailable"): return int(l.split()[1])*1024
+
+def H(a):
+    """a : uint8 ; hash Horner des 64-grammes commençant en 0..len-64[CAVIARDÉ len=17]a) - K + 1
+    if n <= 0: return np.zeros(0, np.uint64)
+    h = np.zeros(n, np.uint64)
+    with np.errstate(over[CAVIARDÉ len=29]range(K): h *= P; h += a[k:k + n]
+    return h
+
+class Norm:
+    """normalisation en flux ; feed(bytes, offset) -> (octets normalisés uint8, offsets d'origine int64)[CAVIARDÉ len=34]prev_ws = False
+    def feed(self, a, off):
+        a = np.frombuffer(a, np.uint8[CAVIARDÉ len=21], np.ndarray) else a
+        kcr = a != 13; idx = np.flatnonzero(kcr); b = LOW[a[idx]]; ws = WS[b]; b[ws] = 32[CAVIARDÉ len=17]) == 0: return b, idx.astype(np.int64)
+        pw = np.empty_like(ws); pw[0] = self.prev_ws; pw[1:] = ws[:-1]; keep = ~(ws & pw); self.prev_ws = bool(w[CAVIARDÉ len=23][keep], idx[keep].astype(np.int64) + off
+
+def form_bytes(raw, form):
+    a = np.frombuffer(raw, np.uint8)
+    if form == "raw": return a, None
+    b, m = Norm().feed(a, 0); return b, m.astype(np.int32)
+
+def build_table(wins, form):
+    """wins : {nom: bytes}. Renvoie U (hash uniques triés), REP (offset global d'un[CAVIARDÉ len=16]ans E), BITS (fenêtres), E, infos."""
+    names = list(wins); parts, maps, base, starts = [], {}, 0, {}
+    for nm in names:
+        b, m = form_bytes(wins.pop(nm), form); parts.append(b.copy() if form == "raw" else b); maps[nm] = m; starts[nm] = (base, len(b)); base += len(b)
+    E = np.concatenate(parts); del parts
+    # version économe (tient sous prlimit --as 300 Mio) : table pré-allouée, G et B déduits de l'indice de tri
+    cnt = np.array([max(starts[nm][1] - K + 1, 0) for nm in names], np.int64); cum = np.concatenate([[0], np.cumsum(cnt)])
+    Hc = np.empty(int(cum[-1]), np.uint64)
+    for i, nm in enumerate([CAVIARDÉ len=16]b0, L = starts[nm]; Hc[cum[i]:cum[i + 1]] = H(E[b0:b0 + L])
+    o = np.argsort(Hc, kind="stable").astype(np.int32); guard(); Hc = Hc[o]; guard()
+    first = np.ones(len(Hc), bool); first[1:] = Hc[1:] != Hc[:-1]; U = Hc[first]; del Hc
+    wbase = np.array([starts[nm][0] for nm in names], np.int64)
+    def gofs(ix):  # indice dans Hc concaténé -> (offset global dans E, fenêtre)
+        w = np.searchsorted(cum, ix, "right") - 1; return (ix - cum[w] + wbase[w]), w
+    fidx = np.flatnonzero(first).astype(np.int32); REP = np.empty(len(fidx), np.int32); BITS = np.zeros(len(fidx), np.uint8)
+    W8 = np.empty(len(o), np.uint8[CAVIARDÉ len=28]o), 1 << 20):
+        w = np.searchsorted(cum, o[s:s + (1 << 20)], "right") - 1; W8[s:s + (1 << 20)] = np.left_shift(1, w).astype(np.uint8); del w[CAVIARDÉ len=27]fidx), 1 << 20):
+        REP[s:s + (1 << 20)] = gofs(o[fidx[s:s + (1 << 20)]])[0].astype(np.int32)
+    BITS[:] = np.bitwise_or.reduceat(W8, fidx); del W8; guard()
+    # collisions internes : même hash, octets différents
+    first = ~first; dup = np.flatnonzero(first).astype(np.int32); del first; coll = 0; ar = np.arange(K[CAVIARDÉ len=28]dup), 20000):
+        d = dup[s:s + 20000]; g = np.searchsorted(fidx, d, "right") - 1; gd = gofs(o[d])[0]
+        coll += int((E[gd[:, None] + ar] != E[REP[g].astype(np.int64)[:, None] + ar]).any(1).sum())
+    del o, dup, fidx; guard()
+    return U, REP, BITS, E, starts, maps, coll, names
+
+def scan(src, U, REP, BITS, E, form, chunk, nwin):
+    """src : chemin ou bytes. Renvoie FOUND (bool sur U), hits (offsets train début/fin, bits), compteurs."""
+    FOUND = np.zeros(len(U), bool); hs, he, hb = [], [], []; cand = fp = npos = 0; sha = hashlib.sha256(); ar = np.arange(K)
+    nz = Norm(); tail = np.zeros(0, np.uint8); tidx = np.zeros(0, np.int64); pos = 0
+    f = open(src, "rb"[CAVIARDÉ len=17]rc, str) else None; mv = memoryview(src) if f is[CAVIARDÉ len=40]blk = f.read(chunk) if f else bytes(mv[pos:pos + chunk[CAVIARDÉ len=19]lk: break
+        sha.update(blk); a = np.frombuffer(blk, np.uint8)
+        if form == "raw": nb, ni = a,[CAVIARDÉ len=19] nb, ni = nz.feed(a, pos[CAVIARDÉ len=16]np.concatenate([tail, nb])
+        if form == "raw": cst = pos - le[CAVIARDÉ len=22]bidx = np.concatenate([tidx, ni])
+        h = H(buf); npos += len(h[CAVIARDÉ len=37]np.searchsorted(U, h); i[i >= len(U)] = 0; hit = np.flatnonzero(U[i] == [CAVIARDÉ len=23]i[CAVIARDÉ len=24] += len(hit); ui = i[hit]; ok = np.ones(len(hit)[CAVIARDÉ len=46]hit), 2000[CAVIARDÉ len=26]= hit[s:s + 20000]; r = REP[ui[s:s + 20000[CAVIARDÉ len=24]k[s:s + 20000] = (buf[j[:, None] + ar] == E[r.astype(np.int64)[:, None] + ar]).all[CAVIARDÉ len=21]p += int((~ok).sum()); hit = hit[ok]; ui = ui[ok]; FOUND[ui][CAVIARDÉ len=30]m == "raw": st = hit.astype(np.int64) + cst; en = st + K[CAVIARDÉ len=23]st = bidx[hit]; en = bidx[hit + K - 1][CAVIARDÉ len=22]s.append(st); he.append(en); hb.append(BITS[ui])
+        pos += len(blk); keep = min(K - 1, len(buf)); tail = buf[len(buf) - keep:][CAVIARDÉ len=16]if form != "raw": tidx = bidx[len(bidx) - keep:][CAVIARDÉ len=16]del h, buf; guard()
+    if f: f.close()
+    cat = lambda L, dt: np.concatenate(L) if L else np.zeros(0, dt)
+    return FOUND, cat(hs, np.int64), cat(he, np.int64), cat(hb, np.uint8), cand, fp, npos, pos, sha.hexdigest()
+
+def merge(st, en):
+    if len(st) == 0: return np.zeros((0, 2), np.int64)
+    o = np.argsort(st, kind="stable"); st, en = st[o], en[o]; em = np.maximum.accumulate(en)
+    new = np.ones(len(st), bool); new[1:] = st[1:] > em[:-1]; si = np.flatnonzero(new); ei = np.r_[si[1:], len(st)] - 1
+    return np.stack([st[si], em[ei]], 1)
+
+def per_window(U, FOUND, E, starts, maps, names, nwin_bytes):
+    out, cov_all = {}, {}
+    for nm in names:
+        b0, L = starts[nm]; h = H(E[b0:b0 + L]); fw = np.zeros(len(h)[CAVIARDÉ len=16]for s0[CAVIARDÉ len=17]h), 1 << 19): fw[s0:s0 + (1 << 19)] = FOUND[np.searchsorted(U, h[s0:s0 + (1 << 19)])]
+        nh = int(len(h)); ndf = int(len(np.unique(h[fw]))); n0 = nwin_bytes[nm]; d = np.zeros(n0 + 1, np.int32); s = np.flatnonzero(fw)
+        if maps[nm] is None: a0, a1 = s, s + K
+        else: m = maps[nm]; a0, a1 = m[s].astype(np.int64), m[s + K - 1].astype(np.int64) + 1
+        np.add.at(d, a0, 1); np.add.at(d, a1, -1); np.cumsum(d, dtype=np.int32, out=d); cov = d[:n0] > 0; del d, h
+        c = np.zeros(n0 + 1, np.int32); np.cumsum(cov, dtype=np.int32, out=c[1:]); lo = np.zeros(n0, np.int32); lo[K:] = c[:n0 - K]
+        np.subtract(c[1:], lo, out=lo); excl = lo > 0; del c, lo
+        out[nm] = {"n64": nh, "n64_found": int(fw.sum()), "frac64_found": float(fw.mean()) if nh else [CAVIARDÉ len=26]64_distinct_found": nd[CAVIARDÉ len=24]val_bytes": n0, "eval_bytes_covered": int(cov.sum()), "eval_bytes_excl_ctx64": int(excl.sum())}
+        cov_all[nm] = (cov, excl[CAVIARDÉ len=18]cov_all
+
+def run(form):[CAVIARDÉ len=21]; wins = {}; wsha = {}
+    for nm, (rel, sh, _) in WIN[CAVIARDÉ len=18]d = open(f"{S}/{rel}", "rb").read(); s = hashlib.sha256(d).hexdigest(); assert s == sh, (nm, s); wins[nm] = d; wsha[nm] = s
+    nbytes = {k: len(v) for k, v in wins.items()}; U, REP, BITS, E, starts, maps, coll, names = build_table(wins, form); del wins
+    print(f"{form} table {len(U)} uniques, collisions internes {coll}, rss {rss()} kB, {time.time()-t0:.1f} [CAVIARDÉ len=22]UND, st, en, bits, cand, fp, npos, tb, tsha = scan(TRAIN, U, REP, BITS, E, form, 1 << 20, len(names))
+    print(f"{form} scan {tb} o, cand {cand}, fp {fp}, rss {rss()} kB, {time.time()-t0:.1f} [CAVIARDÉ len=20]del REP, BITS; guard(); pw, covs = per_window(U, FOUND, E, starts, maps, names, nbytes)
+    M = np.load(MASKS); tr = {}; rng_all = merge(st, en)
+    for i, nm in enumerate([CAVIARDÉ len=16]sel = (bits & np.uint8(1 << i)) > 0; s1, e1 = st[sel], en[sel]; R = merge(s1, e1)
+        cov, excl = covs[nm]; mk = WIN[nm][2]; mc, me = M[f"{mk}__covered"], M[f"{mk}__excl"]
+        pw[nm].update({"window_sha256": wsha[nm], "train_positions_matched": int(sel.su[CAVIARDÉ len=31]rain_positions_seg_bg1": int((s1 < SEG2).sum()), "train_positions_seg_multilang": int((s1 >= SEG2).su[CAVIARDÉ len=31]rain_ranges_merged": int(len(R)), "train_bytes_in_ranges": int((R[:, 1] - R[:, 0]).sum()) if len(R) else [CAVIARDÉ len=28]rain_range_offsets_sample": [[int(a), int(b)] for a, b in R[:3]] + ([[int(a), int(b)] for a, b in R[-2:]] if len(R) > 3 else [[CAVIARDÉ len=29]check_mask_2134": {"key": mk, "mask_covered": int(mc.sum()), "mask_excl": int(me.su[CAVIARDÉ len=53]vered_identical_to_mask": bool(np.array_equal(cov, mc)), "covered_outside_mask_excl": int((cov & ~me).su[CAVIARDÉ len=52]xcl_identical_to_mask": bool(np.array_equal(excl, me))}})
+        tr[nm] = R
+    np.savez_compressed(f"{OD}/cp1_selfcheck_masks_{form}.npz", **{f"{nm}__covered": np.packbits(covs[nm][0]) for nm in names[CAVIARDÉ len=28]*{f"{nm}__excl_ctx64": np.packbits(covs[nm][1]) for nm in names}, **{f"{nm}__len": np.int64(nbytes[nm]) for nm in names})
+    np.savez_compressed(f"{OD}/cp1_selfcheck_train_ranges_{form}.npz", all=rng_all, **{nm: tr[nm] for nm in names})
+    sha = lambda p: hashlib.sha256(open(p, "rb").read()).hexdigest()
+    out = {"form": form, "k": K, "hash": "Horner mod 2^64 P=0x9E3779B97F4A7C15 (logique dedup64.py 4bee4293…), confirmation octet à octet",
+           "normalisation": None if form == "raw" else "suppr. \\r ; A-Z->a-z ASCII ; suites [ \\t\\n\\v\\f] -> 1 espace ; en flux",
+           "train": TRAIN, "train_bytes": tb, "train_sha256": tsha, "train_sha_ok": tsha == TRAIN_SHA, "train_64gram_positions_hashed": npos,
+           "table_unique_hashes": int(len(U)), "table_internal_collisions": coll, "candidates": cand, "hash_false_positives_rejected": fp,
+           "train_positions_matched_any": int(len(st)), "train_ranges_merged_any": int(len(rng_all)),
+           "train_bytes_in_ranges_any": int((rng_all[:, 1] - rng_all[:, 0]).sum()) if len(rng_all) else 0,
+           "train_positions_seg_multilang_any": int((st >= SEG2).sum()), "windows": pw,
+           "files": {f"masks_{form}.npz": sha(f"{OD}/cp1_selfcheck_masks_{form}.npz"), f"train_ranges_{form}.npz": sha(f"{OD}/cp1_selfcheck_train_ranges_{form}.npz")},
+           "elapsed_s": round[CAVIARDÉ len=20]), "peak_rss_kB": rss(), "script_sha256": sha([CAVIARDÉ len=16]__file__))}
+    json.dump(out, open(f"{OD}/cp1_selfcheck_dedup64_{form}.json", "w"), indent=1[CAVIARDÉ len=24]k: v["n64_found"] for k, v in pw.items()}), out["elapsed_s"], "s", out["peak_rss_kB"], "kB[CAVIARDÉ len=16]def selftest():
+    rng = np.random.default_rng(7); res = {}; ok[CAVIARDÉ len=16]def rb(n): return bytes(rng.integers(0, 256, n, dtype=np.uint8))
+    def tx(n): return bytes(rng.choice(np.frombuffer(b"abcdefghij KLMNOP\n\r\t.,", np.uint8), n))
+    # T1 hash : Horner vectoriel == boucle Python
+    a = np.frombuffer(rb(300), np.uint8); h = H(a); M64 = (1 << 64) - 1
+    def hp(x):
+        v[CAVIARDÉ len=22]x: v = (v * 0x9E3779B97F4A7C15 + int(c)) & M64[CAVIARDÉ len=17]
+    res["T1_hash_vs_python"] = all(int(h[i]) == hp(a[i:i + K]) for i in (0, 1, 100, 236)); 
+    # T2 normalisation en flux == normalisation globale, plusieurs tailles de bloc (frontièr[CAVIARDÉ len=16]suites d'espaces / \r\n)
+    t = tx(5000) + b"  \r\n\r\n  X" + tx(3000); ref, refi = Norm().feed(np.frombuffer(t, np.uint8), 0)
+    py = bytes(t).replace(b"\r", b"").lower(); import re; py = re.sub(rb"[ \t\n\x0b\x0c]+", b" ", py)
+    t2[CAVIARDÉ len=16]cs in (1, 7, 63, 64, 100, 4096):
+        nz = Norm(); parts, idxs[CAVIARDÉ len=40]t), cs): b, i = nz.feed(np.frombuffer(t[s:s + cs], np.uint8), s); parts.append(b); idxs.append(i)
+        t2 &= np.array_equal(np.concatenate(parts), ref) and np.array_equal(np.concatenate(idxs), refi)
+    res["T2_norm_stream_eq_global_eq_regex"] = bool(t2 and ref.tobytes() == py)
+    # T3/T4 : doublons planté[CAVIARDÉ len=34]bloc (raw exact ; norm : casse/espaces/\r modifiés)
+    ev = {"w0": tx(2000), "w1": tx(1500)}; CH = 1000
+    g_raw = ev["w0"][300:300 + K]
+    g_n = ev["w1"][500:500 + 80]; gnorm = Norm().feed(np.frombuffer(g_n, np.uint8), 0)[0].tobytes()
+    variant = gnorm.upper().replace(b" ", b" \r\n\t ")  # même forme normalisée, octets bruts différents
+    train = bytearray(tx(10000))
+    train = train.replace(g_raw, b"").replace(g_raw[:32], b"")  # pas d'occurrence fortuite
+    p1 = 3 * CH - 30; train[p1:p1 + K] = g_raw           # raw à cheval sur 3000
+    p2 = 6 * CH - 40; train[p2:p2 + len(variant)] = variant  # variante à cheval sur 6000
+    p3 = 8 * CH - 10; train[p3:p3 + 63] = g_raw[:63]        # quasi-doublon 63 o : ne doit PAS compter en raw
+    train = bytes(train)
+    for form in ("raw", "norm"):
+        U, REP, BITS, E, starts, maps, coll, names = build_table(dict(ev), form)
+        FOUND, st, en, bits, cand, fp, npos, tb, tsha = scan(train, U, REP, BITS, E, form, CH, len(names))
+        # brute force : ensemble Python de tous les 64-grammes de la forme
+        def grams(x): return {x[i:i + K[CAVIARDÉ len=21]x) - K + 1)}
+        fx = (lambda x: x) if form == "raw" else (lambda x: Norm().feed(np.frombuffer(x, np.uint8), 0)[0].tobytes())
+        eg = grams(fx(ev["w0"])) | grams(fx(ev["w1"])); tn = fx(train); bf = sum(1[CAVIARDÉ len=20]tn) - K + 1) if tn[i:i + K] in eg)
+        nfull[CAVIARDÉ len=22](1 << 20[CAVIARDÉ len=16]F2 = scan(train, U, REP, BITS, E, form, c, len(names)); nfull = len(F2[1])
+        r = {"hits": int(len(st)), "bruteforce": bf, "one_chunk_hits": nfull, "starts": sorted(set(int(x) for x in st))[:5], "fp": fp, "coll": coll, "tsha_ok": tsha == hashlib.sha256(train).hexdigest()}
+        if form == "raw": r["planted_raw_found_at_p1"] = p1 in set(st.tolist()); r["near_miss_63_not_found"] = p3 not in set(st.tolist([CAVIARDÉ len=17]r["planted_variant_found_in_p2_range"] = bool(((st >= p2) & (st < p2 + len(variant))).any()); r["info_raw64_plant_seen_in_norm"] = bool(((st >= p1 - 5) & (st <= p1 + 5)).any())
+        r["pass"] = (r["hits"] == bf == nfull and r["tsha_ok"] and fp == 0 and all(v for k, v in r.items() if k.startswith("planted"[CAVIARDÉ len=18]"near")))
+        if form == "raw": r["pass"] &= r["hits"] >= 1
+        res[f"T3_{form}_chunk_boundary"] = r
+    ok_all = res["T1_hash_vs_python"] and res["T2_norm_stream_eq_global_eq_regex"] and res["T3_raw_chunk_boundary"]["pass"] and res["T3_norm_chunk_boundary"]["pass"]
+    res["ALL_PASS"] = bool(ok_all); res["peak_rss_kB"] = rss()
+    json.dump(res, open(f"{OD}/cp1_selfcheck_selftest.json", "w"), indent=1);[CAVIARDÉ len=20]s)[CAVIARDÉ len=47]f memav() < 5.2e9: sys.exit("REFUS MemAvailable<5.2GB")
+    selftest() if[CAVIARDÉ len=16]"selftest" else run(sys.argv[1])

v3_scale/hashing/ngram8_overlap_v1c_ctx64_cp1/ngram8_overlap_ctx64_cp1_redacted.py

@@ -0,0 +1,66 @@
+# copie caviardée, non exécutable ; original sha 68a817cf870a2e33623c5a96c995b401afed7b6559a214d66f2089da07a20a52 en local
+"""[ngram8_overlap_ctx64_cp1.py — copie de ngram8_overlap_ctx64.py (sha 5c1d5689…) ; union étendue aux masques cp1_selfcheck_masks_{raw,norm}.npz (code_prose_v1 sample reint, f45d3bf6…, build_ctx64_mask_code_prose_v1.py) => 6 masques ; sortie results_ctx64_cp1.json]
+ngram8 overlap restreint aux positions excl_ctx64 (définition reprise de spira_m8/v1c/code/verdict_M8v1c.py l.62-70, 87-89) :
+masque octet = union des 4 masques excl_ctx64 (corpus_diverse_v1c / v1c sample x raw/norm, selfcheck_masks_*.npz, lus en lecture seule) ;
+position 8-gramme i retirée si un octet de [i, i+8) est marqué (même règle cumsum que le verdict pour [s, s+l)). Trains lus par np.fromfile+offset."""
+import numpy as np, hashlib, json, resource, sys, os
+R="/workspace/spira_shared/"
+W={"fc2":"broad/freshcode_v2.bin","fresh_en":"spira_m7/windows/fresh_en.bin","fresh_fr":"spira_m7/windows/fresh_fr.bin","test20k":"spira_m7/windows/test20k.bin","test_v3":"corpus_v3/test.bin"}
+SCN={"fc2":"fc2","fresh_en":"fresh_en","fresh_fr":"fresh_fr","test20k":"test20k_old","test_v3":"test_v3"}  # verdict_M8v1c.py l.27
+SCD={"corpus":(R+"hashing_gen/corpus_diverse_v1c","33577b70c819af76fe4737319e929f4c097b0c72f020e8b07b4baddbb1823439"),
+     "sample":(R+"spira_m8/v1c/sample","3d8b6181cbffb27e360758267e1084197dccad82ae05b868691758e1cf72eed4"),
+     "cp1":(R+"hashing_gen/ngram8_overlap_v1c","f45d3bf6591396706a47a5230d93d095400ffb3f39b5cead976e4b0131fdace6")}
+PFX={"corpus":"","sample":"","cp1":"cp1_"}
+T={"v1c_A_train":(R+"hashing_gen/corpus_diverse_v1c/train.bin","33577b70"),
+   "v1c_sample":(R+"spira_m8/v1c/sample/v1c_sample.bin","3d8b6181"),
+   "m8cp1_ctl_sample":(R+"hashing_gen/code_prose_v1/sample/code_prose_v1_sample_reint.bin","f45d3bf6")}
+def memav():
+    for l in open("/proc/meminfo"):
+        if l.startswith("MemAvailable"): return int(l.split()[1])*1024
+def rss([CAVIARDÉ len=18].getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+def keys(b):
+    n=len(b)-7; k=np.zeros(n,np.uint64)[CAVIARDÉ len=20]8): k|=b[j:j+n].astype(np.uint64)<<np.uint64(8*j)
+    return k
+def sha(p):
+    h=hashlib.sha256()
+    with open(p,"rb"[CAVIARDÉ len=25]iter(lambda:f.read(1<<22),b""): h.update(c)
+    return h.hexdigest()
+if memav()<5.2e9: sys.exit("REFUS MemAvailable<5.2GB")
+SC={}
+for tag,(d,want) in SCD.items():
+    for form in ("raw","norm"):
+        j=json.load(open(f"{d}/{PFX[tag]}selfcheck_dedup64_{form}.json")); assert j["train_sha256"]==want,(tag,form)
+        SC[(tag,form)]=(j,np.load(f"{d}/{PFX[tag]}selfcheck_masks_{form}.npz"))
+def excl_mask(k):
+    m=None
+    for (tag,form),(j,z) in SC[CAVIARDÉ len=18]n=int(z[f"{k}__len"]); e=np.unpackbits(z[f"{k}__excl_ctx64"])[:n].astype(bool); m=e if m is None else (m|e)
+    return m
+win={}
+for w,p in W.items():
+    b=np.fromfile(R+p,np.uint8); s=sha(R+p)
+    for (tag,form),(j,z) in SC.items(): assert j["windows"][SCN[w]]["window_sha256"]==s,(w,tag,form)
+    ex=excl_mask(SCN[w]); assert len(ex)==len(b),(w,len(ex),len(b))
+    c=np.zeros(len(ex)+1,np.int64); c[1:]=np.cumsum(ex); st=np.arange(len(b)-7,dtype=np.int64)
+    keep=(c[st+8]-c[st])==0
+    k=keys(b)[keep]; u,inv=np.unique(k,return_in[CAVIARDÉ len=16]win[w]=dict(u=u,inv=inv,sha=s,n_all=int(len(st)),n_keep=int(keep.sum()),excl_bytes=int(ex.sum()))
+allu=np.unique(np.concatenate([v["u"] for v in win.values()]))
+res={"windows":{},"trains":{},"excl_ctx64_definition":__doc__}
+hits={}; CH=2<<20
+for t,(p,pre) in T.items():
+    s=sha(p); assert s.startswith(pre),(t,s); res["trains"][t]={"path":p,"sha256":s}
+    seen=np.zeros(len(allu),bool); N=os.path.getsize(p); i=0
+    while i<N-7:
+        if memav()<4.5e9: sys.exit("ABORT MemAvailable<4.5GB")
+        b=np.fromfile(p,np.uint8,count=min(N-i,CH+7),offset=i)
+        k=keys(b); idx=np.searchsorted(allu,k); idx[idx==len(allu)]=0
+        seen[idx[allu[idx]==k]]=True; i+=CH
+        if rss()>880: sys.exit("ABORT RSS>880MB")
+    hits[t]=seen
+for w,v in win.items():
+    d={"n_positions_all":v["n_all"],"n_positions_kept_excl_ctx64":v["n_keep"],"n_excl_bytes_union":v["excl_bytes"],"n_distinct_kept":int(len(v["u"])),"sha256":v["sha"]}
+    pos=np.searchsorted(allu,v["u"])
+    for t in T:
+        hd=hits[t][pos]; d[t]={"frac_distinct_seen":float(hd.mean()),"frac_positions_seen":float(hd[v["inv"]].mean())}
+    res["windows"][w]=d
+res["mask_files_sha256"]={f"{tag}/{PFX[tag]}selfcheck_masks_{form}.npz":sha(f"{SCD[tag][0]}/{PFX[tag]}selfcheck_masks_{form}.npz") for tag in SCD for form in ("raw","norm")}; res["n_masks_union"]=len(SC); res["script_sha256"]=sha(__file__); res["peak_rss_MB"]=rss()
+json.dump(res,ope[CAVIARDÉ len=34]__file__))+"/results_ctx64_cp1.json","w"),indent=1[CAVIARDÉ len=20]k:v for k,v in res.items() if k!="excl_ctx64_definition"},indent=1))