results/hashing: SPIRA Hashing scripts, reports, artefacts (final push) (part 6)

results/hashing/covb_v3_eval/v42/verdict_v42.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+"""Checker 24 oct. 09 h 3x (Toronto) — verdict de rétention v4.2 (correctif méthodologique ; verdict_v41.py CONSERVÉ tel quel, non modifié).
+Problème v4.1 (signalé Hashing/Relay 09 h 27, confirmé par Checker) : tokenizers différents ⇒ les spans (tgt_start, tgt_len) IDENTIQUES entre bras ne
+sont qu'un sous-ensemble biaisé (m8_v3 vs old_ref : 565/8000 fc2, 2 494/20 000 test20k, 2 895/20 000 test_v3) et les taux affichés étaient NON APPARIÉS
+(fc2 : 11,50 → 14,45 non apparié, alors que sur les spans communs 5,66 → 0,71).
+Méthode v4.2 (segmentation commune FIGÉE par fenêtre et par paire, en octets) :
+  - frontières de chaque bras = début/fin de ses patches cibles (tgt_start/tgt_len, contigus) ; frontières COMMUNES = intersection, restreinte à la
+    plage d'octets couverte par les deux bras ; unité = [c_k, c_{k+1}) ; chaque unité est pavée exactement par des patches entiers de chaque bras.
+  - unité réussie pour un bras ssi TOUS ses patches dans l'unité sont fused_hit (ou m9g_hit pour v2b) ; unité gardée ssi TOUS les patches des DEUX bras
+    y sont in-vocab (in-vocab pour les deux bras) ET aucun octet exclu par le masque propre (union des masques stricts des corpus des deux bras ;
+    patch exclu ssi E[s:s+l].any(), règle ctx64+span des masques).
+  - fichier de spans figé : verdicts_v42/spans/<BASE><suf>__<TAG>__<w>.npz (unit_start, unit_len) + sha256 enregistré dans le json.
+  - McNemar exact sur les unités appariées ; taux APPARIÉS (hits/unités) rapportés à côté ; bootstrap par blocs de 64 unités, 2 000 tirages, seed 777 (indicatif).
+  - Identité : si les deux bras ont la même segmentation (ex. old_ref fused vs old_ref v2b), unités = patches et discordants == v4.1 (testé).
+Règle : RETENU ssi aucune perte significative (p<0,05) sur aucune fenêtre (fc2, fresh_en, fresh_fr, test20k, test_v3, queries) ET ≥ 1 gain significatif
+sur une fenêtre fraîche propre (queries exclu des gains frais). seen jamais apparié (octets différents par corpus). neither_code / freshcode exclus.
+Usage : verdict_v42.py TAG BASE [SUFFIX KEY]  (ex. old_ref __v2b m9g_hit)  → verdicts_v42/<TAG>__vs_<BASE><suf>.json/.md"""
+import os, sys, json, math, hashlib, numpy as np
+D = os.path.dirname(os.path.abspath(__file__)); H = os.environ.get("HITS", f"{D}/hits_v4"); OUTD = os.environ.get("OUTD", f"{D}/verdicts_v42")
+FRESH = ["fc2", "fresh_en", "fresh_fr", "test20k", "test_v3"]; VW = FRESH + ["queries"]
+tag, base = sys.argv[1], sys.argv[2]; suf = sys.argv[3] if len(sys.argv) > 3 else ""; kref = sys.argv[4] if len(sys.argv) > 4 else "fused_hit"
+MASKS = {"corpus_v3": f"{D}/clean_masks/corpus_v3_full_0450.npz", "broad_local_v1": f"{D}/clean_masks/broad_gutenberg_v1_2134.npz",
+         "broad_gutenberg_v1": f"{D}/clean_masks/broad_gutenberg_v1_2134.npz", "corpus_diverse_v1": f"{D}/clean_masks/broad_gutenberg_v1_2134.npz"}
+def corpus_of(t):
+    for l in open(f"{D}/jobs_v4.tsv"):
+        f = l.rstrip("\n").split("\t")
+        if len(f) >= 3 and f[0] == t: return f[2]
+def binp(b, c):
+    n = b + c; return 1.0 if n == 0 else min(1.0, 2 * sum(math.comb(n, i) for i in range(min(b, c) + 1)) / 2 ** n)
+def sha(*a): h = hashlib.sha256(); [h.update(np.ascontiguousarray(x).tobytes()) for x in a]; return h.hexdigest()
+def units(A, B, E, ka, kb):
+    assert str(A["window_sha256"]) == str(B["window_sha256"]), "octets différents"
+    def arm(Z): s = Z["tgt_start"].astype(np.int64); l = Z["tgt_len"].astype(np.int64); assert np.all(s[1:] == s[:-1] + l[:-1]); return s, s + l
+    sa, ea = arm(A); sb, eb = arm(B); lo, hi = max(sa[0], sb[0]), min(ea[-1], eb[-1])
+    c = np.intersect1d(np.concatenate([sa, ea[-1:]]), np.concatenate([sb, eb[-1:]])); c = c[(c >= lo) & (c <= hi)]
+    us, ue = c[:-1], c[1:]
+    def agg(s, e, ok):  # unité vraie ssi tous les patches du bras dans l'unité sont ok
+        m = (s >= c[0]) & (e <= c[-1]); k = np.searchsorted(c, s[m], side="right") - 1; u = np.ones(len(us), bool); np.logical_and.at(u, k, ok[m]); return u
+    iva, ivb = A["invocab"].astype(bool), B["invocab"].astype(bool)
+    ha = agg(sa, ea, A[ka].astype(bool) & iva); hb = agg(sb, eb, B[kb].astype(bool) & ivb)
+    keep = agg(sa, ea, iva) & agg(sb, eb, ivb)
+    ce = np.concatenate([[0], np.cumsum(E.astype(np.int64))]); clean = (ce[ue] - ce[us]) == 0  # E déjà dilaté (ctx64+span)
+    keep &= clean
+    return us[keep], (ue - us)[keep], ha[keep], hb[keep], {"n_units_all": int(len(us)), "n_units_kept": int(keep.sum()), "excluded_mask": int((~clean).sum()), "byte_range": [int(lo), int(hi)]}
+def boot(x, y, blk=64, nb=2000, seed=777):
+    n = len(x) // blk
+    if n < 2: return None
+    d = y[:n * blk].reshape(n, blk).mean(1) - x[:n * blk].reshape(n, blk).mean(1); bs = d[np.random.default_rng(seed).integers(0, n, (nb, n))].mean(1)
+    return [100 * float(d.mean()), *(100 * np.percentile(bs, [2.5, 97.5])).tolist()]
+ld = lambda t, w, s="": np.load(f"{H}/{t}__{w}{s}.npz")
+mk = sorted({MASKS[c] for c in (corpus_of(tag), corpus_of(base)) if c in MASKS}); MZ = [np.load(m) for m in mk]
+os.makedirs(f"{OUTD}/spans", exist_ok=True); R = {}
+for w in VW:
+    A, B = ld(base, w, suf), ld(tag, w); E = np.zeros(int(B["window_bytes"]), bool)
+    for M in MZ: assert str(M[f"sha__bench_{w}"]) == str(B["window_sha256"]); E |= M[f"bench_{w}__excl"]
+    us, ul, x, y, info = units(A, B, E, kref, "fused_hit")
+    sp = f"{OUTD}/spans/{base}{suf}__{tag}__{w}.npz"; np.savez(sp, unit_start=us, unit_len=ul); sh = sha(us, ul)
+    ro, co = int((x & ~y).sum()), int((~x & y).sum()); p = binp(ro, co); n = len(x)
+    R[w] = {"units": n, **info, "spans_file": os.path.relpath(sp, D), "spans_sha256": sh, "ref_hits": int(x.sum()), "cand_hits": int(y.sum()),
+            "paired_rate_ref_pct": 100 * float(x.mean()) if n else None, "paired_rate_cand_pct": 100 * float(y.mean()) if n else None,
+            "ref_only": ro, "cand_only": co, "p_mcnemar_exact": p, "sig_gain": bool(p < 0.05 and co > ro), "sig_loss": bool(p < 0.05 and ro > co),
+            "boot_units64_delta_ci95_pct": boot(x, y)}
+L = [w for w in VW if R[w]["sig_loss"]]; G = [w for w in FRESH if R[w]["sig_gain"]]
+dec = "RETENU" if (not L and G) else "NON RETENU"
+out = {"tag": tag, "base": base + suf, "base_key": kref, "method": "v4.2 segmentation commune figée (frontières communes, in-vocab deux bras, masque propre union)",
+       "masks": mk, "windows": R, "losses": L, "gains_fresh": G, "gains_queries": [w for w in ["queries"] if R[w]["sig_gain"]], "decision": dec,
+       "script_sha256": hashlib.sha256(open(os.path.abspath(__file__), "rb").read()).hexdigest()}
+f = lambda v: "—" if v is None else f"{v:.2f}"
+rows = [f"# Verdict v4.2 (apparié, segmentation commune figée) — {tag} contre {base}{suf} ({kref})", "", f"**{dec}** (pertes : {', '.join(L) or 'aucune'} ; gains frais : {', '.join(G) or 'aucun'})", "",
+        "| fenêtre | unités appariées (gardées/toutes) | taux APPARIÉ réf % | taux APPARIÉ cand % | ref_only | cand_only | p McNemar exact | DÉCISION | bootstrap Δ % [IC95] blocs 64 unités (indicatif) | sha256 spans (16) |", "|---|---|---|---|---|---|---|---|---|---|"]
+for w in VW:
+    r = R[w]; b = r["boot_units64_delta_ci95_pct"]; d = "GAIN" if r["sig_gain"] else ("PERTE" if r["sig_loss"] else "n.s.")
+    rows.append(f"| {w}{' (à part)' if w == 'queries' else ''} | {r['units']}/{r['n_units_all']} | {f(r['paired_rate_ref_pct'])} | {f(r['paired_rate_cand_pct'])} | {r['ref_only']} | {r['cand_only']} | {r['p_mcnemar_exact']:.2g} | {d} | {'—' if b is None else f'{b[0]:+.2f} [{b[1]:+.2f} ; {b[2]:+.2f}]'} | {r['spans_sha256'][:16]} |")
+rows += ["", f"Masques propres (union des corpus des deux bras) : {', '.join(os.path.basename(m) for m in mk) or 'aucun'}. Unités = segments entre frontières de patch COMMUNES aux deux bras ; unité gardée ssi tous ses patches sont in-vocab dans les DEUX bras et aucun octet exclu ; unité réussie ssi tous ses patches sont des hits. neither_code/freshcode exclus ; seen non apparié."]
+open(f"{OUTD}/{tag}__vs_{base}{suf}.md", "w").write("\n".join(rows) + "\n"); json.dump(out, open(f"{OUTD}/{tag}__vs_{base}{suf}.json", "w"), indent=1)
+print("\n".join(rows))

results/hashing/covb_v3_eval/verdict_covb_v42.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""SPIRA Hashing 24 oct. 2026 ~09 h 39 (Toronto) — verdict APPARIÉ v4.2 pour cov b (PRINCIPAL), via une COPIE OCTET-IDENTIQUE de
+checker/scale/verdict_v42.py (v42/verdict_v42.py, sha256 vérifié ci-dessous ; aucun fichier Checker modifié).
+La copie lit, relativement à son propre dossier v42/ : jobs_v4.tsv (copie de checker/scale/jobs_v4.tsv + 2 lignes Hashing ajoutées à la fin :
+covb_broad → broad_local_v1, covb_v3 → corpus_v3 ; vérifiées à l'exécution) et clean_masks/ (copies des masques Checker, sha dans COPIES.sha256).
+Hits : HITS = dossier de mise en scène <out>/hits/ (copies figées : hits cov b + hits de référence Checker hits_v4, sha256 consignés).
+Méthode = verdict_v42.py tel quel : segmentation commune figée par fenêtre et par paire, unités in-vocab dans les DEUX bras, masque propre union,
+taux APPARIÉS + McNemar exact ; bootstrap blocs 64 unités (indicatif). seen jamais apparié : seen_snn rapporté à part (trained_frac ≥ 0,99).
+Usage : verdict_covb_v42.py TAG CAND_HITS_DIR OUTDIR REF[,REF...]   (REF = tag ou tag:SUFFIX:KEY, ex. old_ref:__v2b:m9g_hit)
+Sortie : OUTDIR/verdicts/<TAG>__vs_<REF>.json/.md (écrits par la copie) + OUTDIR/v42_<TAG>.json (résumé)."""
+import os, sys, json, shutil, hashlib, subprocess, resource
+D = os.path.dirname(os.path.abspath(__file__)); V = f"{D}/v42"
+CHK_SHA = "a055feaa96f5f1344332bf46678929385a4c5c595e51bf8949dd6592c55d5687"  # checker/scale/verdict_v42.py lu le 24 oct. 09 h 35
+CHV4 = "/workspace/spira_shared/checker/scale/hits_v4"
+VW = ["fc2", "fresh_en", "fresh_fr", "test20k", "test_v3", "queries"]
+LOCAL_JOBS = {"covb_broad": "broad_local_v1", "covb_v3": "corpus_v3"}
+def fsha(p): h = hashlib.sha256(); h.update(open(p, "rb").read()); return h.hexdigest()
+def main(tag, cand_dir, out, refs):
+    assert fsha(f"{V}/verdict_v42.py") == CHK_SHA, "copie verdict_v42.py modifiée"
+    jt = {l.split("\t")[0]: l.split("\t")[2] for l in open(f"{V}/jobs_v4.tsv").read().splitlines() if l.count("\t") >= 2}
+    for t, c in LOCAL_JOBS.items(): assert jt.get(t) == c, f"jobs_v4.tsv local : {t} → {jt.get(t)} ≠ {c}"
+    H = f"{out}/hits"; O = f"{out}/verdicts"; os.makedirs(H, exist_ok=True); os.makedirs(O, exist_ok=True)
+    S = {"tag": tag, "method": "v4.2 Checker (copie octet-identique, sha " + CHK_SHA[:16] + "…)", "copied_hits_sha256": {}, "refs": {}}
+    for w in VW:
+        src = f"{cand_dir}/{tag}__{w}.npz"
+        if os.path.exists(src): shutil.copyfile(src, f"{H}/{tag}__{w}.npz"); S["copied_hits_sha256"][f"{tag}__{w}"] = fsha(src)
+    miss_c = [w for w in VW if not os.path.exists(f"{H}/{tag}__{w}.npz")]
+    if miss_c: S["status"] = f"hits candidat manquants : {miss_c} → pas de verdict apparié"; return S
+    for r in refs:
+        base, suf, key = (r.split(":") + ["", "fused_hit"])[:3] if ":" in r else (r, "", "fused_hit")
+        name = base + suf; e = {"base": base, "suffix": suf, "key": key}
+        srcs = {w: f"{CHV4}/{base}__{w}{suf}.npz" for w in VW}; miss = [w for w, p in srcs.items() if not os.path.exists(p)]
+        if miss: e["status"] = f"ABSENT (hits {name} manquants : {miss})"; S["refs"][name] = e; continue
+        for w, p in srcs.items(): shutil.copyfile(p, f"{H}/{base}__{w}{suf}.npz"); S["copied_hits_sha256"][f"{base}__{w}{suf}"] = fsha(p)
+        cmd = [sys.executable, f"{V}/verdict_v42.py", tag, base] + ([suf, key] if suf else [])
+        env = dict(os.environ, HITS=H, OUTD=O)
+        p = subprocess.run(cmd, env=env, capture_output=True, text=True)
+        if p.returncode != 0: e["status"] = "ÉCHEC verdict_v42 : " + p.stderr.strip()[-400:]; S["refs"][name] = e; continue
+        j = json.load(open(f"{O}/{tag}__vs_{name}.json")); assert j["script_sha256"] == CHK_SHA
+        e.update({"status": "OK", "decision": j["decision"], "losses": j["losses"], "gains_fresh": j["gains_fresh"], "masks": [os.path.basename(m) for m in j["masks"]],
+                  "json": os.path.relpath(f"{O}/{tag}__vs_{name}.json", D), "md": os.path.relpath(f"{O}/{tag}__vs_{name}.md", D), "windows": j["windows"]})
+        S["refs"][name] = e
+    S["peak_rss_mb_children"] = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss / 1024
+    return S
+def md(S, primary):
+    f = lambda v: "—" if v is None else f"{v:.2f}"
+    L = []
+    for name, e in S["refs"].items():
+        star = " (DÉCISION PRINCIPALE)" if name == primary else ""
+        if e["status"] != "OK": L += [f"### contre {name}{star} — {e['status']}", ""]; continue
+        L += [f"### contre {name}{star} : **{e['decision']}** (pertes : {', '.join(e['losses']) or 'aucune'} ; gains frais : {', '.join(e['gains_fresh']) or 'aucun'} ; masques union : {', '.join(e['masks']) or 'aucun'})", "",
+              "| fenêtre | unités gardées/toutes | taux APPARIÉ réf % | taux APPARIÉ cand (cov b) % | ref_only | cand_only | p McNemar exact | décision | bootstrap Δ % [IC95] (indicatif) | sha256 spans (16) |", "|---|---|---|---|---|---|---|---|---|---|"]
+        for w in VW:
+            r = e["windows"][w]; b = r["boot_units64_delta_ci95_pct"]; d = "GAIN" if r["sig_gain"] else ("PERTE" if r["sig_loss"] else "n.s.")
+            L.append(f"| {w}{' (à part)' if w == 'queries' else ''} | {r['units']}/{r['n_units_all']} | {f(r['paired_rate_ref_pct'])} | {f(r['paired_rate_cand_pct'])} | {r['ref_only']} | {r['cand_only']} | {r['p_mcnemar_exact']:.2g} | {d} | {'—' if b is None else f'{b[0]:+.2f} [{b[1]:+.2f} ; {b[2]:+.2f}]'} | {r['spans_sha256'][:16]} |")
+        L.append("")
+    return "\n".join(L)
+if __name__ == "__main__":
+    tag, cand, out, refs = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4].split(",")
+    S = main(tag, cand, out, refs); json.dump(S, open(f"{out}/v42_{tag}.json", "w"), indent=1, ensure_ascii=False)
+    print(json.dumps({k: (v.get("decision"), v.get("status")) for k, v in S.get("refs", {}).items()}, ensure_ascii=False)); print(S.get("status", ""))

results/hashing/dg_need_r1/DG_NEED.diff

@@ -0,0 +1,11 @@
+--- run_C_B.copy_r3.sh	2026-10-31 08:20:03.872285075 -0700
++++ run_C_B.dgneed_proposed.sh	2026-10-31 08:20:29.000135782 -0700
+@@ -5,7 +5,7 @@
+ set -euo pipefail
+ S=/workspace/spira_shared; H=$S/checker/recount/v1d_sample_nobroad; RUN=$H/run_C_B; C=$S/spira_m8/v1c/code
+ PY=/workspace/SPIRA-venv/bin/python; DG=$S/hashing_gen/proposed_disk_guard_runC/disk_guard.py
+-DG_MIN=1000000000; DG_NEED=2500000000  # arrêt propre si libre < 1,0 Go ; lancement exigé à ≥ 1,0 + 2,5 Go (empreinte keep=1 ≈ 2,4 Go)
++DG_MIN=1000000000; DG_NEED=2600000000  # arrêt propre si libre < 1,0 Go ; lancement exigé à ≥ 1,0 + 2,6 Go (keep=1 cas haut mesuré 2 525,2 Mo + marge 74,8 Mo ; dg_need_r1)
+ EV=$S/checker/recount/m8_v1d_eval_prep/eval_m8_v1d_seen_fresh_r4.py
+ GO=$H/GO_RUN_C_B_PHILIPPE_ANTOINE
+ export OMP_NUM_THREADS=1 PYTHONDONTWRITEBYTECODE=1 TMPDIR=$RUN/tmp

results/hashing/dg_need_r1/DG_NEED.md

@@ -0,0 +1,34 @@
+# DG_NEED — valeur minimale justifiée (dg_need_r1, à sec, 31 oct. 2026 08:20 heure de Vancouver)
+Cible : checker/recount/runC_ready/proposed_r3/run_C_B.proposed_r3.sh (sha a7611e25…, re-vérifié 08:20) l.8 `DG_NEED=2500000000`, porte l.83 : libre ≥ DG_MIN + DG_NEED.
+
+## Tailles réelles (octets, Mo = 10^6 o)
+| poste (keep 1) | source mesurée | Mo |
+|---|---|---|
+| pré-cal ckpt ctl | v1c_ctl_data ckpt 188 664 992 o | 188,665 |
+| pré-cal ckpt cm | ckpt cm pré-cal v1c 465 569 968 o (max des 2) | 465,570 |
+| principal ckpt ctl | idem ctl | 188,665 |
+| principal ctl.npz | m8_cp1/runs 197 909 389 o (mandataire) | 197,909 |
+| principal ckpt cm | idem cm | 465,570 |
+| principal cm.npz | m8_cp1/runs 446 137 688 o (mandataire) | 446,138 |
+| **sous-total persistant** | | **1 952,52** |
+| pic de rotation keep+1 (1 ckpt cm transitoire) | | 465,57 |
+| évals/logs/TMPDIR/génération, cas haut DISK_BUDGET_r2 (e9d12cb6) | 5 + 2 + 100 + 0 | 107,1 |
+| **minimum justifié (cas haut)** | | **2 525,2** |
+
+## Marge et incertitude
+- ctl ×2, ctl.npz, cm.npz : mandataires non mesurés sur le run C (≈ 1 021 Mo), ±5 % → ±51 Mo. Les ckpt cm (931 Mo) sont mesurés au format exact.
+- TMPDIR 100 Mo est déjà un majorant (le code n'écrit pas dans TMPDIR, cf. audit_r17).
+- Dérive concurrente mesurée : 4 646 850 560 o (06:21) → 4 625 047 552 o (08:20), soit −21,8 Mo en ≈ 2 h ≈ 11 Mo/h ; ≈ 370 Mo sur 33 h. Ce n'est pas à la charge de DG_NEED : DG_MIN (1 Go) assure l'arrêt propre et reprenable (exit 75).
+- **Proposition : DG_NEED = 2 600 000 000** (2 525,2 + 51 incertitude + 23,8 arrondi), porte = 3,60 Go.
+- Variante prudente (dérive incluse) : 2 970 000 000, porte = 3,97 Go.
+
+## Tranchage du 3,03 Go
+3,03 Go = DISK_BUDGET r1 avec TMPDIR 500 Mo + évals 100 Mo, hypothèses réfutées (audit_r17). Il n'y a **aucune justification à le reprendre**. Le 3,02 Go « haut » de CHECKLIST r3 repose sur la même hypothèse : à corriger. L'actuel 2,5 Go est **inférieur de 25,2 Mo** au cas haut mesuré (FAIL marginal).
+
+## df actuel
+08:20 heure de Vancouver : 4 625 047 552 o libres, 97 %, disque 136 782 041 088 o. Relay dit « ~4,4 Go » : c'est **≈ 4,63 Go** (≈ 4,31 Gio, d'où probablement le 4,4).
+
+## Keep 1 possible ?
+Oui. Porte 3,60 Go → marge 1,03 Go. Variante 3,97 Go → marge 0,66 Go. Fin de run estimée : libre ≈ 4,625 − 2,53 − 0,37 de dérive ≈ 1,73 Go, au-dessus de DG_MIN. Keep 2 (+931 Mo de cm et +377 de ctl) : ≈ 3,83 Go, plus 1 Go de DG_MIN = 4,83 Go > 4,63 Go → **impossible**.
+
+Correctif proposé (NON appliqué) : DG_NEED.diff sur la copie run_C_B.copy_r3.sh → run_C_B.dgneed_proposed.sh (l.8, valeur et commentaire ; bash -n OK).

results/hashing/dg_need_r1/SHA256SUMS

@@ -0,0 +1,4 @@
+c37805a5f3c7b70e592b0c277d07f0a3bb37c9e9acb8bcd7f4bce5a58e6ab266  DG_NEED.md
+b17f04126beccad4bf2838788bf4cf0892e7228afc8578e7d58c7ff64f9d123a  DG_NEED.diff
+a7611e258bb98225ae22c337570eb39876b898a844fce8cb923dc631e626dbb7  run_C_B.copy_r3.sh
+2478d901fe37365b600985737336f585778af0ca5b6a589dc4e83e8c5608105f  run_C_B.dgneed_proposed.sh

results/hashing/dg_need_r1/run_C_B.copy_r3.sh

@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+# SPIRA Checker — M8 v1d, option C (pré-cal à budget complet) sur l'échantillon B. PRÉPARÉ, NON LANCÉ. CANDIDAT NON ADOPTÉ.
+# Ne fait RIEN sans le fichier GO signé par Philippe-Antoine (le Checker ne le crée jamais). Tout écart de sha => sortie 3.
+# Aucun code d'entraînement modifié : appelle spira_m8/v1c/code/{run_guarded_v2,train_m8v1c,freeze_scale}.py et eval r4 tels quels.
+set -euo pipefail
+S=/workspace/spira_shared; H=$S/checker/recount/v1d_sample_nobroad; RUN=$H/run_C_B; C=$S/spira_m8/v1c/code
+PY=/workspace/SPIRA-venv/bin/python; DG=$S/hashing_gen/proposed_disk_guard_runC/disk_guard.py
+DG_MIN=1000000000; DG_NEED=2500000000  # arrêt propre si libre < 1,0 Go ; lancement exigé à ≥ 1,0 + 2,5 Go (empreinte keep=1 ≈ 2,4 Go)
+EV=$S/checker/recount/m8_v1d_eval_prep/eval_m8_v1d_seen_fresh_r4.py
+GO=$H/GO_RUN_C_B_PHILIPPE_ANTOINE
+export OMP_NUM_THREADS=1 PYTHONDONTWRITEBYTECODE=1 TMPDIR=$RUN/tmp
+die(){ echo "REFUS: $*" >&2; exit 3; }
+# 1) porte : GO signé, qui doit citer le sha de CE script
+[ -f "$GO" ] || die "pas de $GO (oui écrit de Philippe-Antoine requis)"
+ME=$(sha256sum "$0" | cut -d' ' -f1)
+grep -q "^OUI Philippe-Antoine Robert" "$GO" || die "GO non signé « OUI Philippe-Antoine Robert »"
+grep -q "run_C_B.sh $ME" "$GO" || die "GO ne cite pas run_C_B.sh $ME"
+# 2) shas vérifiés au démarrage
+while read -r h f; do
+  [ -f "$f" ] || die "absent : $f"; [ "$(sha256sum "$f" | cut -d' ' -f1)" = "$h" ] || die "sha ≠ : $f"
+done <<L
+b7404e88146b1f80572a8d5a41c50075e61f29e3e4088a16e5111dac94c75609 $H/v1d_nobroad_B.bin
+992d20aa8a447efb16b90b6693414bb20def980e089f6e8a2db51e23ad99068b $H/precal_prefix_nobroad_B.bin
+903970c3319a2770432565ecf5a8616bf3ae34c4f8384669221a93ab998833e5 $S/hashing_gen/corpus_diverse_v1d/train.bin
+b052096b0c1e2b301cd79921502f85e61912f209fc0027151bbf82237de8c2e3 $S/broad/freshcode_v2.bin
+67ff20b26ce5e3401cea57c817b1a8417e95f6cd131fd8fb631cd5cd8625331a $S/spira_m7/windows/fresh_en.bin
+875f0906a320e3f7b87e446aab646cf49c74786cb1a0db78b54fa4c62e872715 $S/spira_m7/windows/fresh_fr.bin
+ce0526c62048f8e590a96cca5393a87c2e631f194eb4a2e974c584fd443b6a22 $S/spira_m7/windows/test20k.bin
+a6730bbe877e7b28a5529793b27a0b59f16d04db05ea895f0ee6d4f91807ca84 $S/corpus_v3/test.bin
+5aab38d62b2f000df6a913d2574ccd314300e350ccc76e71e16768ed5c73b706 $S/spira_m7/windows/queries.bin
+1c6f64b44403932905b2f3ffee80db90de9b979e8d27441d6e58b3cd85a28bc1 $C/run_guarded_v2.py
+4049327ca9882c7baccd3f106a0b6ea4a58335990a6cfda24e39be530157ada7 $C/train_m8v1c.py
+dafc1de2654fdf3171ed8284994442d9b1efc8a8ee005715478e723cb4c9ba48 $C/freeze_scale.py
+8fc571577fdd48c99e0c070d175f4544a21b4084d99896ea390de626e156fe77 $EV
+363c16c9f3b135a8671d3d79eb6760437e8e1b1da7462083227d424dad4f5ff1 $S/hashing_gen/proposed_disk_guard_runC/disk_guard.py
+889c603f0d17cb54060951bcf4c4f9b8c9ebd9e52b392c70209bbb9755d797d9 /usr/bin/python3.13
+81c5998fb4c64b7be806cb85fbcff8bd9393b8c9fa55918bbe69d552183d019f $S/checker/recount/runC_ready/proposed_r3/pkg_freeze.txt
+cddb0099ff51c991b2d2a19ac82d3d347f336749f75655bb38ab50077b088f1f /workspace/SPIRA/python/spira/__init__.py
+d8222aabb3d4617bdce64977f38c454f69615c8681cb1652e9f44c36bf7a1c04 /workspace/SPIRA/python/spira/ept.py
+f26903e19cd92963e84d6271fc296d4998c4b364ce28be712241236391c2ab02 /workspace/SPIRA/python/spira/hashing.py
+e48148a81b0cce703ffa25327c1db228939323f88d5b945a25cf88f6fec91d78 /workspace/SPIRA/python/spira/hippocampus.py
+81021ea0bebd1a1743e2b4f232a825c532db56b9cee2f59df401b8b1df24d6ab /workspace/SPIRA/python/spira/model.py
+beabae44a2833b29288f8dd8c5b20c7e688d96b790570d3d35ae2ead810478a3 /workspace/SPIRA/python/spira/spiking.py
+09295a80660f17925ae23765ce8cbd7ff7ceae968d5f2f89349f1cb74c0b9e11 /workspace/SPIRA-venv/lib/python3.13/site-packages/numpy/__init__.py
+344c87cc95d8d56eb4addbefe63e798107e1ab2b32141097dd5623ebe166f7ba /workspace/SPIRA-venv/lib/python3.13/site-packages/numpy/version.py
+405abe5c8f52c1d15b71bafbd155693237b6e1742bc598638b7fc0626bcc1576 /workspace/SPIRA-venv/lib/python3.13/site-packages/numpy/_core/_multiarray_umath.cpython-313-x86_64-linux-gnu.so
+f15d9f7f32bd87fc0082e3d4eb46304b0fafbfd27bae712811c0bae20cf3d563 /workspace/SPIRA-venv/lib/python3.13/site-packages/numpy/linalg/_umath_linalg.cpython-313-x86_64-linux-gnu.so
+caab50405e73eba7981f73a63fb1709139aed55383501c3e830e174cb02b8d3b $S/hashing_gen/m8_v1b/code/countmin.py
+7d9ed224012184b19723c0254982878bc0a56102ef4722dd3d443ed6486869c1 $S/checker/windows_v2.py
+e2c8cd1deabc228faed302bebdb81bf8d10405df63e393b4a8ddb6152401c10e $S/checker/eval_ref_v3.py
+56efcbc6c12f71221ddfb8d57b5e3fbb29d9b054cc363aa062fa1e396639bb7e $S/checker/scale/bench_common.py
+aeba27c3c695987bba722f725035aa2fbe6f69be82a527c06dac6d56eabc4c64 $S/checker/recount/m8_v1d_eval_prep/stats_m8_v1d.py
+29e6d015a08b97a19d6f05721e09cdd7f6bf852b8e26c9f24bbe3f2cdc1ef7ca $S/checker/recount/m8_v1d_eval_prep/leak16_m8_v1d.py
+71b577724fe46d2f221415dc7ceac2c063cfa412f98a1d8c02ee525fc9f25c02 $S/checker/recount/v1d_sample_nobroad/masks_B/selfcheck_dedup64_norm.json
+b2ec9f64d6f6ba594f330fa7b1da0b6aaf39b8fd4a257a2edd5583d1eab10e22 $S/checker/recount/v1d_sample_nobroad/masks_B/selfcheck_dedup64_raw.json
+7ac2ea01c4454ce739c0a900e8c60fad2b676dbc4be29a7414eee3be7bb83135 $S/checker/recount/v1d_sample_nobroad/masks_B/selfcheck_masks_norm.npz
+7ac2ea01c4454ce739c0a900e8c60fad2b676dbc4be29a7414eee3be7bb83135 $S/checker/recount/v1d_sample_nobroad/masks_B/selfcheck_masks_raw.npz
+e2cde9351b8ffff8f357d8c850c722d87a3d0e45839f84a12d8deea4f9620d0d $S/checker/recount/v1d_sample_nobroad/masks_B/selfcheck_train_ranges_norm.npz
+e2cde9351b8ffff8f357d8c850c722d87a3d0e45839f84a12d8deea4f9620d0d $S/checker/recount/v1d_sample_nobroad/masks_B/selfcheck_train_ranges_raw.npz
+1b1075619cfee614e6173342ae65377eb7b7720a555394bc94e90f8db870fd58 $S/hashing_gen/corpus_diverse_v1d/SHA256SUMS
+1350f86f2a839e0552be4135a7cc0a60723bcdd4e10ef4a260f6300788fd42ac $S/hashing_gen/corpus_diverse_v1d/build.log
+c0e97baf10dfd7c0353b0db156b3f875cc432d954e3281ee77dbb7d7bd7479cb $S/hashing_gen/corpus_diverse_v1d/build_v1d.py
+cace49cb7b37a57ae2d7a1959a5972560dcf529d1690640be183a4f5197a4f48 $S/hashing_gen/corpus_diverse_v1d/lcs_kcode_broad.json
+a91a17f7a9c254d925150f989de09ec0583cc027336bdb6be47cdf343ae7ad37 $S/hashing_gen/corpus_diverse_v1d/lcs_kcode_broad.py
+89e34682bc7a850f713c22bd3777c7c5c5fa4696b8a0d039711b1dafb0c0cda6 $S/hashing_gen/corpus_diverse_v1d/manifest.json
+52e1b9464caa1b727f05ae4c88c923b701b8a67b552032aa3bdb3f862d723e6c $S/hashing_gen/corpus_diverse_v1d/manifest_r2.json
+6fd789c63f37f60c177b0b865da4a4670a6276f53543361cb23abf1d3cc82090 $S/hashing_gen/corpus_diverse_v1d/selfcheck_dedup64.py
+fb628ea5f77477dcac9e275c37d437f6091f57a2468f9f4ec7ef11a2bc637632 $S/hashing_gen/corpus_diverse_v1d/selfcheck_dedup64_norm.json
+2b3b7c5b96c3e84df5073a76e16219f86bd912bb209c0a3fd5cb9515322b9410 $S/hashing_gen/corpus_diverse_v1d/selfcheck_dedup64_raw.json
+cefc96926246eddb33824c6941e21bb4201970bccf2fcb3389624178c528cbe8 $S/hashing_gen/corpus_diverse_v1d/selfcheck_masks_norm.npz
+cefc96926246eddb33824c6941e21bb4201970bccf2fcb3389624178c528cbe8 $S/hashing_gen/corpus_diverse_v1d/selfcheck_masks_raw.npz
+cb87b2bb5c93a49be1284cd3311fb48a4e3fe2ce2a62c8105ffbbbd579ddbc53 $S/hashing_gen/corpus_diverse_v1d/selfcheck_norm.log
+f9276bf52c87cf3194058e0d91190a2b909e327ec6ea77217c6c23ac368258a7 $S/hashing_gen/corpus_diverse_v1d/selfcheck_raw.log
+4d14b14a2aea0a1b24afd228d4cc3a396264989553730e05e6b39b75234bca4b $S/hashing_gen/corpus_diverse_v1d/selfcheck_train_ranges_norm.npz
+4d14b14a2aea0a1b24afd228d4cc3a396264989553730e05e6b39b75234bca4b $S/hashing_gen/corpus_diverse_v1d/selfcheck_train_ranges_raw.npz
+L
+[ "$(readlink -f "$PY")" = /usr/bin/python3.13 ] || die "interpréteur ≠ /usr/bin/python3.13"
+[ "$($PY -c 'import importlib.metadata as m; print(chr(10).join(sorted(f"{d.name}=={d.version}" for d in m.distributions())))' | sha256sum | cut -d' ' -f1)" = "$(sha256sum < $S/checker/recount/runC_ready/proposed_r3/pkg_freeze.txt | cut -d' ' -f1)" ] || die "paquets ≠ pkg_freeze.txt"
+[ -f $S/spira_m8/GO_CHECKER_DEDUP_V1D ] || die "drapeau GO_CHECKER_DEDUP_V1D absent (exigé par eval r4)"
+MA=$(awk '/MemAvailable/{print $2*1024}' /proc/meminfo); [ "$MA" -ge 6500000000 ] || die "MemAvailable $MA < 6,5 Go"
+[ ! -e "$RUN/precal/PRECAL_HORS_BORNES.json" ] || die "pré-cal déjà hors bornes : NON MESURABLE, pas de relance"
+mkdir -p "$RUN"/{precal,runs,eval,logs,tmp}
+FREE=$(df -B1 --output=avail "$RUN" | tail -1); [ "$FREE" -ge $((DG_MIN+DG_NEED)) ] || die "disque libre $FREE < $((DG_MIN+DG_NEED))"
+G(){ local n=$1; shift; nice -n 10 $PY $C/run_guarded_v2.py --name "$n" --log "$RUN/logs/$n.log" --guard-log "$RUN/logs/guard_$n.log" \
+  --peak 2500000000 --min-launch 5200000000 --floor 4000000000 --rss-cap 3500000000 --mode checkpoint --max-heavy 4 --nice 10 -- $PY $DG --path $RUN --min-free $DG_MIN --poll 30 --marker $RUN/logs/DISK_LOW_$n.json -- "$@"; }
+# 3) pré-cal C, budget complet, séquentielle (ctl puis cm), préfixe B, sans fenêtre
+for A in ctl cm; do
+  [ -f "$RUN/precal/precal_$A.json" ] || G precal_$A $PY $C/train_m8v1c.py --arm $A --data $H/precal_prefix_nobroad_B.bin \
+    --outdir $RUN/precal/out_$A --ckpt-dir $RUN/precal/ckpt_$A --resume auto --fresh --ckpt-keep 1 \
+    --snn-tokens 600000 --am-tokens 1200000 --epochs 2 --windows "" --no-save --precal-out $RUN/precal/precal_$A.json
+done
+# 4) gel : sortie 10 => PRECAL_HORS_BORNES, arrêt (NON MESURABLE, pas de sauvetage)
+set +e; $PY $C/freeze_scale.py $RUN/precal; rc=$?; set -e
+[ $rc -eq 0 ] || { echo "freeze_scale rc=$rc : NON MESURABLE, arrêt"; exit 10; }
+# 5) runs principaux, budget égal, échantillon B
+for A in ctl cm; do
+  [ -f "$RUN/runs/$A/$A.npz" ] || G main_$A $PY $C/train_m8v1c.py --arm $A --data $H/v1d_nobroad_B.bin --outdir $RUN/runs/$A \
+    --resume auto --fresh --ckpt-keep 1 --snn-tokens 600000 --am-tokens 1200000 --epochs 2 --maxpos 4000 --frozen-scale-json $RUN/precal/FROZEN_SCALE.json
+done
+# 6) éval r4 (leak16 diagnostic seul)
+G eval_r4 $PY $EV --ref $RUN/runs/ctl/ctl.npz --cand $RUN/runs/cm/cm.npz --ref-arm ctl --cand-arm cm \
+  --data $H/v1d_nobroad_B.bin --data-sha256 b7404e88146b1f80572a8d5a41c50075e61f29e3e4088a16e5111dac94c75609 \
+  --snn-tokens 600000 --frozen-scale $RUN/precal/FROZEN_SCALE.json --corpus-sc $S/hashing_gen/corpus_diverse_v1d \
+  --sample-sc $H/masks_B --flag-dedup $S/spira_m8/GO_CHECKER_DEDUP_V1D --outdir $RUN/eval --maxpos 4000
+echo FIN

results/hashing/dg_need_r1/run_C_B.dgneed_proposed.sh

@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+# SPIRA Checker — M8 v1d, option C (pré-cal à budget complet) sur l'échantillon B. PRÉPARÉ, NON LANCÉ. CANDIDAT NON ADOPTÉ.
+# Ne fait RIEN sans le fichier GO signé par Philippe-Antoine (le Checker ne le crée jamais). Tout écart de sha => sortie 3.
+# Aucun code d'entraînement modifié : appelle spira_m8/v1c/code/{run_guarded_v2,train_m8v1c,freeze_scale}.py et eval r4 tels quels.
+set -euo pipefail
+S=/workspace/spira_shared; H=$S/checker/recount/v1d_sample_nobroad; RUN=$H/run_C_B; C=$S/spira_m8/v1c/code
+PY=/workspace/SPIRA-venv/bin/python; DG=$S/hashing_gen/proposed_disk_guard_runC/disk_guard.py
+DG_MIN=1000000000; DG_NEED=2600000000  # arrêt propre si libre < 1,0 Go ; lancement exigé à ≥ 1,0 + 2,6 Go (keep=1 cas haut mesuré 2 525,2 Mo + marge 74,8 Mo ; dg_need_r1)
+EV=$S/checker/recount/m8_v1d_eval_prep/eval_m8_v1d_seen_fresh_r4.py
+GO=$H/GO_RUN_C_B_PHILIPPE_ANTOINE
+export OMP_NUM_THREADS=1 PYTHONDONTWRITEBYTECODE=1 TMPDIR=$RUN/tmp
+die(){ echo "REFUS: $*" >&2; exit 3; }
+# 1) porte : GO signé, qui doit citer le sha de CE script
+[ -f "$GO" ] || die "pas de $GO (oui écrit de Philippe-Antoine requis)"
+ME=$(sha256sum "$0" | cut -d' ' -f1)
+grep -q "^OUI Philippe-Antoine Robert" "$GO" || die "GO non signé « OUI Philippe-Antoine Robert »"
+grep -q "run_C_B.sh $ME" "$GO" || die "GO ne cite pas run_C_B.sh $ME"
+# 2) shas vérifiés au démarrage
+while read -r h f; do
+  [ -f "$f" ] || die "absent : $f"; [ "$(sha256sum "$f" | cut -d' ' -f1)" = "$h" ] || die "sha ≠ : $f"
+done <<L
+b7404e88146b1f80572a8d5a41c50075e61f29e3e4088a16e5111dac94c75609 $H/v1d_nobroad_B.bin
+992d20aa8a447efb16b90b6693414bb20def980e089f6e8a2db51e23ad99068b $H/precal_prefix_nobroad_B.bin
+903970c3319a2770432565ecf5a8616bf3ae34c4f8384669221a93ab998833e5 $S/hashing_gen/corpus_diverse_v1d/train.bin
+b052096b0c1e2b301cd79921502f85e61912f209fc0027151bbf82237de8c2e3 $S/broad/freshcode_v2.bin
+67ff20b26ce5e3401cea57c817b1a8417e95f6cd131fd8fb631cd5cd8625331a $S/spira_m7/windows/fresh_en.bin
+875f0906a320e3f7b87e446aab646cf49c74786cb1a0db78b54fa4c62e872715 $S/spira_m7/windows/fresh_fr.bin
+ce0526c62048f8e590a96cca5393a87c2e631f194eb4a2e974c584fd443b6a22 $S/spira_m7/windows/test20k.bin
+a6730bbe877e7b28a5529793b27a0b59f16d04db05ea895f0ee6d4f91807ca84 $S/corpus_v3/test.bin
+5aab38d62b2f000df6a913d2574ccd314300e350ccc76e71e16768ed5c73b706 $S/spira_m7/windows/queries.bin
+1c6f64b44403932905b2f3ffee80db90de9b979e8d27441d6e58b3cd85a28bc1 $C/run_guarded_v2.py
+4049327ca9882c7baccd3f106a0b6ea4a58335990a6cfda24e39be530157ada7 $C/train_m8v1c.py
+dafc1de2654fdf3171ed8284994442d9b1efc8a8ee005715478e723cb4c9ba48 $C/freeze_scale.py
+8fc571577fdd48c99e0c070d175f4544a21b4084d99896ea390de626e156fe77 $EV
+363c16c9f3b135a8671d3d79eb6760437e8e1b1da7462083227d424dad4f5ff1 $S/hashing_gen/proposed_disk_guard_runC/disk_guard.py
+889c603f0d17cb54060951bcf4c4f9b8c9ebd9e52b392c70209bbb9755d797d9 /usr/bin/python3.13
+81c5998fb4c64b7be806cb85fbcff8bd9393b8c9fa55918bbe69d552183d019f $S/checker/recount/runC_ready/proposed_r3/pkg_freeze.txt
+cddb0099ff51c991b2d2a19ac82d3d347f336749f75655bb38ab50077b088f1f /workspace/SPIRA/python/spira/__init__.py
+d8222aabb3d4617bdce64977f38c454f69615c8681cb1652e9f44c36bf7a1c04 /workspace/SPIRA/python/spira/ept.py
+f26903e19cd92963e84d6271fc296d4998c4b364ce28be712241236391c2ab02 /workspace/SPIRA/python/spira/hashing.py
+e48148a81b0cce703ffa25327c1db228939323f88d5b945a25cf88f6fec91d78 /workspace/SPIRA/python/spira/hippocampus.py
+81021ea0bebd1a1743e2b4f232a825c532db56b9cee2f59df401b8b1df24d6ab /workspace/SPIRA/python/spira/model.py
+beabae44a2833b29288f8dd8c5b20c7e688d96b790570d3d35ae2ead810478a3 /workspace/SPIRA/python/spira/spiking.py
+09295a80660f17925ae23765ce8cbd7ff7ceae968d5f2f89349f1cb74c0b9e11 /workspace/SPIRA-venv/lib/python3.13/site-packages/numpy/__init__.py
+344c87cc95d8d56eb4addbefe63e798107e1ab2b32141097dd5623ebe166f7ba /workspace/SPIRA-venv/lib/python3.13/site-packages/numpy/version.py
+405abe5c8f52c1d15b71bafbd155693237b6e1742bc598638b7fc0626bcc1576 /workspace/SPIRA-venv/lib/python3.13/site-packages/numpy/_core/_multiarray_umath.cpython-313-x86_64-linux-gnu.so
+f15d9f7f32bd87fc0082e3d4eb46304b0fafbfd27bae712811c0bae20cf3d563 /workspace/SPIRA-venv/lib/python3.13/site-packages/numpy/linalg/_umath_linalg.cpython-313-x86_64-linux-gnu.so
+caab50405e73eba7981f73a63fb1709139aed55383501c3e830e174cb02b8d3b $S/hashing_gen/m8_v1b/code/countmin.py
+7d9ed224012184b19723c0254982878bc0a56102ef4722dd3d443ed6486869c1 $S/checker/windows_v2.py
+e2c8cd1deabc228faed302bebdb81bf8d10405df63e393b4a8ddb6152401c10e $S/checker/eval_ref_v3.py
+56efcbc6c12f71221ddfb8d57b5e3fbb29d9b054cc363aa062fa1e396639bb7e $S/checker/scale/bench_common.py
+aeba27c3c695987bba722f725035aa2fbe6f69be82a527c06dac6d56eabc4c64 $S/checker/recount/m8_v1d_eval_prep/stats_m8_v1d.py
+29e6d015a08b97a19d6f05721e09cdd7f6bf852b8e26c9f24bbe3f2cdc1ef7ca $S/checker/recount/m8_v1d_eval_prep/leak16_m8_v1d.py
+71b577724fe46d2f221415dc7ceac2c063cfa412f98a1d8c02ee525fc9f25c02 $S/checker/recount/v1d_sample_nobroad/masks_B/selfcheck_dedup64_norm.json
+b2ec9f64d6f6ba594f330fa7b1da0b6aaf39b8fd4a257a2edd5583d1eab10e22 $S/checker/recount/v1d_sample_nobroad/masks_B/selfcheck_dedup64_raw.json
+7ac2ea01c4454ce739c0a900e8c60fad2b676dbc4be29a7414eee3be7bb83135 $S/checker/recount/v1d_sample_nobroad/masks_B/selfcheck_masks_norm.npz
+7ac2ea01c4454ce739c0a900e8c60fad2b676dbc4be29a7414eee3be7bb83135 $S/checker/recount/v1d_sample_nobroad/masks_B/selfcheck_masks_raw.npz
+e2cde9351b8ffff8f357d8c850c722d87a3d0e45839f84a12d8deea4f9620d0d $S/checker/recount/v1d_sample_nobroad/masks_B/selfcheck_train_ranges_norm.npz
+e2cde9351b8ffff8f357d8c850c722d87a3d0e45839f84a12d8deea4f9620d0d $S/checker/recount/v1d_sample_nobroad/masks_B/selfcheck_train_ranges_raw.npz
+1b1075619cfee614e6173342ae65377eb7b7720a555394bc94e90f8db870fd58 $S/hashing_gen/corpus_diverse_v1d/SHA256SUMS
+1350f86f2a839e0552be4135a7cc0a60723bcdd4e10ef4a260f6300788fd42ac $S/hashing_gen/corpus_diverse_v1d/build.log
+c0e97baf10dfd7c0353b0db156b3f875cc432d954e3281ee77dbb7d7bd7479cb $S/hashing_gen/corpus_diverse_v1d/build_v1d.py
+cace49cb7b37a57ae2d7a1959a5972560dcf529d1690640be183a4f5197a4f48 $S/hashing_gen/corpus_diverse_v1d/lcs_kcode_broad.json
+a91a17f7a9c254d925150f989de09ec0583cc027336bdb6be47cdf343ae7ad37 $S/hashing_gen/corpus_diverse_v1d/lcs_kcode_broad.py
+89e34682bc7a850f713c22bd3777c7c5c5fa4696b8a0d039711b1dafb0c0cda6 $S/hashing_gen/corpus_diverse_v1d/manifest.json
+52e1b9464caa1b727f05ae4c88c923b701b8a67b552032aa3bdb3f862d723e6c $S/hashing_gen/corpus_diverse_v1d/manifest_r2.json
+6fd789c63f37f60c177b0b865da4a4670a6276f53543361cb23abf1d3cc82090 $S/hashing_gen/corpus_diverse_v1d/selfcheck_dedup64.py
+fb628ea5f77477dcac9e275c37d437f6091f57a2468f9f4ec7ef11a2bc637632 $S/hashing_gen/corpus_diverse_v1d/selfcheck_dedup64_norm.json
+2b3b7c5b96c3e84df5073a76e16219f86bd912bb209c0a3fd5cb9515322b9410 $S/hashing_gen/corpus_diverse_v1d/selfcheck_dedup64_raw.json
+cefc96926246eddb33824c6941e21bb4201970bccf2fcb3389624178c528cbe8 $S/hashing_gen/corpus_diverse_v1d/selfcheck_masks_norm.npz
+cefc96926246eddb33824c6941e21bb4201970bccf2fcb3389624178c528cbe8 $S/hashing_gen/corpus_diverse_v1d/selfcheck_masks_raw.npz
+cb87b2bb5c93a49be1284cd3311fb48a4e3fe2ce2a62c8105ffbbbd579ddbc53 $S/hashing_gen/corpus_diverse_v1d/selfcheck_norm.log
+f9276bf52c87cf3194058e0d91190a2b909e327ec6ea77217c6c23ac368258a7 $S/hashing_gen/corpus_diverse_v1d/selfcheck_raw.log
+4d14b14a2aea0a1b24afd228d4cc3a396264989553730e05e6b39b75234bca4b $S/hashing_gen/corpus_diverse_v1d/selfcheck_train_ranges_norm.npz
+4d14b14a2aea0a1b24afd228d4cc3a396264989553730e05e6b39b75234bca4b $S/hashing_gen/corpus_diverse_v1d/selfcheck_train_ranges_raw.npz
+L
+[ "$(readlink -f "$PY")" = /usr/bin/python3.13 ] || die "interpréteur ≠ /usr/bin/python3.13"
+[ "$($PY -c 'import importlib.metadata as m; print(chr(10).join(sorted(f"{d.name}=={d.version}" for d in m.distributions())))' | sha256sum | cut -d' ' -f1)" = "$(sha256sum < $S/checker/recount/runC_ready/proposed_r3/pkg_freeze.txt | cut -d' ' -f1)" ] || die "paquets ≠ pkg_freeze.txt"
+[ -f $S/spira_m8/GO_CHECKER_DEDUP_V1D ] || die "drapeau GO_CHECKER_DEDUP_V1D absent (exigé par eval r4)"
+MA=$(awk '/MemAvailable/{print $2*1024}' /proc/meminfo); [ "$MA" -ge 6500000000 ] || die "MemAvailable $MA < 6,5 Go"
+[ ! -e "$RUN/precal/PRECAL_HORS_BORNES.json" ] || die "pré-cal déjà hors bornes : NON MESURABLE, pas de relance"
+mkdir -p "$RUN"/{precal,runs,eval,logs,tmp}
+FREE=$(df -B1 --output=avail "$RUN" | tail -1); [ "$FREE" -ge $((DG_MIN+DG_NEED)) ] || die "disque libre $FREE < $((DG_MIN+DG_NEED))"
+G(){ local n=$1; shift; nice -n 10 $PY $C/run_guarded_v2.py --name "$n" --log "$RUN/logs/$n.log" --guard-log "$RUN/logs/guard_$n.log" \
+  --peak 2500000000 --min-launch 5200000000 --floor 4000000000 --rss-cap 3500000000 --mode checkpoint --max-heavy 4 --nice 10 -- $PY $DG --path $RUN --min-free $DG_MIN --poll 30 --marker $RUN/logs/DISK_LOW_$n.json -- "$@"; }
+# 3) pré-cal C, budget complet, séquentielle (ctl puis cm), préfixe B, sans fenêtre
+for A in ctl cm; do
+  [ -f "$RUN/precal/precal_$A.json" ] || G precal_$A $PY $C/train_m8v1c.py --arm $A --data $H/precal_prefix_nobroad_B.bin \
+    --outdir $RUN/precal/out_$A --ckpt-dir $RUN/precal/ckpt_$A --resume auto --fresh --ckpt-keep 1 \
+    --snn-tokens 600000 --am-tokens 1200000 --epochs 2 --windows "" --no-save --precal-out $RUN/precal/precal_$A.json
+done
+# 4) gel : sortie 10 => PRECAL_HORS_BORNES, arrêt (NON MESURABLE, pas de sauvetage)
+set +e; $PY $C/freeze_scale.py $RUN/precal; rc=$?; set -e
+[ $rc -eq 0 ] || { echo "freeze_scale rc=$rc : NON MESURABLE, arrêt"; exit 10; }
+# 5) runs principaux, budget égal, échantillon B
+for A in ctl cm; do
+  [ -f "$RUN/runs/$A/$A.npz" ] || G main_$A $PY $C/train_m8v1c.py --arm $A --data $H/v1d_nobroad_B.bin --outdir $RUN/runs/$A \
+    --resume auto --fresh --ckpt-keep 1 --snn-tokens 600000 --am-tokens 1200000 --epochs 2 --maxpos 4000 --frozen-scale-json $RUN/precal/FROZEN_SCALE.json
+done
+# 6) éval r4 (leak16 diagnostic seul)
+G eval_r4 $PY $EV --ref $RUN/runs/ctl/ctl.npz --cand $RUN/runs/cm/cm.npz --ref-arm ctl --cand-arm cm \
+  --data $H/v1d_nobroad_B.bin --data-sha256 b7404e88146b1f80572a8d5a41c50075e61f29e3e4088a16e5111dac94c75609 \
+  --snn-tokens 600000 --frozen-scale $RUN/precal/FROZEN_SCALE.json --corpus-sc $S/hashing_gen/corpus_diverse_v1d \
+  --sample-sc $H/masks_B --flag-dedup $S/spira_m8/GO_CHECKER_DEDUP_V1D --outdir $RUN/eval --maxpos 4000
+echo FIN

results/hashing/gap_vs_size/REPORT.md

@@ -0,0 +1,27 @@
+# Écart vu/frais vs taille du corpus — diagnostic léger (Hashing)
+Lecture seule de résultats existants (gap_vs_size.py → results.json). Aucun entraînement, aucune éval relancée, pic RSS 18 Mo, nice 10. Métrique : fused_invocab_pct ; écart = seen − moyenne(fc2, fresh_en, fresh_fr, test20k, test_v3), neither_code exclu.
+
+| run | train (o) | sha train | octets lus SNN | seen | moy. frais (n fen.) | écart |
+|---|---|---|---|---|---|---|
+| v1b ctl | 35 128 951 | 9ea562e4 | 2 791 343 | 98,440 | 2,012 (5) | 96,428 |
+| v1c ctl | 35 948 296 | 3d8b6181 | 2 706 257 | 93,183 | 3,999 (5) | 89,184 |
+| old_ref (plain) | 12 468 350 | e12b6586 | non trouvé | 98,330 | 4,440 (5) | 93,890 |
+| broad14_ref (plain) | 34 835 257 | eda07017 | non trouvé | 91,934 | 4,786 (4) | 87,148 |
+| m8_broad14 (cm) | 34 835 257 | eda07017 | non trouvé | 91,819 | 6,444 (4) | 85,376 |
+| m8_v3 (cm) | 93 284 487 | aabd912d | non trouvé | 91,945 | 5,773 (5) | 86,172 |
+| bg1_m8 (cm) | 290 810 502 | 5af6398f | 2 211 450 | 8,990* | 7,714 (4) | 1,276* |
+
+Sources : v1b hashing_gen/m8_v1b/runs/results.jsonl (5a722834), v1c spira_m8/v1c_ctl_data/runs/results.jsonl (6d6898f5), broad checker/scale/results_v4.jsonl (lecture seule ; sha dans results.json), snn bg1 = checker/scale/cm_bg1d_m8.cov.json. run_iii ratio_ckpt : non utilisé (points précal 60k ticks, aucune éval fraîche).
+
+## Comparabilité — variables confondantes
+1. **La taille n'est pas l'axe réel** : v1b et v1c sont entraînés sur des échantillons de ~35 Mo (corpus pleins 530 Mo et 503 Mo), et le SNN ne lit que ~2,7–2,8 Mo dans tous les runs mesurés (budget fixe 600k ticks SNN / 1,2M AM, 2 époques). Augmenter le corpus ne change pas ce qui est lu, seulement la composition et le vocabulaire.
+2. **v1b vs v1c** : même budget, même code de train/éval, même n=4000 → comparables entre eux ; mais taille quasi identique (+2,3 %) : l'écart −7,2 pt est un effet de **composition**, pas de taille.
+3. **Famille broad** : autre script d'éval (eval_bench_v4 / v4.x de Checker, n=8000/20000), test20k masqué (None) pour broad14/bg1 → 4 fenêtres, bras mélangés (plain vs cm), calibration différente → **non comparable** à v1b/v1c en niveau.
+4. **bg1_m8*** : seen = tranche SNN 32 stratifiée, pas la fenêtre mémorisée → l'écart 1,3 n'a pas le même sens ; exclu de la pente.
+5. Réserve test20k : version « old » ce0526c6 non revérifiée par fichier.
+
+## Pente
+Seuls points vraiment comparables : v1b/v1c → 2 points à la même taille, **pente non définie**. Indicatif, famille broad (old_ref, m8_broad14, m8_v3 ; 3 points, bras mélangés) : −8,9 pt par décade de taille, IC 95 % [−85,7 ; +67,9] (ddl = 1). Franchement : **pas de pente exploitable**.
+
+## Prédiction v1d (903970c3, 503 053 492 o)
+**Non prédictible.** Le run v1d ne lirait lui aussi qu'un échantillon (B b7404e88) avec le même budget SNN ; la taille n'est donc pas une variable explicative mesurée. v1d = v1c moins 247 656 o de code : le plus proche analogue est v1c (89,2), mais rien dans ces données ne permet d'en tirer un intervalle défendable.

results/hashing/gap_vs_size/SHA256SUMS

@@ -0,0 +1,3 @@
+210fd2b49a369c9d9ab3cb1f4bd94f09f9d25603a7b3a9b8e3269c8882210c0f  gap_vs_size.py
+5e5d0c9f1348f65005086d9f7538db3a4c242f383acc362365a3c22c520548cd  results.json
+87a562090deb3f28d237af3b7d13bd43776ca4cc18c0c5bcb1d09277e67944c5  REPORT.md

results/hashing/gap_vs_size/gap_vs_size.py

@@ -0,0 +1,42 @@
+"""Diagnostic léger écart vu/frais vs taille corpus — lecture de résultats existants uniquement."""
+import json, hashlib, os, resource, math
+S = "/workspace/spira_shared"
+FRESH = ["fc2", "fresh_en", "fresh_fr", "test20k", "test_v3"]  # neither_code exclu
+def sha(p, cache={}):
+    h = hashlib.sha256()
+    with open(p, "rb") as f:
+        for b in iter(lambda: f.read(1 << 20), b""): h.update(b)
+    return h.hexdigest()
+def fsha(p): return hashlib.sha256(open(p, "rb").read()).hexdigest()
+rows = []
+# famille M8 eval intégrée au trainer (n=4000, fused_invocab_pct)
+for name, f in [("v1b ctl", "hashing_gen/m8_v1b/runs/results.jsonl"), ("v1c ctl", "spira_m8/v1c_ctl_data/runs/results.jsonl")]:
+    L = {x["window"]: x for x in map(json.loads, open(f"{S}/{f}"))}
+    s = L["seen"]; fr = [L[w]["fused_invocab_pct"] for w in FRESH]
+    rows.append(dict(run=name, famille="M8 trainer-eval n=4000", src=f, src_sha=fsha(f"{S}/{f}"),
+        train_bytes=s["read"]["data_bytes"], train_sha=s["data_sha256"], snn_bytes=s["read"]["snn_bytes_read"],
+        snn_ticks=s["snn_tokens"], am_ticks=s["am_tokens"], epochs=s["epochs"], n_state=s["n_state"],
+        seen=s["fused_invocab_pct"], fresh=dict(zip(FRESH, fr)), n_fresh=5, fresh_mean=sum(fr) / 5))
+# famille broad (fermée) — eval_bench_v4 et variante de Checker, n=8000/20000
+f = "checker/scale/results_v4.jsonl"; L = [json.loads(x) for x in open(f"{S}/{f}") if x.strip()]
+corp = {"old_local_v1": "broad/old_local_v1/train.bin", "broad_local_v1": "broad/broad_local_v1/train.bin",
+        "corpus_v3": "corpus_v3/train.bin", "broad_gutenberg_v1": "broad/broad_gutenberg_v1/train.bin"}
+snn_known = {"bg1_m8": 2211450}  # checker/scale/cm_bg1d_m8.cov.json snn.bytes_covered
+for tag in ["old_ref", "broad14_ref", "m8_broad14", "m8_v3", "bg1_m8"]:
+    W = {x["window"]: x for x in L if x["tag"] == tag}; x0 = W["seen"]; p = f"{S}/{corp[x0['corpus']]}"
+    fr = {w: W[w]["fused_invocab_pct"] for w in FRESH}; ok = [v for v in fr.values() if v is not None]
+    rows.append(dict(run=tag, famille=f"broad {x0['script'].replace('.', '_')} kind={x0['kind']}", src=f, src_sha=fsha(f"{S}/{f}"),
+        train_bytes=os.path.getsize(p), train_sha=sha(p), snn_bytes=snn_known.get(tag), seen=x0["fused_invocab_pct"],
+        n_seen=x0["n_positions"], fresh=fr, n_fresh=len(ok), fresh_mean=sum(ok) / len(ok)))
+for r in rows: r["gap"] = r["seen"] - r["fresh_mean"]
+# pente descriptive : points broad eval_bench, seen = fenêtre mémorisée (bg1 exclu : seen = tranche SNN 32)
+pts = [(math.log10(r["train_bytes"]), r["gap"]) for r in rows if r["run"] in ("old_ref", "m8_broad14", "m8_v3")]
+n = len(pts); mx = sum(a for a, _ in pts) / n; my = sum(b for _, b in pts) / n
+sxx = sum((a - mx) ** 2 for a, _ in pts); b = sum((a - mx) * (c - my) for a, c in pts) / sxx
+res = sum((c - my - b * (a - mx)) ** 2 for a, c in pts); se = math.sqrt(res / (n - 2) / sxx); t = 12.706  # t_0.975, ddl 1
+slope = dict(points=["old_ref", "m8_broad14", "m8_v3"], x="log10(octets train)", n=n, slope_pt_per_decade=b, se=se,
+             ci95=[b - t * se, b + t * se], note="3 points, ddl=1, bras mélangés (plain/cm), indicatif seulement")
+out = dict(rows=rows, slope=slope, v1d="non prédictible", peak_rss_kB=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
+json.dump(out, open("results.json", "w"), indent=1, ensure_ascii=False)
+for r in rows: print(f"{r['run']:12} {r['train_bytes']:>11} {r['train_sha'][:8]} snn={r['snn_bytes']} seen={r['seen']:.3f} fresh={r['fresh_mean']:.3f} (n={r['n_fresh']}) gap={r['gap']:.3f}")
+print(slope, out["peak_rss_kB"])

results/hashing/gap_vs_size/results.json

@@ -0,0 +1,175 @@
+{
+ "rows": [
+  {
+   "run": "v1b ctl",
+   "famille": "M8 trainer-eval n=4000",
+   "src": "hashing_gen/m8_v1b/runs/results.jsonl",
+   "src_sha": "5a7228343cf792b292d9b048f2f4df1867129d6deba8473bb79462f48ec732f8",
+   "train_bytes": 35128951,
+   "train_sha": "9ea562e44f708adda470c53144dcae748796d77b98039b5fc52ac962d877370b",
+   "snn_bytes": 2791343,
+   "snn_ticks": 600000,
+   "am_ticks": 1200000,
+   "epochs": 2,
+   "n_state": 2560,
+   "seen": 98.44,
+   "fresh": {
+    "fc2": 4.684,
+    "fresh_en": 1.606,
+    "fresh_fr": 1.638,
+    "test20k": 1.256,
+    "test_v3": 0.875
+   },
+   "n_fresh": 5,
+   "fresh_mean": 2.0118,
+   "gap": 96.4282
+  },
+  {
+   "run": "v1c ctl",
+   "famille": "M8 trainer-eval n=4000",
+   "src": "spira_m8/v1c_ctl_data/runs/results.jsonl",
+   "src_sha": "6d6898f5dbdf643622cb03ea82ec60a311afa1f0d3b69df0bf0599b1927efb38",
+   "train_bytes": 35948296,
+   "train_sha": "3d8b6181cbffb27e360758267e1084197dccad82ae05b868691758e1cf72eed4",
+   "snn_bytes": 2706257,
+   "snn_ticks": 600000,
+   "am_ticks": 1200000,
+   "epochs": 2,
+   "n_state": 2560,
+   "seen": 93.183,
+   "fresh": {
+    "fc2": 7.852,
+    "fresh_en": 3.415,
+    "fresh_fr": 2.654,
+    "test20k": 3.343,
+    "test_v3": 2.731
+   },
+   "n_fresh": 5,
+   "fresh_mean": 3.999,
+   "gap": 89.18400000000001
+  },
+  {
+   "run": "old_ref",
+   "famille": "broad eval_bench_v4 kind=plain",
+   "src": "checker/scale/results_v4.jsonl",
+   "src_sha": "57a6c85bf9befc7bd9ab5053c0fdb67c63d6c5e23e6af168c7c41b220bcebac8",
+   "train_bytes": 12468350,
+   "train_sha": "e12b658625da3b7bacf84c68f82a7dbc53e196ccfdda0c7fc1d4ad46285ded54",
+   "snn_bytes": null,
+   "seen": 98.33,
+   "n_seen": 8000,
+   "fresh": {
+    "fc2": 11.504,
+    "fresh_en": 3.395,
+    "fresh_fr": 0.283,
+    "test20k": 2.9,
+    "test_v3": 4.117
+   },
+   "n_fresh": 5,
+   "fresh_mean": 4.4398,
+   "gap": 93.8902
+  },
+  {
+   "run": "broad14_ref",
+   "famille": "broad eval_bench_v4 kind=plain",
+   "src": "checker/scale/results_v4.jsonl",
+   "src_sha": "57a6c85bf9befc7bd9ab5053c0fdb67c63d6c5e23e6af168c7c41b220bcebac8",
+   "train_bytes": 34835257,
+   "train_sha": "eda07017a87f2b7bbd55c5c2ea23032bf70a38ae7ad75167523228738f2391a1",
+   "snn_bytes": null,
+   "seen": 91.934,
+   "n_seen": 8000,
+   "fresh": {
+    "fc2": 8.456,
+    "fresh_en": 5.167,
+    "fresh_fr": 2.717,
+    "test20k": null,
+    "test_v3": 2.804
+   },
+   "n_fresh": 4,
+   "fresh_mean": 4.786,
+   "gap": 87.148
+  },
+  {
+   "run": "m8_broad14",
+   "famille": "broad eval_bench_v4_1 kind=cm",
+   "src": "checker/scale/results_v4.jsonl",
+   "src_sha": "57a6c85bf9befc7bd9ab5053c0fdb67c63d6c5e23e6af168c7c41b220bcebac8",
+   "train_bytes": 34835257,
+   "train_sha": "eda07017a87f2b7bbd55c5c2ea23032bf70a38ae7ad75167523228738f2391a1",
+   "snn_bytes": null,
+   "seen": 91.819,
+   "n_seen": 8000,
+   "fresh": {
+    "fc2": 14.315,
+    "fresh_en": 5.105,
+    "fresh_fr": 3.054,
+    "test20k": null,
+    "test_v3": 3.3
+   },
+   "n_fresh": 4,
+   "fresh_mean": 6.4435,
+   "gap": 85.3755
+  },
+  {
+   "run": "m8_v3",
+   "famille": "broad eval_bench_v4_1 kind=cm",
+   "src": "checker/scale/results_v4.jsonl",
+   "src_sha": "57a6c85bf9befc7bd9ab5053c0fdb67c63d6c5e23e6af168c7c41b220bcebac8",
+   "train_bytes": 93284487,
+   "train_sha": "aabd912d1bce05d84122039e4f7e059f7bbb6bd6b448367b91bbf1e9fa75249c",
+   "snn_bytes": null,
+   "seen": 91.945,
+   "n_seen": 8000,
+   "fresh": {
+    "fc2": 14.451,
+    "fresh_en": 5.546,
+    "fresh_fr": 2.035,
+    "test20k": 3.555,
+    "test_v3": 3.276
+   },
+   "n_fresh": 5,
+   "fresh_mean": 5.7726,
+   "gap": 86.1724
+  },
+  {
+   "run": "bg1_m8",
+   "famille": "broad eval_bench_v4_1 kind=cm",
+   "src": "checker/scale/results_v4.jsonl",
+   "src_sha": "57a6c85bf9befc7bd9ab5053c0fdb67c63d6c5e23e6af168c7c41b220bcebac8",
+   "train_bytes": 290810502,
+   "train_sha": "5af6398fc853bce270e430f8336fc2b136318c2e762ead19e92f5d180de083f2",
+   "snn_bytes": 2211450,
+   "seen": 8.99,
+   "n_seen": 8000,
+   "fresh": {
+    "fc2": 13.507,
+    "fresh_en": 4.287,
+    "fresh_fr": 8.778,
+    "test20k": null,
+    "test_v3": 4.283
+   },
+   "n_fresh": 4,
+   "fresh_mean": 7.71375,
+   "gap": 1.27625
+  }
+ ],
+ "slope": {
+  "points": [
+   "old_ref",
+   "m8_broad14",
+   "m8_v3"
+  ],
+  "x": "log10(octets train)",
+  "n": 3,
+  "slope_pt_per_decade": -8.90395666592213,
+  "se": 6.042748616229554,
+  "ci95": [
+   -85.68312058373485,
+   67.87520725189057
+  ],
+  "note": "3 points, ddl=1, bras mélangés (plain/cm), indicatif seulement"
+ },
+ "v1d": "non prédictible",
+ "peak_rss_kB": 17972
+}

results/hashing/gen_copy_recount/code/copy_recount.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python3
+"""SPIRA Hashing — recomptage INDÉPENDANT de la copie du train (claim Checker 25 oct. 00 h 06). Lecture seule.
+Code écrit de zéro (n'importe rien de checker/). Pour chaque requête q (génération, ou contrôle held-out) :
+  m(i) = longueur du plus long préfixe de q[i:] présent dans le train (exact, si >= 16 ; sinon 0).
+  Méthode : fenêtres de 16 o de toutes les requêtes -> hash polynomial 64 bits (numpy) -> tableau trié ; train lu en
+  blocs de 2 Mio + chevauchement 256 o (> 200 = longueur max de requête + 16) ; chaque hit est VÉRIFIÉ par comparaison
+  octet à octet (LCP vectorisé), donc aucune fausse détection par collision.
+  cov>=k = part des octets de q couverts par un intervalle [i, i+m(i)) avec m(i) >= k (= union des k-fenêtres présentes).
+Générations : checker/gen/samples/<tag>__<dec>__<p>.txt, partie après '\\n\\nGÉNÉRÉ:\\n', lue newline='' (octets exacts),
+  U+FFFD = octets perdus à l'écriture (decode 'replace') -> coupure ; dénominateurs : 'known' (octets connus, comme Checker)
+  et '200' (n_bytes publié, octets perdus comptés non couverts).
+Contrôles held-out : (a) les 7 vraies continuations (cont_hex de prompts_v1.json) ; (b) 100 tranches de 200 o tirées
+  (seed 0) dans test.bin du corpus du modèle. Aucune donnée d'éval écrite en sortie (comptes et offsets seulement)."""
+import os, sys, json, hashlib, time, resource
+import numpy as np
+S = "/workspace/spira_shared"; G = f"{S}/checker/gen"; OUTD = f"{S}/hashing_gen/gen_copy_recount/out"
+MODELS = {"old_ref": ("old_local_v1", f"{S}/broad/old_local_v1"), "bg1_m8": ("broad_gutenberg_v1", f"{S}/broad/broad_gutenberg_v1")}
+W, CH, OV, MAXQ = 16, 2 << 20, 256, 200
+RSS_CAP_KB = 280 * 1024
+BASE = np.uint64(1099511628211)
+POW = [np.uint64(1)]
+for _ in range(W - 1): POW.append(POW[-1] * BASE)
+POW = POW[::-1]  # coefficient de b[t] = BASE^(W-1-t)
+
+def rss_kb():
+    for l in open("/proc/self/status"):
+        if l.startswith("VmRSS:"): return int(l.split()[1])
+def guard():
+    r = rss_kb()
+    if r > RSS_CAP_KB: print("ABANDON RSS", r, "kB", flush=True); sys.exit(3)
+    return r
+def fsha(p):
+    h = hashlib.sha256()
+    with open(p, "rb") as f:
+        for b in iter(lambda: f.read(4 << 20), b""): h.update(b)
+    return h.hexdigest()
+def whash(a):  # a: uint8 array ; renvoie hash des fenêtres a[j:j+W], j = 0..len-W
+    n = len(a) - W + 1
+    if n <= 0: return np.zeros(0, np.uint64)
+    h = np.zeros(n, np.uint64)
+    with np.errstate(over="ignore"):
+        for t in range(W): h += a[t:t + n].astype(np.uint64) * POW[t]
+    return h
+
+def load_queries(tag, cname, cdir):
+    Q = []  # dict(kind, key, segs[list[bytes]], n_fffd, n_bytes_pub)
+    rows = [json.loads(l) for l in open(f"{G}/results_gen_v1.jsonl")]
+    for r in rows:
+        if r["tag"] != tag: continue
+        assert r["corpus"] == cname, (r["corpus"], cname)
+        fn = f"{G}/samples/{tag}__{r['decoding']}__{r['prompt']}.txt"
+        s = open(fn, encoding="utf-8", errors="strict", newline="").read()
+        parts = s.split("\n\nGÉNÉRÉ:\n")
+        assert len(parts) == 2 and s.endswith("\n"), fn
+        gt = parts[1][:-1]
+        Q.append({"kind": "gen_" + r["decoding"], "key": r["prompt"], "segs": [x.encode("utf-8") for x in gt.split("\ufffd")],
+                  "n_fffd": gt.count("\ufffd"), "n_pub": r["n_bytes"], "pub_longest": r["longest_copy"],
+                  "file_sha256": hashlib.sha256(open(fn, "rb").read()).hexdigest()})
+    for p in json.load(open(f"{G}/prompts_v1.json")):
+        cb = bytes.fromhex(p["cont_hex"])[:MAXQ]
+        Q.append({"kind": "heldout_cont", "key": p["name"], "segs": [cb], "n_fffd": 0, "n_pub": len(cb)})
+    tb = f"{cdir}/test.bin"; n = os.path.getsize(tb); rng = np.random.default_rng(0)
+    offs = sorted(int(x) for x in rng.integers(0, n - MAXQ, size=100))
+    with open(tb, "rb") as f:
+        for o in offs:
+            f.seek(o); Q.append({"kind": "heldout_test", "key": f"off{o}", "segs": [f.read(MAXQ)], "n_fffd": 0, "n_pub": MAXQ})
+    return Q
+
+def scan(train_path, Q):
+    # tables des requêtes : concat de tous les segments, séparés (pas de fenêtre à cheval)
+    segs = [(qi, si, sb) for qi, q in enumerate(Q) for si, sb in enumerate(q["segs"])]
+    starts = []; hs = []; seglen = []
+    for k, (qi, si, sb) in enumerate(segs):
+        a = np.frombuffer(sb, np.uint8); h = whash(a)
+        hs.append(h); starts.append(np.stack([np.full(len(h), k), np.arange(len(h))], 1) if len(h) else np.zeros((0, 2), int))
+        seglen.append(len(sb))
+    H = np.concatenate(hs) if hs else np.zeros(0, np.uint64); P = np.concatenate(starts).astype(np.int64)
+    order = np.argsort(H, kind="stable"); H = H[order]; P = P[order]
+    Uh, ustart = np.unique(H, return_index=True); uend = np.append(ustart[1:], len(H))
+    # requête paddée (-1) en int16 pour LCP
+    QB = np.full((len(segs), MAXQ + 1), -1, np.int16)
+    for k, (_, _, sb) in enumerate(segs): QB[k, :len(sb)] = np.frombuffer(sb, np.uint8)
+    M = np.zeros((len(segs), MAXQ), np.int32); MOFF = np.full((len(segs), MAXQ), -1, np.int64)
+    nhits = 0; peak = 0; t0 = time.time(); nver = [0, 0]
+    with open(train_path, "rb") as f:
+        pos = 0; tail = b""
+        while True:
+            blk = f.read(CH); last = not blk or len(blk) < CH
+            if not blk and not tail: break
+            buf = tail + blk; base = pos - len(tail)
+            a = np.frombuffer(buf, np.uint8)
+            h = whash(a)
+            lim = len(h) if last else max(0, len(a) - OV)  # débuts traités dans ce bloc
+            h = h[:lim]
+            idx = np.searchsorted(Uh, h); idx[idx >= len(Uh)] = 0
+            hit = np.nonzero(Uh[idx] == h)[0]
+            del h
+            if len(hit):
+                ap = np.append(a.astype(np.int16), np.full(MAXQ + 1, -2, np.int16)); d = np.arange(MAXQ + 1)
+                SL = np.asarray(seglen, np.int64)
+                for h0 in range(0, len(hit), 4000):
+                    hh = hit[h0:h0 + 4000]; u = idx[hh]; cnt = uend[u] - ustart[u]
+                    tp = np.repeat(hh, cnt)
+                    qrow = np.repeat(ustart[u], cnt) + (np.arange(int(cnt.sum())) - np.repeat(np.cumsum(cnt) - cnt, cnt))
+                    sk = P[qrow, 0]; si = P[qrow, 1]
+                    keep = M[sk, si] < (SL[sk] - si)  # position déjà saturée : inutile
+                    tp, sk, si = tp[keep], sk[keep], si[keep]; nhits += len(tp)
+                    for b0 in range(0, len(tp), 8000):
+                        t_ = tp[b0:b0 + 8000]; k_ = sk[b0:b0 + 8000]; i_ = si[b0:b0 + 8000]
+                        tb_ = ap[t_[:, None] + d[None, :]]
+                        qb_ = QB[k_[:, None], np.minimum(i_[:, None] + d[None, :], MAXQ)]
+                        lcp = np.argmin((tb_ == qb_) & (qb_ >= 0), axis=1).astype(np.int32)
+                        del tb_, qb_
+                        ok = lcp >= W; nver[0] += int(len(lcp)); nver[1] += int((~ok).sum())
+                        lcp, key, off = lcp[ok], (k_ * MAXQ + i_)[ok], (base + t_)[ok]
+                        o = np.argsort(lcp, kind="stable"); lcp, key, off = lcp[o], key[o], off[o]
+                        better = lcp > M.reshape(-1)[key]
+                        M.reshape(-1)[key[better]] = lcp[better]; MOFF.reshape(-1)[key[better]] = off[better]
+                del ap
+            peak = max(peak, guard())
+            pos += len(blk); tail = buf[-OV:] if not last else b""
+            if last: break
+            if (pos // CH) % 16 == 0: print(f"  {pos/1e6:.0f} Mo, {time.time()-t0:.0f} s, hits {nhits}, rss {rss_kb()} kB", flush=True)
+    print('  vérifiés', nver[0], 'collisions rejetées', nver[1], flush=True)
+    return segs, seglen, M, MOFF, nhits, peak, pos
+
+def summarize(Q, segs, seglen, M, MOFF):
+    for q in Q: q.update({"cov": {16: 0, 32: 0, 64: 0}, "longest": 0, "longest_train_off": -1, "known": 0})
+    for k, (qi, si, sb) in enumerate(segs):
+        q = Q[qi]; n = seglen[k]; q["known"] += n
+        for thr in (16, 32, 64):
+            c = np.zeros(n, bool)
+            for i in range(n):
+                L = int(M[k, i])
+                if L >= thr: c[i:i + L] = True
+            q["cov"][thr] += int(c.sum())
+        j = int(np.argmax(M[k, :n])) if n else 0
+        if n and M[k, j] > q["longest"]: q["longest"] = int(M[k, j]); q["longest_train_off"] = int(MOFF[k, j])
+    for q in Q:
+        q["cov_known"] = {t: q["cov"][t] / max(q["known"], 1) for t in q["cov"]}
+        q["cov_200"] = {t: q["cov"][t] / q["n_pub"] for t in q["cov"]}
+
+def boot(x, B=10000, seed=0):
+    x = np.asarray(x, float); rng = np.random.default_rng(seed)
+    mm = x[rng.integers(0, len(x), (B, len(x)))].mean(1)
+    return [float(x.mean()), float(np.percentile(mm, 2.5)), float(np.percentile(mm, 97.5))]
+
+def brute_check(train_path, Q, segs, M):  # contrôle croisé exact (old_ref seulement : train entier en RAM, 12 Mo)
+    T = open(train_path, "rb").read(); bad = 0; n = 0
+    for k, (qi, si, sb) in enumerate(segs):
+        if not Q[qi]["kind"].startswith("gen"): continue
+        for i in range(len(sb)):
+            lo = 0
+            while i + lo < len(sb) and sb[i:i + lo + 1] in T: lo += 1
+            want = lo if lo >= W else 0; n += 1; bad += int(want != M[k, i])
+    del T; return {"positions": n, "mismatch": bad}
+
+if __name__ == "__main__":
+    t00 = time.time(); res = {"script_sha256": hashlib.sha256(open(__file__, "rb").read()).hexdigest(), "models": {}}
+    for tag, (cname, cdir) in MODELS.items():
+        tp = f"{cdir}/train.bin"; man = None
+        Q = load_queries(tag, cname, cdir)
+        print(tag, "requêtes", len(Q), flush=True)
+        segs, seglen, M, MOFF, nhits, peak, nread = scan(tp, Q)
+        summarize(Q, segs, seglen, M, MOFF)
+        bc = brute_check(tp, Q, segs, M) if tag == "old_ref" else None
+        guard()
+        out = {"corpus": cname, "train": {"path": tp, "bytes": os.path.getsize(tp), "bytes_read": nread, "sha256": fsha(tp)},
+               "test_bin_sha256": fsha(f"{cdir}/test.bin"), "hits16": nhits, "peak_vmrss_kB_scan": peak, "brute_check": bc,
+               "per_query": [{k: v for k, v in q.items() if k != "segs"} for q in Q], "agg": {}}
+        for kind in ("gen_fused", "gen_v2b", "heldout_cont", "heldout_test"):
+            qs = [q for q in Q if q["kind"] == kind]; ag = {"n": len(qs)}
+            for den in ("cov_known", "cov_200"):
+                for t in (16, 32, 64): ag[f"{den}_{t}"] = boot([q[den][t] for q in qs])
+            ag["longest_max"] = max(q["longest"] for q in qs); ag["longest_mean"] = float(np.mean([q["longest"] for q in qs]))
+            p5 = [q for q in qs if q["key"] in ("en1", "en2", "fr1", "fr2", "code")]
+            if kind.startswith("gen"): ag["cov_known_32_P5_mean"] = float(np.mean([q["cov_known"][32] for q in p5]))
+            out["agg"][kind] = ag
+        res["models"][tag] = out
+        print(tag, json.dumps(out["agg"], indent=None)[:3000], "brute", bc, flush=True)
+        del Q, segs, M, MOFF
+    res["elapsed_s"] = round(time.time() - t00, 1); res["ru_maxrss_kB_self"] = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+    json.dump(res, open(f"{OUTD}/copy_recount.json", "w"), indent=1, ensure_ascii=False)
+    print("FINI", res["elapsed_s"], "s", flush=True)

results/hashing/gen_copy_recount/code/run_wrapped.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+"""Wrapper parent : lance le script en enfant (nice 10, OMP=1, PYTHONHASHSEED=0), mesure le pic RSS de l'enfant via wait4/getrusage."""
+import os, sys, subprocess, json, time
+D = os.path.dirname(os.path.abspath(__file__)); OUT = os.path.join(D, "..", "out")
+env = dict(os.environ, OMP_NUM_THREADS="1", OPENBLAS_NUM_THREADS="1", MKL_NUM_THREADS="1", PYTHONHASHSEED="0", TMPDIR=os.path.abspath(OUT))
+t0 = time.time()
+with open(os.path.join(OUT, "copy_recount.log"), "w") as lg:
+    p = subprocess.Popen(["nice", "-n", "10", sys.executable, os.path.join(D, "copy_recount.py")], stdout=lg, stderr=subprocess.STDOUT, env=env)
+    _, status, ru = os.wait4(p.pid, 0)
+r = {"exit_status": os.waitstatus_to_exitcode(status), "child_peak_rss_kB_getrusage": ru.ru_maxrss, "elapsed_s": round(time.time() - t0, 1)}
+json.dump(r, open(os.path.join(OUT, "copy_recount.mem.json"), "w")); print(r)

results/hashing/gen_copy_recount/relay_entry.md

@@ -0,0 +1,40 @@
+
+## 25 oct 00 h 49 (Toronto) — SPIRA Hashing — recomptage indépendant de la copie du train (claim Checker 00 h 06 + erratum 00 h 45)
+
+Code écrit de zéro, rien importé de checker/. Lecture seule des samples checker/gen et des train.bin. Fichiers dans hashing_gen/gen_copy_recount/ : code/copy_recount.py (sha256 f4e533e3…dfb29), code/run_wrapped.py (9498c36b…80d6), out/copy_recount.json (3cf49b9b…4466), out/compare_checker.txt (c697f9f0…36c5).
+
+Conditions : nice 10, OMP=1, PYTHONHASHSEED=0, un seul process. **Pic RSS de l'enfant = 154,6 Mo** (getrusage, mesuré par le wrapper parent). Durée 104 s. cm 925745 non touché. Rien sous /tmp, aucun push HF.
+
+Corpus vérifiés (jobs_v4.tsv, colonne corpus du jsonl, meta.train_bytes des ckpt) :
+- old_ref → old_local_v1/train.bin, 12 468 350 o (sha e12b6586…) ;
+- bg1_m8 → broad_gutenberg_v1/train.bin, 290 810 502 o (sha 5af6398f…).
+
+Méthode : on hache les fenêtres de 16 o, puis on vérifie chaque hit octet à octet (24,4 M et 11,9 M vérifications, 0 collision). Train lu par blocs de 2 Mio, avec 256 o de chevauchement. Lecture newline="" ; U+FFFD traité comme une coupure. Un contrôle brute-force sur old_ref donne 2 796/2 796 positions identiques.
+
+v2b, 7 générations, moyenne [IC95 bootstrap, 10 000 tirages, seed 0] :
+- old_ref : cov≥32 = 83,6 % [68,9 ; 94,9] ; cov≥16 = 93,5 % ; cov≥64 = 59,7 % ; plus longue copie 196 o ; P5 = 88,5 %.
+- bg1_m8 : cov≥32 = 83,6 % [77,1 ; 89,6] ; cov≥16 = 90,6 % ; cov≥64 = 53,7 % ; plus longue copie 182 o ; P5 = 88,1 %.
+
+Fused :
+- old_ref : cov≥32 = 27,3 % [8,8 ; 47,8], max 123 o ;
+- bg1_m8 : cov≥32 = 56,5 % [26,5 ; 83,2], max 189 o.
+
+Témoin held-out (comptes seulement) :
+- 7 vraies continuations : cov≥32 = 2,3 % [0 ; 6,9], max 32 o (les deux modèles).
+- 100 tranches de 200 o de test.bin (seed 0) : old 4,7 % [2,5 ; 7,2], max 49 o ; bg1 9,0 % [4,4 ; 14,2], max 200 o. 3 tranches sur 100 sont entières dans le train : test.bin de broad_gutenberg_v1 n'est pas disjoint de son train.
+
+Comparaison avec Checker, par génération (28) :
+- longest_copy, cov≥32, octets connus et offset : **IDENTIQUES** sur 25/28.
+- old_ref v2b/fr1 : cov 87,9 % contre 68,0 % au relais de 00 h 06. C'est **DIFF avec 00 h 06** et **IDENTIQUE à l'erratum cr_fix 00 h 45**.
+- bg1_m8 fused en1 et fr1 : Checker donne 12 et 15 o ; ma méthode a un plancher à 16 o, donc « < 16 ». Pas de contradiction, mais non vérifiable par ma méthode.
+- Moyennes : old_ref v2b 83,6 / 88,5 (**DIFF** avec 80,7 / 84,5 de 00 h 06, **IDENTIQUE** à l'erratum) ; bg1_m8 v2b 83,6 / 88,1 **IDENTIQUE** ; fused 27,3 / 56,5 **IDENTIQUE**.
+- Plus longue copie 196 o : **IDENTIQUE**.
+
+Verdict : la copie du train en v2b est **CONFIRMÉE**. Le témoin held-out est à 2–9 %, contre environ 84 % pour v2b.
+
+Réserves :
+- n = 7 par modèle, donc des IC larges.
+- old_ref v2b/fr1 donne 88 contre 89 publié : l'octet perdu au U+FFFD ne peut pas être reconstruit depuis les samples.
+- La couverture est mesurée contre le fichier train entier, pas contre la seule tranche vue par le SNN (le n-gramme v2b est compté sur le train).
+
+Lu/appliqué — SPIRA Hashing

results/hashing/gen_kit_check_r1/REPORT.md

@@ -0,0 +1,33 @@
+# gen_kit_check_r1 — kit de test de génération : figé et caviardé (contrôle statique, test NON lancé)
+Aucun texte de prompt n'est cité ici, ni dans gen_kit_check_result.json.
+
+## Figé : PASS
+| objet | sha attendu | sha mesuré |
+|---|---|---|
+| PREDECL_gen_test_runC_r1.md | e451dfc7 | e451dfc7 |
+| gen_kit_r2_result.json | 026dc89f | 026dc89f (gen_test_kit_r2/SHA256SUMS -c OK) |
+| gen_prompts.py | 57fed6cb | 57fed6cb |
+| gen_prompts_manifest.json | 70acee7b | 70acee7b |
+| NEVER_PUSH/prompts_and_cont.bin | ca47ea80 | ca47ea80 |
+
+- Les 5 prompts et les 5 continuations de 200 o, relus via le manifeste depuis les fenêtres, ont tous le bon sha (prompts faf784ed, 54cdebf1, 90d5ab55, 01f41694, b97f1cf2), et les sha des fenêtres sont OK.
+- gen_test/SHA256SUMS (23 lignes) : -c OK.
+
+## Caviardé : PASS sur le périmètre demandé
+- Méthode : gen_kit_check_v2.py, PID 1171344, nice 19, 434,3 s, maxrss 39,9 Mo.
+- Périmètre : 55 386 fichiers texte de spira_shared (dont spira_relay.md, tous les .md, .json et .log), tout hf_ready (8 dossiers) et /workspace/SPIRA/web/public.
+- Pour chaque fichier : recherche des 200 o exacts et des 169 sous-chaînes de 32 o, pour chaque prompt et chaque continuation.
+- Hors périmètre : NEVER_PUSH*, broad/, binaires .bin/.npz/.npy, venv.
+- Relais, rapports, hf_ready, web/public : **0 hit**.
+- Les 3 fichiers de plus de 20 Mo ont été rescannés à part : 0 hit. Ce sont spira_deploy.json (public), sha_manifest_corpora_FULL_LOCAL.json et pg_catalog.csv.
+- 19 hits, tous dans des corpus bruts sources (corpus_v3/raw, corpus_gutenberg, corpus_gutenberg_v2/raw) ; c'est attendu, car ce sont les textes d'origine des fenêtres :
+  - test_v3 : 200 o exacts dans corpus_v3/raw/pg1184.txt, prompt et continuation ;
+  - fresh_en : 107/169 et 96/169 dans pg5200.txt et metamorphosis.txt ;
+  - test20k_old : 107/169 et 97/169 dans frankenstein.txt et pg84.txt ;
+  - quelques sous-chaînes isolées (1 à 7) dans d'autres livres.
+- **Fait nouveau** : fresh_en recoupe un texte Gutenberg ancien (Metamorphosis). « fresh » ne veut donc pas dire « postérieur ».
+
+## Erreurs de ma part
+- La 1re version, gen_kit_check.py (PID 1163831), avait un périmètre trop large (298 497 fichiers) ; je l'ai tuée après plus de 10 min, sans résultat, et j'ai supprimé ma copie, donc il n'y a pas de sha.
+- Un 1er lancement de la v2 a échoué (SyntaxError) avant tout calcul.
+- py_compile a créé un __pycache__ dans mon dossier ; je l'ai supprimé.

results/hashing/gen_kit_check_r1/SHA256SUMS

@@ -0,0 +1,4 @@
+e0ebfd9292c4191d0957704da14db58e7ab7b3514a54c837005e9c2809d8014c  gen_kit_check_v2.py
+017d085a748e1e984c0490172abd813203906a6647b9c3396373b47d1549aa48  gen_kit_check_result.json
+d3cc43bf4e16b379cb341f655316b6eb9a728ca44d6070dbbdcc03e5e2ca1499  run.log
+e65221b52abfed9b13b56ac3a8498f52fc0e5ecaca0c93b155c2baedc231ee1a  REPORT.md

results/hashing/gen_kit_check_r1/SHA256SUMS_v3

@@ -0,0 +1,7 @@
+e0ebfd9292c4191d0957704da14db58e7ab7b3514a54c837005e9c2809d8014c  gen_kit_check_v2.py
+e01d2f2a2110a2d8803cf8f74cb5d978bc63b7a9fd602ec20177b1e3a4fda42b  gen_kit_check_v3.py
+79be901c5fa010885b4d2f88a357cf16e89f1f3d4b1b88b7612423feb70cec2d  gen_kit_check_v3b.py
+da94f7ae45b171d7a366fa5bf2154b62ebd58fd53576201966a9c11427750a2c  gen_kit_check_v3_result.json
+c1b083d49a82ffbf880a81f945ab88a9d71896b5a140071007aaa61a51286f4d  run_v3.log
+898a0d9be827c190615c82241f6e5550875bd8293ef61597b74b5a87a5fb7327  selftest_v3.sh
+bc6a8f1e930501b240e1f7f5a5156deb54d678893bacb4d96c7e879a5e2921d4  selftest_v3_result.txt

results/hashing/gen_kit_check_r1/SHA256SUMS_v3b_20261031

@@ -0,0 +1,7 @@
+e0ebfd9292c4191d0957704da14db58e7ab7b3514a54c837005e9c2809d8014c  gen_kit_check_v2.py
+e01d2f2a2110a2d8803cf8f74cb5d978bc63b7a9fd602ec20177b1e3a4fda42b  gen_kit_check_v3.py
+79be901c5fa010885b4d2f88a357cf16e89f1f3d4b1b88b7612423feb70cec2d  gen_kit_check_v3b.py
+da94f7ae45b171d7a366fa5bf2154b62ebd58fd53576201966a9c11427750a2c  gen_kit_check_v3_result.json
+c1b083d49a82ffbf880a81f945ab88a9d71896b5a140071007aaa61a51286f4d  run_v3.log
+898a0d9be827c190615c82241f6e5550875bd8293ef61597b74b5a87a5fb7327  selftest_v3.sh
+bc6a8f1e930501b240e1f7f5a5156deb54d678893bacb4d96c7e879a5e2921d4  selftest_v3_result.txt

results/hashing/gen_kit_check_r1/gen_kit_check_result.json

@@ -0,0 +1,200 @@
+{
+ "sha": {
+  "gen_prompts.py": "57fed6cb540f5fb1c91f23a62d96a5c9c9587030e60bf9b4685eede5438ac029",
+  "gen_prompts_manifest.json": "70acee7babde01f77e53d3691c40d4cafae3e55c3d7043e1966de36a9320c2d9",
+  "NEVER_PUSH/prompts_and_cont.bin": "ca47ea802b09312c1419563a28fe1e2ccab13df1511bf339c8008183dce47902",
+  "checker/recount/runC_ready/PREDECL_gen_test_runC_r1.md": "e451dfc7978bd92ada40b45e7c4efc94776555722e2b42a47b74fb1db5dfa66a",
+  "checker/recount/runC_ready/gen_test_kit_r2/gen_kit_r2_result.json": "026dc89f20c58833790ffca86b74572f12f933e7cb2145f703b12649e99aeb9d"
+ },
+ "items": [
+  {
+   "window": "fc2",
+   "window_sha_ok": true,
+   "prompt_sha_ok": true,
+   "cont_sha_ok": true,
+   "prompt_sha": "faf784ed0e85331ff8db7cb0caf356543ed1c3e02134a76429677347d4167192"
+  },
+  {
+   "window": "fresh_en",
+   "window_sha_ok": true,
+   "prompt_sha_ok": true,
+   "cont_sha_ok": true,
+   "prompt_sha": "54cdebf119f3f390226922515f3ebce66cc549062caeb2ed3df6ffee7babfa16"
+  },
+  {
+   "window": "fresh_fr",
+   "window_sha_ok": true,
+   "prompt_sha_ok": true,
+   "cont_sha_ok": true,
+   "prompt_sha": "90d5ab55d2729276b0eb6084625a1ef7ab9bb6fe7e282f34f71672798927942f"
+  },
+  {
+   "window": "test20k_old",
+   "window_sha_ok": true,
+   "prompt_sha_ok": true,
+   "cont_sha_ok": true,
+   "prompt_sha": "01f4169450cd9eaf617e1f7c2f5d6f1c07f58f255ac6df1ab35791cd4d8f50ec"
+  },
+  {
+   "window": "test_v3",
+   "window_sha_ok": true,
+   "prompt_sha_ok": true,
+   "cont_sha_ok": true,
+   "prompt_sha": "b97f1cf219ca6cf62798e6c1df03a4dd5ff7bd01c015e9202e66aa9fcbcc99e1"
+  }
+ ],
+ "fichiers_scannes": 55386,
+ "octets_scannes": 947372343,
+ "ignores_gt20Mo": [
+  [
+   "/workspace/spira_shared/checker/recount/diverse_dedup_1509/sha_manifest_corpora_FULL_LOCAL.json",
+   44891889
+  ],
+  [
+   "/workspace/spira_shared/corpus_gutenberg_v2/pg_catalog.csv",
+   21263721
+  ],
+  [
+   "/workspace/SPIRA/web/public/spira_deploy.json",
+   29063956
+  ]
+ ],
+ "exclusions": "NEVER_PUSH*, broad/, .git, __pycache__, binaires .bin .npz .npy .pt .ckpt .safetensors .lock ; seuls les fichiers texte .md .json .txt .log .csv .diff .py .sh .yaml .yml .html, tout hf_ready, tout web/public ; venv/site-packages exclus ; ce script et son r\u00e9sultat ne contiennent aucun texte",
+ "hits": [
+  {
+   "file": "/workspace/spira_shared/corpus_v3/raw/pg1184.txt",
+   "window": "test_v3",
+   "seg": "prompt",
+   "full200": true,
+   "sub32_hits_sur_169": 169
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_v3/raw/pg1184.txt",
+   "window": "test_v3",
+   "seg": "cont",
+   "full200": true,
+   "sub32_hits_sur_169": 169
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_v3/raw/pg5200.txt",
+   "window": "fresh_en",
+   "seg": "prompt",
+   "full200": false,
+   "sub32_hits_sur_169": 107
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_v3/raw/pg5200.txt",
+   "window": "fresh_en",
+   "seg": "cont",
+   "full200": false,
+   "sub32_hits_sur_169": 96
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg/corpus/books/frankenstein.txt",
+   "window": "test20k_old",
+   "seg": "prompt",
+   "full200": false,
+   "sub32_hits_sur_169": 107
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg/corpus/books/frankenstein.txt",
+   "window": "test20k_old",
+   "seg": "cont",
+   "full200": false,
+   "sub32_hits_sur_169": 97
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg/corpus/books/metamorphosis.txt",
+   "window": "fresh_en",
+   "seg": "prompt",
+   "full200": false,
+   "sub32_hits_sur_169": 107
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg/corpus/books/metamorphosis.txt",
+   "window": "fresh_en",
+   "seg": "cont",
+   "full200": false,
+   "sub32_hits_sur_169": 96
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg_v2/raw/pg83.txt",
+   "window": "test_v3",
+   "seg": "prompt",
+   "full200": false,
+   "sub32_hits_sur_169": 2
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg_v2/raw/pg203.txt",
+   "window": "test_v3",
+   "seg": "prompt",
+   "full200": false,
+   "sub32_hits_sur_169": 6
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg_v2/raw/pg200.txt",
+   "window": "fresh_en",
+   "seg": "cont",
+   "full200": false,
+   "sub32_hits_sur_169": 3
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg_v2/raw/pg90.txt",
+   "window": "test_v3",
+   "seg": "cont",
+   "full200": false,
+   "sub32_hits_sur_169": 1
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg_v2/raw/pg82.txt",
+   "window": "test_v3",
+   "seg": "prompt",
+   "full200": false,
+   "sub32_hits_sur_169": 6
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg_v2/raw/pg415.txt",
+   "window": "test_v3",
+   "seg": "prompt",
+   "full200": false,
+   "sub32_hits_sur_169": 2
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg_v2/raw/pg84.txt",
+   "window": "test20k_old",
+   "seg": "prompt",
+   "full200": false,
+   "sub32_hits_sur_169": 107
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg_v2/raw/pg84.txt",
+   "window": "test20k_old",
+   "seg": "cont",
+   "full200": false,
+   "sub32_hits_sur_169": 97
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg_v2/raw/pg308.txt",
+   "window": "test_v3",
+   "seg": "cont",
+   "full200": false,
+   "sub32_hits_sur_169": 2
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg_v2/raw/pg344.txt",
+   "window": "test_v3",
+   "seg": "cont",
+   "full200": false,
+   "sub32_hits_sur_169": 2
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg_v2/raw/pg164.txt",
+   "window": "test_v3",
+   "seg": "cont",
+   "full200": false,
+   "sub32_hits_sur_169": 7
+  }
+ ],
+ "secondes": 434.3,
+ "maxrss_Mo": 39.91015625
+}

results/hashing/gen_kit_check_r1/gen_kit_check_v2.py

@@ -0,0 +1,32 @@
+"""Vérif statique du kit de génération (figé + caviardé). N'affiche ni n'écrit aucun texte de prompt : seulement chemins, sha, compteurs."""
+import os, json, hashlib, time, resource
+T0 = time.perf_counter(); S = "/workspace/spira_shared/"; G = S + "checker/recount/gen_test/"
+sha = lambda b: hashlib.sha256(b).hexdigest()
+man = json.load(open(G + "gen_prompts_manifest.json")); R = {"sha": {}, "items": []}
+for f in ["gen_prompts.py", "gen_prompts_manifest.json", "NEVER_PUSH/prompts_and_cont.bin"]: R["sha"][f] = sha(open(G + f, "rb").read())
+for f in ["checker/recount/runC_ready/PREDECL_gen_test_runC_r1.md", "checker/recount/runC_ready/gen_test_kit_r2/gen_kit_r2_result.json"]: R["sha"][f] = sha(open(S + f, "rb").read())
+P = []
+for it in man["items"]:
+    w = open(S + it["path"], "rb").read(); p = w[it["start"]:it["start"] + 200]; c = w[it["start"] + 200:it["start"] + 400]
+    R["items"].append({"window": it["window"], "window_sha_ok": sha(w) == it["window_sha256"], "prompt_sha_ok": sha(p) == it["prompt_sha256"], "cont_sha_ok": sha(c) == it["cont_sha256"], "prompt_sha": it["prompt_sha256"]})
+    P.append((it["window"], p, c))
+TXT = (".md", ".json", ".txt", ".log", ".csv", ".diff", ".py", ".sh", ".yaml", ".yml", ".html"); EXCL_DIR = ("NEVER_PUSH",); EXCL_SUFFIX = (".bin", ".npz", ".npy", ".pt", ".ckpt", ".safetensors", ".lock")
+roots = [S, "/workspace/SPIRA/web/public"]; hits = []; n = 0; skipped = []; nbytes = 0
+for root in roots:
+    for dp, dn, fn in os.walk(root):
+        dn[:] = [d for d in dn if not d.startswith(EXCL_DIR) and d not in ("broad", "__pycache__", ".git")]
+        for f in fn:
+            fp = os.path.join(dp, f)
+            if "site-packages" in fp or "/venv" in fp or "/.venv" in fp: continue
+            if not (f.endswith(TXT) or "hf_ready" in fp or root != S or f == "spira_relay.md"): continue
+            try: sz = os.path.getsize(fp)
+            except OSError: continue
+            if sz > 20 << 20: skipped.append([fp, sz]); continue
+            b = open(fp, "rb").read(); n += 1; nbytes += sz
+            for wname, p, c in P:
+                for kind, t in (("prompt", p), ("cont", c)):
+                    full = t in b; k32 = sum(1 for i in range(0, 169) if t[i:i + 32] in b)
+                    if full or k32: hits.append({"file": fp, "window": wname, "seg": kind, "full200": full, "sub32_hits_sur_169": k32})
+R.update({"fichiers_scannes": n, "octets_scannes": nbytes, "ignores_gt20Mo": skipped, "exclusions": "NEVER_PUSH*, broad/, .git, __pycache__, binaires " + " ".join(EXCL_SUFFIX) + " ; seuls les fichiers texte " + " ".join(TXT) + ", tout hf_ready, tout web/public ; venv/site-packages exclus ; ce script et son résultat ne contiennent aucun texte", "hits": hits, "secondes": round(time.perf_counter() - T0, 1), "maxrss_Mo": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024})
+json.dump(R, open(os.path.dirname(os.path.abspath(__file__)) + "/gen_kit_check_result.json", "w"), indent=1)
+print(n, len(hits), len(skipped), R["secondes"])

results/hashing/gen_kit_check_r1/gen_kit_check_v3.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""gen_kit_check_v3 — SPIRA Hashing. Vérif STATIQUE du kit de génération : figé (sha) + caviardé (aucune copie de prompt/continuation hors sources).
+N'affiche ni n'écrit aucun texte de prompt : seulement chemins, sha, compteurs. v2 (e0ebfd92) inchangé.
+rc : 0 = PASS ; 3 = FAIL ; 2 = analyse impossible.
+Règles :
+- figé : sha des fichiers du kit = attendus ; pour chaque item du manifeste, window_sha_ok, prompt_sha_ok, cont_sha_ok tous vrais ; manifeste à 5 items, blob_sha = sha du blob.
+- caviardé : dans le périmètre scanné, tout fichier contenant les 200 o exacts ou ≥ 1 sous-chaîne de 32 o d'un prompt/continuation → hit ;
+  un hit est TOLÉRÉ seulement sous un préfixe de corpus source déclaré (--source-prefix) ; tout autre hit → FAIL.
+- périmètre : fichiers texte (TXT) + tout fichier sous hf_ready ou sous une racine publique (--public-root) ; EXCL_SUFFIX appliqué partout
+  SAUF sous hf_ready et racines publiques (ce qui est publiable est scanné quel que soit le suffixe) ; dossiers NEVER_PUSH*, broad, .git, __pycache__, venv exclus.
+  Limite déclarée : broad/ et /tmp ne sont pas lus."""
+import os, sys, json, hashlib, time, resource, argparse
+T0 = time.perf_counter(); sha = lambda b: hashlib.sha256(b).hexdigest()
+S = "/workspace/spira_shared/"; G = S + "checker/recount/gen_test/"
+ap = argparse.ArgumentParser()
+ap.add_argument("--data-root", default=S)
+ap.add_argument("--manifest", default=G + "gen_prompts_manifest.json")
+ap.add_argument("--expect", default=json.dumps({
+    G + "gen_prompts.py": "57fed6cb540f5fb1c91f23a62d96a5c9c9587030e60bf9b4685eede5438ac029",
+    G + "gen_prompts_manifest.json": "70acee7babde01f77e53d3691c40d4cafae3e55c3d7043e1966de36a9320c2d9",
+    G + "NEVER_PUSH/prompts_and_cont.bin": "ca47ea802b09312c1419563a28fe1e2ccab13df1511bf339c8008183dce47902",
+    S + "checker/recount/runC_ready/PREDECL_gen_test_runC_r1.md": "e451dfc7978bd92ada40b45e7c4efc94776555722e2b42a47b74fb1db5dfa66a",
+    S + "checker/recount/runC_ready/gen_test_kit_r2/gen_kit_r2_result.json": "026dc89f20c58833790ffca86b74572f12f933e7cb2145f703b12649e99aeb9d"}))
+ap.add_argument("--blob", default=G + "NEVER_PUSH/prompts_and_cont.bin")
+ap.add_argument("--n-items", type=int, default=5)
+ap.add_argument("--scan-root", action="append")
+ap.add_argument("--public-root", action="append")
+ap.add_argument("--source-prefix", action="append")
+ap.add_argument("--out", required=True)
+a = ap.parse_args()
+scan_roots = a.scan_root or [S]; pub_roots = a.public_root or ["/workspace/SPIRA/web/public"]
+src_pfx = a.source_prefix or [S + "corpus_v3/raw/", S + "corpus_gutenberg/", S + "corpus_gutenberg_v2/raw/"]
+TXT = (".md", ".json", ".txt", ".log", ".csv", ".diff", ".py", ".sh", ".yaml", ".yml", ".html")
+EXCL_SUFFIX = (".bin", ".npz", ".npy", ".pt", ".ckpt", ".safetensors", ".lock")
+SKIP_DIR = ("NEVER_PUSH", "broad", ".git", "__pycache__", "venv", ".venv", "site-packages")
+R = {"version": "v3", "rc": 2, "verdict": None, "erreur": None}
+def fin(rc, err=None):
+    R["rc"] = rc; R["erreur"] = err; R["verdict"] = {0: "PASS", 3: "FAIL"}.get(rc, "ANALYSE IMPOSSIBLE")
+    R["secondes"] = round(time.perf_counter() - T0, 1); R["maxrss_Mo"] = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024
+    try: json.dump(R, open(a.out, "w"), indent=1)
+    except Exception as e: print("sortie impossible", e, file=sys.stderr); sys.exit(2)
+    print(json.dumps({k: R[k] for k in ("rc", "verdict", "erreur", "fichiers_scannes", "n_hits_hors_source", "n_hits_source") if k in R})); sys.exit(rc)
+try:
+    exp = json.loads(a.expect); man = json.load(open(a.manifest)); blob = open(a.blob, "rb").read()
+except Exception as e: fin(2, f"lecture impossible : {e}")
+fails = []
+R["kit_sha"] = {}
+for f, e in exp.items():
+    try: h = sha(open(f, "rb").read())
+    except Exception as x: fin(2, f"fichier du kit absent : {f}")
+    ok = h.startswith(e); R["kit_sha"][f] = {"sha": h, "attendu": e, "ok": ok}
+    if not ok: fails.append("kit_sha:" + f)
+if man.get("blob_sha256") != sha(blob): fails.append("blob_sha_manifeste")
+items = man.get("items", [])
+if len(items) != a.n_items: fails.append(f"n_items {len(items)}≠{a.n_items}")
+P = []; R["items"] = []
+for it in items:
+    try: w = open(os.path.join(a.data_root, it["path"]), "rb").read()
+    except Exception: fin(2, f"fenêtre absente : {it.get('path')}")
+    s0 = it["start"]; p = w[s0:s0 + 200]; c = w[s0 + 200:s0 + 400]
+    d = {"window": it["window"], "window_sha_ok": sha(w) == it["window_sha256"], "prompt_sha_ok": sha(p) == it["prompt_sha256"], "cont_sha_ok": sha(c) == it["cont_sha256"], "prompt_len_ok": len(p) == 200 and len(c) == 200}
+    R["items"].append(d)
+    for k in ("window_sha_ok", "prompt_sha_ok", "cont_sha_ok", "prompt_len_ok"):
+        if not d[k]: fails.append(f"{it['window']}:{k}")
+    P.append((it["window"], p, c))
+hits = []; n = 0; nb = 0; skipped = []
+for root in scan_roots + pub_roots:
+    pub = root in pub_roots
+    for dp, dn, fn in os.walk(root):
+        dn[:] = [x for x in dn if not x.startswith(SKIP_DIR) and x not in SKIP_DIR]
+        for f in fn:
+            fp = os.path.join(dp, f); forced = pub or "hf_ready" in fp
+            if fp.startswith("/tmp/"): continue
+            if not forced and (f.endswith(EXCL_SUFFIX) or not (f.endswith(TXT) or f == "spira_relay.md")): continue
+            try: b = open(fp, "rb").read()
+            except Exception as e: skipped.append([fp, str(e)[:80]]); continue
+            n += 1; nb += len(b)
+            for wn, p, c in P:
+                for kind, t in (("prompt", p), ("cont", c)):
+                    full = t in b; k32 = sum(1 for i in range(169) if t[i:i + 32] in b)
+                    if full or k32: hits.append({"file": fp, "window": wn, "seg": kind, "full200": full, "sub32_sur_169": k32, "source_toleree": any(fp.startswith(x) for x in src_pfx)})
+bad = [h for h in hits if not h["source_toleree"]]
+R.update({"perimetre": {"scan_roots": scan_roots, "public_roots": pub_roots, "source_prefixes": src_pfx, "TXT": TXT, "EXCL_SUFFIX": EXCL_SUFFIX, "SKIP_DIR": SKIP_DIR},
+          "fichiers_scannes": n, "octets_scannes": nb, "illisibles": skipped, "hits": hits, "n_hits_source": len(hits) - len(bad), "n_hits_hors_source": len(bad), "fails_fige": fails})
+if bad: fails.append(f"caviardage: {len(bad)} hit(s) hors source")
+if skipped: fails.append(f"{len(skipped)} fichier(s) illisible(s)")
+R["fails"] = fails
+fin(3 if fails else 0)

results/hashing/gen_kit_check_r1/gen_kit_check_v3_result.json

@@ -0,0 +1,277 @@
+{
+ "version": "v3",
+ "rc": 0,
+ "verdict": "PASS",
+ "erreur": null,
+ "kit_sha": {
+  "/workspace/spira_shared/checker/recount/gen_test/gen_prompts.py": {
+   "sha": "57fed6cb540f5fb1c91f23a62d96a5c9c9587030e60bf9b4685eede5438ac029",
+   "attendu": "57fed6cb540f5fb1c91f23a62d96a5c9c9587030e60bf9b4685eede5438ac029",
+   "ok": true
+  },
+  "/workspace/spira_shared/checker/recount/gen_test/gen_prompts_manifest.json": {
+   "sha": "70acee7babde01f77e53d3691c40d4cafae3e55c3d7043e1966de36a9320c2d9",
+   "attendu": "70acee7babde01f77e53d3691c40d4cafae3e55c3d7043e1966de36a9320c2d9",
+   "ok": true
+  },
+  "/workspace/spira_shared/checker/recount/gen_test/NEVER_PUSH/prompts_and_cont.bin": {
+   "sha": "ca47ea802b09312c1419563a28fe1e2ccab13df1511bf339c8008183dce47902",
+   "attendu": "ca47ea802b09312c1419563a28fe1e2ccab13df1511bf339c8008183dce47902",
+   "ok": true
+  },
+  "/workspace/spira_shared/checker/recount/runC_ready/PREDECL_gen_test_runC_r1.md": {
+   "sha": "e451dfc7978bd92ada40b45e7c4efc94776555722e2b42a47b74fb1db5dfa66a",
+   "attendu": "e451dfc7978bd92ada40b45e7c4efc94776555722e2b42a47b74fb1db5dfa66a",
+   "ok": true
+  },
+  "/workspace/spira_shared/checker/recount/runC_ready/gen_test_kit_r2/gen_kit_r2_result.json": {
+   "sha": "026dc89f20c58833790ffca86b74572f12f933e7cb2145f703b12649e99aeb9d",
+   "attendu": "026dc89f20c58833790ffca86b74572f12f933e7cb2145f703b12649e99aeb9d",
+   "ok": true
+  }
+ },
+ "items": [
+  {
+   "window": "fc2",
+   "window_sha_ok": true,
+   "prompt_sha_ok": true,
+   "cont_sha_ok": true,
+   "prompt_len_ok": true
+  },
+  {
+   "window": "fresh_en",
+   "window_sha_ok": true,
+   "prompt_sha_ok": true,
+   "cont_sha_ok": true,
+   "prompt_len_ok": true
+  },
+  {
+   "window": "fresh_fr",
+   "window_sha_ok": true,
+   "prompt_sha_ok": true,
+   "cont_sha_ok": true,
+   "prompt_len_ok": true
+  },
+  {
+   "window": "test20k_old",
+   "window_sha_ok": true,
+   "prompt_sha_ok": true,
+   "cont_sha_ok": true,
+   "prompt_len_ok": true
+  },
+  {
+   "window": "test_v3",
+   "window_sha_ok": true,
+   "prompt_sha_ok": true,
+   "cont_sha_ok": true,
+   "prompt_len_ok": true
+  }
+ ],
+ "perimetre": {
+  "scan_roots": [
+   "/workspace/spira_shared/"
+  ],
+  "public_roots": [
+   "/workspace/SPIRA/web/public"
+  ],
+  "source_prefixes": [
+   "/workspace/spira_shared/corpus_v3/raw/",
+   "/workspace/spira_shared/corpus_gutenberg/",
+   "/workspace/spira_shared/corpus_gutenberg_v2/raw/"
+  ],
+  "TXT": [
+   ".md",
+   ".json",
+   ".txt",
+   ".log",
+   ".csv",
+   ".diff",
+   ".py",
+   ".sh",
+   ".yaml",
+   ".yml",
+   ".html"
+  ],
+  "EXCL_SUFFIX": [
+   ".bin",
+   ".npz",
+   ".npy",
+   ".pt",
+   ".ckpt",
+   ".safetensors",
+   ".lock"
+  ],
+  "SKIP_DIR": [
+   "NEVER_PUSH",
+   "broad",
+   ".git",
+   "__pycache__",
+   "venv",
+   ".venv",
+   "site-packages"
+  ]
+ },
+ "fichiers_scannes": 53727,
+ "octets_scannes": 1034296244,
+ "illisibles": [],
+ "hits": [
+  {
+   "file": "/workspace/spira_shared/corpus_v3/raw/pg1184.txt",
+   "window": "test_v3",
+   "seg": "prompt",
+   "full200": true,
+   "sub32_sur_169": 169,
+   "source_toleree": true
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_v3/raw/pg1184.txt",
+   "window": "test_v3",
+   "seg": "cont",
+   "full200": true,
+   "sub32_sur_169": 169,
+   "source_toleree": true
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_v3/raw/pg5200.txt",
+   "window": "fresh_en",
+   "seg": "prompt",
+   "full200": false,
+   "sub32_sur_169": 107,
+   "source_toleree": true
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_v3/raw/pg5200.txt",
+   "window": "fresh_en",
+   "seg": "cont",
+   "full200": false,
+   "sub32_sur_169": 96,
+   "source_toleree": true
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg/corpus/books/frankenstein.txt",
+   "window": "test20k_old",
+   "seg": "prompt",
+   "full200": false,
+   "sub32_sur_169": 107,
+   "source_toleree": true
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg/corpus/books/frankenstein.txt",
+   "window": "test20k_old",
+   "seg": "cont",
+   "full200": false,
+   "sub32_sur_169": 97,
+   "source_toleree": true
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg/corpus/books/metamorphosis.txt",
+   "window": "fresh_en",
+   "seg": "prompt",
+   "full200": false,
+   "sub32_sur_169": 107,
+   "source_toleree": true
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg/corpus/books/metamorphosis.txt",
+   "window": "fresh_en",
+   "seg": "cont",
+   "full200": false,
+   "sub32_sur_169": 96,
+   "source_toleree": true
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg_v2/raw/pg83.txt",
+   "window": "test_v3",
+   "seg": "prompt",
+   "full200": false,
+   "sub32_sur_169": 2,
+   "source_toleree": true
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg_v2/raw/pg203.txt",
+   "window": "test_v3",
+   "seg": "prompt",
+   "full200": false,
+   "sub32_sur_169": 6,
+   "source_toleree": true
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg_v2/raw/pg200.txt",
+   "window": "fresh_en",
+   "seg": "cont",
+   "full200": false,
+   "sub32_sur_169": 3,
+   "source_toleree": true
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg_v2/raw/pg90.txt",
+   "window": "test_v3",
+   "seg": "cont",
+   "full200": false,
+   "sub32_sur_169": 1,
+   "source_toleree": true
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg_v2/raw/pg82.txt",
+   "window": "test_v3",
+   "seg": "prompt",
+   "full200": false,
+   "sub32_sur_169": 6,
+   "source_toleree": true
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg_v2/raw/pg415.txt",
+   "window": "test_v3",
+   "seg": "prompt",
+   "full200": false,
+   "sub32_sur_169": 2,
+   "source_toleree": true
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg_v2/raw/pg84.txt",
+   "window": "test20k_old",
+   "seg": "prompt",
+   "full200": false,
+   "sub32_sur_169": 107,
+   "source_toleree": true
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg_v2/raw/pg84.txt",
+   "window": "test20k_old",
+   "seg": "cont",
+   "full200": false,
+   "sub32_sur_169": 97,
+   "source_toleree": true
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg_v2/raw/pg308.txt",
+   "window": "test_v3",
+   "seg": "cont",
+   "full200": false,
+   "sub32_sur_169": 2,
+   "source_toleree": true
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg_v2/raw/pg344.txt",
+   "window": "test_v3",
+   "seg": "cont",
+   "full200": false,
+   "sub32_sur_169": 2,
+   "source_toleree": true
+  },
+  {
+   "file": "/workspace/spira_shared/corpus_gutenberg_v2/raw/pg164.txt",
+   "window": "test_v3",
+   "seg": "cont",
+   "full200": false,
+   "sub32_sur_169": 7,
+   "source_toleree": true
+  }
+ ],
+ "n_hits_source": 19,
+ "n_hits_hors_source": 0,
+ "fails_fige": [],
+ "fails": [],
+ "secondes": 432.9,
+ "maxrss_Mo": 77.12890625
+}

results/hashing/gen_kit_check_r1/gen_kit_check_v3b.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""gen_kit_check_v3b (= v3 e01d2f2a, en-tête corrigé seulement) — SPIRA Hashing. Vérif STATIQUE du kit de génération : figé (sha) + caviardé (aucune copie de prompt/continuation hors sources).
+N'affiche ni n'écrit aucun texte de prompt : seulement chemins, sha, compteurs. v2 (e0ebfd92) inchangé.
+rc : 0 = PASS ; 3 = FAIL ; 2 = analyse impossible.
+Règles :
+- figé : sha des fichiers du kit = attendus ; pour chaque item du manifeste, window_sha_ok, prompt_sha_ok, cont_sha_ok tous vrais ; manifeste à 5 items, blob_sha = sha du blob.
+- caviardé : dans le périmètre scanné, tout fichier contenant les 200 o exacts ou ≥ 1 sous-chaîne de 32 o d'un prompt/continuation → hit ;
+  un hit est TOLÉRÉ seulement sous un préfixe de corpus source déclaré (--source-prefix) ; tout autre hit → FAIL.
+- périmètre : fichiers texte (TXT) + tout fichier sous hf_ready ou sous une racine publique (--public-root)  ; EXCL_SUFFIX N'EST PAS appliqué
+  aux fichiers forcés (sous hf_ready ou une racine publique : scannés quel que soit le suffixe) ; il l'est seulement ailleurs ; dossiers NEVER_PUSH*, broad, .git, __pycache__, venv exclus.
+  Limite déclarée : broad/ et /tmp ne sont pas lus."""
+import os, sys, json, hashlib, time, resource, argparse
+T0 = time.perf_counter(); sha = lambda b: hashlib.sha256(b).hexdigest()
+S = "/workspace/spira_shared/"; G = S + "checker/recount/gen_test/"
+ap = argparse.ArgumentParser()
+ap.add_argument("--data-root", default=S)
+ap.add_argument("--manifest", default=G + "gen_prompts_manifest.json")
+ap.add_argument("--expect", default=json.dumps({
+    G + "gen_prompts.py": "57fed6cb540f5fb1c91f23a62d96a5c9c9587030e60bf9b4685eede5438ac029",
+    G + "gen_prompts_manifest.json": "70acee7babde01f77e53d3691c40d4cafae3e55c3d7043e1966de36a9320c2d9",
+    G + "NEVER_PUSH/prompts_and_cont.bin": "ca47ea802b09312c1419563a28fe1e2ccab13df1511bf339c8008183dce47902",
+    S + "checker/recount/runC_ready/PREDECL_gen_test_runC_r1.md": "e451dfc7978bd92ada40b45e7c4efc94776555722e2b42a47b74fb1db5dfa66a",
+    S + "checker/recount/runC_ready/gen_test_kit_r2/gen_kit_r2_result.json": "026dc89f20c58833790ffca86b74572f12f933e7cb2145f703b12649e99aeb9d"}))
+ap.add_argument("--blob", default=G + "NEVER_PUSH/prompts_and_cont.bin")
+ap.add_argument("--n-items", type=int, default=5)
+ap.add_argument("--scan-root", action="append")
+ap.add_argument("--public-root", action="append")
+ap.add_argument("--source-prefix", action="append")
+ap.add_argument("--out", required=True)
+a = ap.parse_args()
+scan_roots = a.scan_root or [S]; pub_roots = a.public_root or ["/workspace/SPIRA/web/public"]
+src_pfx = a.source_prefix or [S + "corpus_v3/raw/", S + "corpus_gutenberg/", S + "corpus_gutenberg_v2/raw/"]
+TXT = (".md", ".json", ".txt", ".log", ".csv", ".diff", ".py", ".sh", ".yaml", ".yml", ".html")
+EXCL_SUFFIX = (".bin", ".npz", ".npy", ".pt", ".ckpt", ".safetensors", ".lock")
+SKIP_DIR = ("NEVER_PUSH", "broad", ".git", "__pycache__", "venv", ".venv", "site-packages")
+R = {"version": "v3", "rc": 2, "verdict": None, "erreur": None}
+def fin(rc, err=None):
+    R["rc"] = rc; R["erreur"] = err; R["verdict"] = {0: "PASS", 3: "FAIL"}.get(rc, "ANALYSE IMPOSSIBLE")
+    R["secondes"] = round(time.perf_counter() - T0, 1); R["maxrss_Mo"] = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024
+    try: json.dump(R, open(a.out, "w"), indent=1)
+    except Exception as e: print("sortie impossible", e, file=sys.stderr); sys.exit(2)
+    print(json.dumps({k: R[k] for k in ("rc", "verdict", "erreur", "fichiers_scannes", "n_hits_hors_source", "n_hits_source") if k in R})); sys.exit(rc)
+try:
+    exp = json.loads(a.expect); man = json.load(open(a.manifest)); blob = open(a.blob, "rb").read()
+except Exception as e: fin(2, f"lecture impossible : {e}")
+fails = []
+R["kit_sha"] = {}
+for f, e in exp.items():
+    try: h = sha(open(f, "rb").read())
+    except Exception as x: fin(2, f"fichier du kit absent : {f}")
+    ok = h.startswith(e); R["kit_sha"][f] = {"sha": h, "attendu": e, "ok": ok}
+    if not ok: fails.append("kit_sha:" + f)
+if man.get("blob_sha256") != sha(blob): fails.append("blob_sha_manifeste")
+items = man.get("items", [])
+if len(items) != a.n_items: fails.append(f"n_items {len(items)}≠{a.n_items}")
+P = []; R["items"] = []
+for it in items:
+    try: w = open(os.path.join(a.data_root, it["path"]), "rb").read()
+    except Exception: fin(2, f"fenêtre absente : {it.get('path')}")
+    s0 = it["start"]; p = w[s0:s0 + 200]; c = w[s0 + 200:s0 + 400]
+    d = {"window": it["window"], "window_sha_ok": sha(w) == it["window_sha256"], "prompt_sha_ok": sha(p) == it["prompt_sha256"], "cont_sha_ok": sha(c) == it["cont_sha256"], "prompt_len_ok": len(p) == 200 and len(c) == 200}
+    R["items"].append(d)
+    for k in ("window_sha_ok", "prompt_sha_ok", "cont_sha_ok", "prompt_len_ok"):
+        if not d[k]: fails.append(f"{it['window']}:{k}")
+    P.append((it["window"], p, c))
+hits = []; n = 0; nb = 0; skipped = []
+for root in scan_roots + pub_roots:
+    pub = root in pub_roots
+    for dp, dn, fn in os.walk(root):
+        dn[:] = [x for x in dn if not x.startswith(SKIP_DIR) and x not in SKIP_DIR]
+        for f in fn:
+            fp = os.path.join(dp, f); forced = pub or "hf_ready" in fp
+            if fp.startswith("/tmp/"): continue
+            if not forced and (f.endswith(EXCL_SUFFIX) or not (f.endswith(TXT) or f == "spira_relay.md")): continue
+            try: b = open(fp, "rb").read()
+            except Exception as e: skipped.append([fp, str(e)[:80]]); continue
+            n += 1; nb += len(b)
+            for wn, p, c in P:
+                for kind, t in (("prompt", p), ("cont", c)):
+                    full = t in b; k32 = sum(1 for i in range(169) if t[i:i + 32] in b)
+                    if full or k32: hits.append({"file": fp, "window": wn, "seg": kind, "full200": full, "sub32_sur_169": k32, "source_toleree": any(fp.startswith(x) for x in src_pfx)})
+bad = [h for h in hits if not h["source_toleree"]]
+R.update({"perimetre": {"scan_roots": scan_roots, "public_roots": pub_roots, "source_prefixes": src_pfx, "TXT": TXT, "EXCL_SUFFIX": EXCL_SUFFIX, "SKIP_DIR": SKIP_DIR},
+          "fichiers_scannes": n, "octets_scannes": nb, "illisibles": skipped, "hits": hits, "n_hits_source": len(hits) - len(bad), "n_hits_hors_source": len(bad), "fails_fige": fails})
+if bad: fails.append(f"caviardage: {len(bad)} hit(s) hors source")
+if skipped: fails.append(f"{len(skipped)} fichier(s) illisible(s)")
+R["fails"] = fails
+fin(3 if fails else 0)

results/hashing/gen_kit_check_r1/relay_entry_debut.md

@@ -0,0 +1,4 @@
+### Sat Oct 31 12:13:17 EDT 2026 — SPIRA Hashing — gen_kit_check_r1 DÉBUT
+- gen_kit_check.py (scan statique de caviardage, aucun texte écrit) : PID 1163831, nice 19, lancé vers 09:06 heure de Vancouver ; plus de 10 min écoulées ; ETA inconnue (scan de tout spira_shared hors binaires) ; MemAvailable 9,9 Go.
+
+Lu/appliqué — SPIRA Hashing

results/hashing/gen_kit_check_r1/relay_entry_v3_debut.md

@@ -0,0 +1,4 @@
+### Sat Oct 31 12:30:33 EDT 2026 — SPIRA Hashing — gen_kit_check v3 DÉBUT
+- gen_kit_check_v3.py (statique seulement, nice 19, PYTHONDONTWRITEBYTECODE=1 ; v2 e0ebfd92 non modifié). Selftest 18/18. Lancé à 09:30 heure de Vancouver, PID 1213458, ETA ≈ 8 min (v2 : 434 s), RSS attendu < 100 Mo.
+
+Lu/appliqué — SPIRA Hashing

results/hashing/gen_kit_check_r1/relay_entry_v3_fin.md

@@ -0,0 +1,5 @@
+### Sat Oct 31 12:39:43 EDT 2026 — SPIRA Hashing — gen_kit_check v3 FIN + v3b
+- v3 FIN : PID 1213458, 432,9 s, maxrss 77 Mo. rc 0 PASS : 53 727 fichiers (1,03 Go lus), figé OK (5 sha du kit + 15 *_sha_ok), 0 hit hors source, 19 hits en sources tolérées, 0 illisible. gen_kit_check_v3_result.json da94f7ae ; v3.py e01d2f2a ; selftest 18/18 (898a0d9b / résultat bc6a8f1e).
+- v3b (signalement Checker t31) : en-tête l.9-10 corrigé (EXCL_SUFFIX NON appliqué aux fichiers forcés hf_ready/public) + l.2 ; diff = docstring seule, AST du code identique. v3b 79be901c ; SHA256SUMS_v3 f79446b7. v3 intact. Note : c'est un choix délibéré de v3 (le publiable est scanné quel que soit le suffixe), ce n'est pas un oubli du code.
+
+Lu/appliqué — SPIRA Hashing

results/hashing/go_package_missing_r1/MISSING_DECISIONS.md

@@ -0,0 +1,81 @@
+# MISSING_DECISIONS r1 — SPIRA Hashing — 31 oct. 2026 — document seulement
+Complète GO_PACKAGE_runC_B.md (5ee520ff), qui ne couvre que D1–D10 (audit_r13, AUDIT.md 03583d4f, point 2 FAIL).
+Aucun PID, aucun job lancé. Rien poussé, rien supprimé, aucun drapeau GO. Sources : DECISIONS_EN_ATTENTE r8 (fe2308ed, reprenant r4–r7), relais SPIRA.
+Aucune des décisions ci-dessous n'a de défaut : les « défauts » des fiches Checker antérieures sont des propositions, pas des choix.
+
+## D11 — Option d'échelle de l'éval appariée (A à E)
+- Question : quelle option de scale pour imposed_pos_eval ?
+- Options : A descriptif (aucun verdict) ; B bornes élargies (change la règle) ; C épingler à 0,03 ; D scale commun 0,026973 (moyenne arithmétique vérifiée) ; E recalibrer (nouveau PREDECL).
+- Éclairage : NOTE_options_scale r2 2269b82b ; measure_common_r2.py d2941de2 (selftest PASS) ; audits Hashing 52fbc37d, 9a21be82, bbb367de. RAM mesurée 0,96–0,97 Go par bras en éval native, 1,13 Go au chargement d'un ckpt ; plafond déclaré 1,3 Go par bras. Lien avec D3 (rôle de r4, option C recommandée non tranchée).
+- Aucun défaut — EN ATTENTE DE DÉCISION de Philippe-Antoine.
+
+## D12 — Politique de scale et de bornes pour v1d
+- Question : que faire des bornes fusion.scale [0,03 ; 0,11] pour v1d ?
+- Options : A abaisser la borne (changement de règle permanente) ; C garder [0,03 ; 0,11] ; E autre scale.
+- Éclairage : scales ctl EXACT 0,02399 (1 155 793 ticks) et 0,02566 (1 200 000) → 0 point EXACT dans les bornes (scale_table r2 1b0da0a8). Brouillons C 49fab81d et E 87f03b10 (spira_m8/v1c/drafts_r1/, 5/5 OK). Bench recalibré (results_v4.jsonl) : 7 paires, |Δ| max 2,87 %, 4/7 dans les bornes. Conditionne la mesurabilité de D1.
+- Aucun défaut — EN ATTENTE DE DÉCISION de Philippe-Antoine.
+
+## D13 — Option (iii) : usage du ratio σf/σs des checkpoints existants
+- Question : quel usage faire de run_iii pour la question du scale ?
+- Options : en prendre acte comme éclairage de D12 ; ne pas l'utiliser.
+- Éclairage : run_iii r3 6/6 PASS (de7738d8) ; coût mesuré 44,3 min, pic 1 129,7 Mo (SHA256SUMS 71fce159 et 46358f8c) ; cm 60k = 0,90554 EXACT. Levier « ckpts existants » épuisé : aucun ckpt entre 60k et ~1,16M ticks (d'où D15).
+- Aucun défaut — EN ATTENTE DE DÉCISION de Philippe-Antoine.
+
+## D14 — Pré-cal r3 sur la porte v1c
+- Question : appliquer ou non le correctif r3 (fix 1cfc5aa9) à la porte v1c ?
+- Options : appliquer ; ne pas appliquer.
+- Éclairage : REPORT_r3 PASS d335e3ed, réserve : préfixe 103f97dc de 3 999 704 o contre échantillon de 35 948 296 o. Sans effet sur v1d (PREDECL v1d r2 l.21 écarte r3 ; le run C utilise le préfixe 992d20aa).
+- Aucun défaut — EN ATTENTE DE DÉCISION de Philippe-Antoine.
+
+## D15 — GO du sweep de précal (balayage de longueur)
+- Question : lancer le balayage de PREDECL_precal_sweep_BROUILLON_r2 (0f2db9f9, ré-audit Checker PASS 609bb24a), et avec quels paramètres ?
+- Options à fixer (PREDECL r2 l.58) :
+  - lancement : oui / non ;
+  - voies : 2 voies (1 cm + 1 ctl) ≈ 20–21 h murales, RAM ≤ ~3,2 Go ; ou 4 voies (3 cm + 1 ctl) ≈ 11 h, RAM ≤ ~6,5 Go pire cas ;
+  - données : precal_prefix.bin 103f97dc (~4 Mo, capacité à 1,2M ticks en 1 époque NON vérifiée) ou v1c_sample.bin 3d8b6181 (35,9 Mo ; change la comparaison directe avec le point 60k) ;
+  - bornes et nombre de points (proposé : 5 points log-espacés, 60 000 → 1 200 000 ticks, 2 bras, 10 runs) ;
+  - lieu de mesure de l'échelle : data[:60000] d'entraînement ou v1d.
+- Éclairage : CPU total ≈ 26 h ; RAM cm à 1,2M NON MESURÉE (1,2–2 Go, ±50 %) ; débit ±30 % ; garde run_guarded_v2 1c6f64b4 (MemAvailable ≥ 5,2 Go et < 4 jobs lourds, jamais de kill) ; trainer 4049327c ; v1d 903970c3.
+- Aucun défaut — EN ATTENTE DE DÉCISION de Philippe-Antoine.
+
+## D16 — leak16 sur 6 ou 8 fenêtres
+- Question : la règle leak16 porte-t-elle sur 6 fenêtres, ou sur 8 avec neither_code et bg1_test ? (Distinct de D10, qui demande seulement si leak16 est bloquant.)
+- Options : 6 fenêtres ; 8 fenêtres.
+- Éclairage : ref_checker_8w 01cb94f6 ; leak16_reconcile_r2 9d7d8383 (écart dû aux seules fenêtres neither_code) ; hf_ready r3 : maxima identiques en 6 et 8 (13 o), MATCH 6/6 avec Hashing a94c0833. À ce jour tout est ≤ 15 o dans les deux cas : seule la règle future change.
+- Aucun défaut — EN ATTENTE DE DÉCISION de Philippe-Antoine.
+
+## D17 — Exception .py pour la garde de push
+- Question : accorder une exception ou un seuil propre aux .py (et runner.sh) ?
+- Options (DECISION_py_exception f976b8aa ; 7 fichiers audités) : O1 pas d'exception, 0/7 publiables ; O2 liste blanche masquée à 16 o, 3/7 ; O2b liste blanche + commentaires et docstrings masqués, 4/7 ; O3 seuil 32 o pour le code, 5/7.
+- Éclairage : recoupement Hashing dc308884, 154/154 maxima identiques. **La liste blanche masque des fuites** : sous WL, un passage masqué ≥ 16 o apparaît dans 10/11 fenêtres (max 94 o neither_code ; import/from 72 o fresh_en, 76 o test_v3) ; avec WL+doc, jusqu'à 4 725 o (neither_code) et 2 480 o (bg1_test). La liste blanche est donc FAIL comme garde. O3 laisse passer des phrases de 17–31 o. Toute exception modifie la garde de push.
+- Aucun défaut — EN ATTENTE DE DÉCISION de Philippe-Antoine.
+
+## D18 — Push de run_iii r3 et des autres arbres prêts
+- Question : pousser sur HF (préfixe v3_scale/ seulement) les arbres prêts ? Ce oui est DISTINCT de D5 : décider la garde de push n'autorise aucun push.
+- Options, arbre par arbre : hf_ready_run_iii r3 (de7738d8, 6/6 PASS) ; push-prep Checker 977daff2 (recompte Hashing MATCH, max 15 o, 67dab986) ; imposed_pos_eval_20261031 ef183508 (READY-NOT-PUSHED, max 15 o sur 8 fenêtres, scripts exclus) ; oui / non pour chacun.
+- Éclairage : HF HEAD c2edf5e5 inchangé ; dépend de D16 et D17 pour tout fichier de code ; destination d'own_push_audit_0137 non écrite. Action irréversible.
+- Aucun défaut — EN ATTENTE DE DÉCISION de Philippe-Antoine.
+
+## D19 — Pause de la routine horaire
+- Question : mettre en pause, ou non, la routine horaire de surveillance SPIRA ?
+- Options : maintenir ; mettre en pause (jusqu'à quand) ; arrêter.
+- Éclairage : listée ouverte au relais (l.16141, 31 oct.) ; aucun job SPIRA lourd en cours (passes 07 h 11 et 07 h 17 Toronto, 0/4 par bot). Aucun chiffre de coût publié.
+- Aucun défaut — EN ATTENTE DE DÉCISION de Philippe-Antoine.
+
+## D20 — Échantillon A ou B pour le run C
+- Question : quel échantillon de données pour le run C ?
+- Options : A b6655e90 (34 557 670 o) ; B b7404e88 (35 716 621 o) ; e0acd131 (1 259 666 o de licence inconnue, déconseillé).
+- Éclairage : licence inconnue 0 o pour A et B (ne départage pas) ; budget B −0,28 % contre A −3,52 % ; leak16 queries B 31,78 % contre A 18,17 % ; seul B a un recompte croisé PASS (dc312572) ; PREDECL_sampling_rule_DRAFT 0f61dc29 recommande B (brouillon non gelé) ; règle d'exclusion de B non pré-déclarée. Défaut retiré en r6 (audit Hashing 0e6ccae4 point 5). Préalable à D1.
+- Aucun défaut — EN ATTENTE DE DÉCISION de Philippe-Antoine.
+
+## D21 — Test de génération : lancement et choix du checkpoint
+- Question : lancer le test de génération, et sur quel checkpoint ?
+- Options : ne pas lancer ; lancer en mode descriptif sur un ckpt à désigner (ex. v1c ctl) ; attendre un ckpt MESURABLE.
+- Éclairage : PREDECL_gen_test_r2 36bcfff6 ; gen_test.py ca2b41c6 ; coût ~4–8 min CPU (estimation), RSS mesuré 464–557 Mo ; 0 modèle ADOPTÉ, v1b et v1c NON MESURABLES ; métrique de copie en information seulement (un seuil demande un PREDECL r3).
+- Hashing n'écrit aucun code, harness ni référence jouet pour ce test tant que Philippe-Antoine ne l'a pas ordonné.
+- Aucun défaut — EN ATTENTE DE DÉCISION de Philippe-Antoine.
+
+## Corrections à apporter aux décisions existantes du GO_PACKAGE
+1. D1 : le libellé « Lancer le run C sur l'échantillon B » suppose B, alors que le choix est ouvert (D20). Il indique « Défaut actuel : aucun », alors que DECISIONS r6 et r8 donnent défaut NON tant que le scale n'est pas résolu (0/5 EXACT dans les bornes). À reformuler : « run C sur l'échantillon choisi en D20 », dépend de D12 et D20, et citer le NON de r6/r8 comme ancienne proposition Checker.
+2. Règle ferme « results/ n'est PAS approuvé » : aucune source citée (ni écrit de Philippe-Antoine, ni décision numérotée). Ajouter la source, ou la reformuler en point ouvert.
+3. Règle « Rien n'est poussé sur HF tant que la garde de push (D5) n'est pas décidée » : D5 ne suffit pas. Tout push exige en plus le oui écrit de D18, arbre par arbre, et D16/D17 pour le code. À écrire : « Aucun push sans D5 ET le oui de D18. »

results/hashing/go_package_missing_r1/SHA256SUMS

@@ -0,0 +1 @@
+bac947b79dbd49fb3eb79c3d9fa5205effd4a9215cf6a6781bc455b0183623bb  MISSING_DECISIONS.md

results/hashing/hf_pushed.txt

@@ -0,0 +1,4 @@
+2026-10-30 19:15:56 d3c24c89 v3_scale/hashing/m8cp1/ (8 fichiers verdict/gap/script/FROZEN) Checker PASS 19h16
+2026-10-31 01:44:36 c77978b7 v3_scale/hashing/v1c_ctl_data/ (DIAG_push.md 4fbada39, gap_ctl_data_push.md 789e53f4 ; DESCRIPTIF NON MESURABLE) Checker PASS 01h45
+2026-10-31 02:56:32 f765c61e v3_scale/hashing/ngram8_overlap_v1c_ctx64_cp1/ (cp1_selfcheck_dedup64_norm_sanitized.json 55314114, cp1_selfcheck_dedup64_raw_sanitized.json 4855da6f, cp1_selfcheck_selftest.json 03431af0, results_ctx64_cp1.json 25c1cf5c) Checker PASS 02h55
+2026-10-31 03:46:54 c2edf5e5 v3_scale/hashing/ngram8_overlap_v1c_ctx64_cp1/ (README_push_redacted.md 6d394d5d, build_ctx64_mask_code_prose_v1_redacted.py 390a5223, ngram8_overlap_ctx64_cp1_redacted.py c3015f04 ; r3 caviardé, leakcheck_r3.json NON poussé) Checker PASS 03h46

results/hashing/leak16_baseline_recount/SHA256SUMS

@@ -0,0 +1,6 @@
+9b870e4a3c65057999567662ffbdf690c66cea163f234ef45b8f1b75aa9d9478  recount_leak16.py
+e2ed4e52d646df94a0fe1455e5c0e8e74249a07aeccae378d37d7af26e3b41a0  results.json
+8ef3d3d8523741eb95463015445a2c1f58a06aad24b13d7eb17ba9b77d2567a0  recount_leak16_stream.py
+7aa14491a0010b6254d736f7c86468a0763442d78e27cff1e042943d18a4b7c9  results_stream.json
+066d5a7a0b3c7d95d6e84532b3def027ca2c32d9da9864c092202d10fcf966e1  leakcheck_hf.py
+5dceec4a2501cac734df4785a8830ff24b9fd0bdd51febd0e8ddc8a33adbecb8  leakcheck_hf_ready.out

results/hashing/leak16_baseline_recount/hf_ready/README.md

@@ -0,0 +1,24 @@
+# SPIRA — leak16 baseline recount (sanitized)
+
+Recomptage exact du recouvrement par 16-grammes d'octets, fenêtres d'évaluation SPIRA
+(fc2, fresh_en, fresh_fr, test20k_old, test_v3, queries) vs sources d'entraînement
+(v1c_corpus, v1c_sample, v1b_corpus, v1b_sample).
+
+- `recount_leak16.py` : version d'origine (pic RSS ≈ 365 Mo).
+- `recount_leak16_stream.py` : version streaming (clés V16 brutes, tri en place, searchsorted ; blocs de 256 Kio,
+  recouvrement 15 octets ; pic RSS mesuré 239,5 Mo). Résultats numériques identiques à l'origine.
+- `results_counts.json` : uniquement comptes, pourcentages, tailles et sha256. Aucun fragment de texte, aucun octet de fenêtre.
+
+Méthode : exacte, sans hachage. `n_16gram_positions_shared` = positions (avec répétitions) (fenêtre) dont le 16-gramme
+apparaît dans la source ; `bytes_covered_ge16` = octets couverts par au moins un tel 16-gramme.
+Chaque valeur concordent avec le comptage de Checker (`all_checks_match: true`).
+
+Les scripts lisent des chemins locaux (/workspace/spira_shared/...) ; les données ne sont pas incluses.
+Contrôle de fuite : aucun fichier de ce dossier ne partage une suite de plus de 15 octets avec une fenêtre d'évaluation.
+Modifications par rapport aux originaux du dossier parent (contrôle ast.dump, fichier par fichier) :
+- `recount_leak16.py` : réindentation (espaces -> tabulations) et blancs avant `#` seulement. AST identique.
+- `recount_leak16_stream.py` : même réindentation, plus une réécriture d'une comparaison : `len(sys.argv)>1` devient `1<len(sys.argv)`
+  (opérandes inversés, même comportement). AST NON identique (un seul nœud Compare diffère), sémantique inchangée.
+- `results_counts.json` : extrait (comptes seulement) de results.json et results_stream.json ; pas un script.
+
+Destination HF prévue : `v3_scale/hashing/leak16_baseline_recount/` (pas results/). Non poussé ; fichiers gardés localement dans hf_ready/.

results/hashing/leak16_baseline_recount/hf_ready/SHA256SUMS

@@ -0,0 +1,4 @@
+aad2672b4de5b06fbc7410561d4fb43004a050833cc152d3f388a42bbe004b4a  README.md
+b2854be82b592f5743387a2d284bd5bfd3304f5822d504dcfb4b27a00ea00375  recount_leak16.py
+734270e1f593e95de6fd22635357b9473517bc6f416aa04bde1e5e593c263a79  recount_leak16_stream.py
+7f4d1ce499361abb0d4039c7e5819b99fbee78b9878df5032de9b4b97bb7041c  results_counts.json

results/hashing/leak16_baseline_recount/hf_ready/recount_leak16.py

@@ -0,0 +1,58 @@
+# Recount indépendant leak16 (clés exactes 2x8 o, sans hachage, corpus en blocs de 1 Mo). Lecture seule.
+import json,hashlib,sys,resource,time,numpy as np
+R='/workspace/spira_shared/'; T=R+'checker/recount/m8_v1d_eval_prep/leak16_baseline_v1b_v1c.json'
+def sha(p):
+	h=hashlib.sha256()
+	with open(p,'rb') as f:
+		for b in iter(lambda:f.read(1<<20),b''): h.update(b)
+	return h.hexdigest()
+def keys(a): # a uint8 -> (hi,lo) big-endian u64 of each 16-gram
+	n=len(a)-15; hi=np.zeros(n,np.uint64); lo=np.zeros(n,np.uint64)
+	for i in range(8):
+		hi=(hi<<np.uint64(8))|a[i:i+n].astype(np.uint64); lo=(lo<<np.uint64(8))|a[i+8:i+8+n].astype(np.uint64)
+	return hi,lo
+J=json.load(open(T)); out={'target':T,'target_sha256':sha(T),'windows':{},'sources':{}}
+W={};allh=[];alll=[]
+for w,m in J['windows'].items():
+	p=R+m['path']; s=sha(p); a=np.fromfile(p,np.uint8)
+	out['windows'][w]={'path':p,'sha256':s,'sha_ok':s==m['sha256'],'bytes':len(a),'bytes_ok':len(a)==m['bytes']}
+	h,l=keys(a); W[w]=(h,l); allh.append(h); alll.append(l)
+H=np.concatenate(allh);L=np.concatenate(alll); o=np.lexsort((L,H)); H=H[o];L=L[o]
+k=np.ones(len(H),bool); k[1:]=(H[1:]!=H[:-1])|(L[1:]!=L[:-1]); UH=H[k];UL=L[k]; del H,L,o,k,allh,alll
+idx={w:None for w in W}
+for w,(h,l) in W.items():
+	i=np.searchsorted(UH,h,'left'); r=np.searchsorted(UH,h,'right')
+	while True: # exact match on lo within hi group
+		bad=UL[np.minimum(i,len(UL)-1)]!=l
+		if not bad.any(): break
+		i[bad]+=1
+	assert (UH[i]==h).all() and (UL[i]==l).all(); idx[w]=i
+def member(h,l):
+	i=np.searchsorted(UH,h,'left'); r=np.searchsorted(UH,h,'right'); f=np.zeros(len(h),bool); g=np.full(len(h),-1)
+	c=np.nonzero(r>i)[0]; ii=i[c].copy(); rr=r[c]
+	while len(c):
+		m=UL[ii]==l[c]; f[c[m]]=True; g[c[m]]=ii[m]
+		keep=(~m)&(ii+1<rr); c=c[keep]; ii=ii[keep]+1; rr=rr[keep]
+	return g[f]
+for s,m in J['sources'].items():
+	t=time.time(); p=m['path']; sh=sha(p); found=np.zeros(len(UH),bool); n=0; prev=b''
+	with open(p,'rb') as f:
+		for b in iter(lambda:f.read(1<<20),b''):
+			a=np.frombuffer(prev+b,np.uint8); n+=len(b); prev=bytes(a[-15:])
+			if len(a)>=16: found[member(*keys(a))]=True
+	pw={}
+	for w,(h,l) in W.items():
+		hit=found[idx[w]]; N=len(hit)+15; d=np.zeros(N+1,np.int32); pos=np.nonzero(hit)[0]
+		np.add.at(d,pos,1); np.add.at(d,pos+16,-1); cov=np.cumsum(d[:N])>0
+		e=np.diff(np.concatenate(([0],cov.astype(np.int8),[0]))); runs=np.nonzero(e==-1)[0]-np.nonzero(e==1)[0]
+		c=m['per_window'][w]; mine={'bytes_covered_ge16':int(cov.sum()),'n_16gram_positions_shared':int(hit.sum()),
+		'n_16grams_shared_unique':int(len(np.unique(idx[w][hit]))),'longest_covered_run_bytes':int(runs.max() if len(runs) else 0)}
+		mine['frac_covered_ge16']=mine['bytes_covered_ge16']/N
+		mine['checker']=c; mine['covered_match']=mine['bytes_covered_ge16']==c['bytes_covered_ge16']
+		mine['longest_match']=mine['longest_covered_run_bytes']==c['longest_covered_run_bytes']
+		mine['n16_match_positions']=mine['n_16gram_positions_shared']==c['n_16grams_shared']
+		mine['n16_match_unique']=mine['n_16grams_shared_unique']==c['n_16grams_shared']; pw[w]=mine
+	out['sources'][s]={'path':p,'sha256':sh,'sha_ok':sh==m['sha256'],'bytes':n,'bytes_ok':n==m['bytes'],'per_window':pw,'seconds':round(time.time()-t,1)}
+	print(s,round(time.time()-t),flush=True)
+out['peak_rss_mb']=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+json.dump(out,open('results.json','w'),indent=1,ensure_ascii=False); print(out['peak_rss_mb'])

results/hashing/leak16_baseline_recount/hf_ready/recount_leak16_stream.py

@@ -0,0 +1,59 @@
+# Recount leak16 en streaming (< 300 Mo) : version de recount_leak16.py (9b870e4a).
+# Exact : clés = 16 octets bruts (dtype V16, ordre memcmp), aucun hachage ; tableau trié unique + searchsorted ;
+# corpus lu en blocs de 256 Kio avec recouvrement de 15 octets ; n_16grams_shared compté en positions.
+# Lecture seule hors de ce dossier. Sortie : argv[1] (défaut results_stream.json).
+import json,hashlib,sys,resource,time,numpy as np
+from numpy.lib.stride_tricks import sliding_window_view as swv
+R='/workspace/spira_shared/'; T=R+'checker/recount/m8_v1d_eval_prep/leak16_baseline_v1b_v1c.json'
+OUT=sys.argv[1] if 1<len(sys.argv) else 'results_stream.json'; BLK=1<<18
+def sha(p):
+	h=hashlib.sha256()
+	with open(p,'rb') as f:
+		for b in iter(lambda:f.read(1<<20),b''): h.update(b)
+	return h.hexdigest()
+def keys(a): return np.ascontiguousarray(swv(a,16)).view('V16').ravel() # a uint8 -> 16-grammes (copie n*16 o)
+J=json.load(open(T)); out={'target':T,'target_sha256':sha(T),'windows':{},'sources':{}}
+NW={}
+for w,m in J['windows'].items():
+	p=R+m['path']; s=sha(p); NW[w]=n=int(np.fromfile(p,np.uint8).size)
+	out['windows'][w]={'path':p,'sha256':s,'sha_ok':s==m['sha256'],'bytes':n,'bytes_ok':n==m['bytes']}
+U=np.empty(sum(n-15 for n in NW.values()),'V16'); k=0 # un seul tableau préalloué, rempli par blocs
+for w,m in J['windows'].items():
+	a=np.fromfile(R+m['path'],np.uint8)
+	for s0 in range(0,len(a)-15,BLK):
+		q=keys(a[s0:s0+BLK+15]); U[k:k+len(q)]=q; k+=len(q)
+	del a
+assert k==len(U); U.sort() # tri en place (ordre memcmp)
+keep=np.ones(len(U),bool); keep[1:]=U[1:]!=U[:-1]; nu=int(keep.sum())
+U[:nu]=U[keep]; U=U[:nu].copy(); del keep # dédoublonnage exact
+def lookup(q): # -> (indices dans U, masque présent), exact octet à octet
+	i=np.searchsorted(U,q); j=np.minimum(i,len(U)-1); f=U[j]==q; return j,f
+idx={}
+for w,m in J['windows'].items():
+	a=np.fromfile(R+m['path'],np.uint8); ii=[]
+	for s0 in range(0,len(a)-15,BLK):
+		j,f=lookup(keys(a[s0:s0+BLK+15])); assert f.all(); ii.append(j.astype(np.int32))
+	idx[w]=np.concatenate(ii); assert len(idx[w])==NW[w]-15
+for s,m in J['sources'].items():
+	t=time.time(); p=m['path']; sh=sha(p); found=np.zeros(len(U),bool); n=0; prev=b''
+	with open(p,'rb') as f:
+		for b in iter(lambda:f.read(BLK),b''):
+			a=np.frombuffer(prev+b,np.uint8); n+=len(b); prev=bytes(a[-15:])
+			if len(a)>=16:
+				j,fm=lookup(keys(a)); found[j[fm]]=True
+	pw={}
+	for w in idx:
+		hit=found[idx[w]]; N=len(hit)+15; d=np.zeros(N+1,np.int32); pos=np.nonzero(hit)[0]
+		np.add.at(d,pos,1); np.add.at(d,pos+16,-1); cov=np.cumsum(d[:N])>0
+		e=np.diff(np.concatenate(([0],cov.astype(np.int8),[0]))); runs=np.nonzero(e==-1)[0]-np.nonzero(e==1)[0]
+		c=m['per_window'][w]; mine={'bytes_covered_ge16':int(cov.sum()),'n_16gram_positions_shared':int(hit.sum()),
+		'n_16grams_shared_unique':int(len(np.unique(idx[w][hit]))),'longest_covered_run_bytes':int(runs.max() if len(runs) else 0)}
+		mine['frac_covered_ge16']=mine['bytes_covered_ge16']/N
+		mine['checker']=c; mine['covered_match']=mine['bytes_covered_ge16']==c['bytes_covered_ge16']
+		mine['longest_match']=mine['longest_covered_run_bytes']==c['longest_covered_run_bytes']
+		mine['n16_match_positions']=mine['n_16gram_positions_shared']==c['n_16grams_shared']
+		mine['n16_match_unique']=mine['n_16grams_shared_unique']==c['n_16grams_shared']; pw[w]=mine
+	out['sources'][s]={'path':p,'sha256':sh,'sha_ok':sh==m['sha256'],'bytes':n,'bytes_ok':n==m['bytes'],'per_window':pw,'seconds':round(time.time()-t,1)}
+	print(s,round(time.time()-t),flush=True)
+out['peak_rss_mb']=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+json.dump(out,open(OUT,'w'),indent=1,ensure_ascii=False); print(out['peak_rss_mb'])

results/hashing/leak16_baseline_recount/hf_ready/results_counts.json

@@ -0,0 +1,297 @@
+{
+ "description": "leak16 baseline recount (exact 16-byte n-grams, no hashing): eval windows vs training sources v1b/v1c. Counts, percentages, sizes and sha256 only.",
+ "target_json_sha256": "a9ee65d4ad394ada0af487a0cb7eec66073f5440fb405ec19d5df641b5334598",
+ "windows": {
+  "fc2": {
+   "bytes": 199792,
+   "sha256": "b052096b0c1e2b301cd79921502f85e61912f209fc0027151bbf82237de8c2e3"
+  },
+  "fresh_en": {
+   "bytes": 119763,
+   "sha256": "67ff20b26ce5e3401cea57c817b1a8417e95f6cd131fd8fb631cd5cd8625331a"
+  },
+  "fresh_fr": {
+   "bytes": 1088688,
+   "sha256": "875f0906a320e3f7b87e446aab646cf49c74786cb1a0db78b54fa4c62e872715"
+  },
+  "test20k_old": {
+   "bytes": 319940,
+   "sha256": "ce0526c62048f8e590a96cca5393a87c2e631f194eb4a2e974c584fd443b6a22"
+  },
+  "test_v3": {
+   "bytes": 3486653,
+   "sha256": "a6730bbe877e7b28a5529793b27a0b59f16d04db05ea895f0ee6d4f91807ca84"
+  },
+  "queries": {
+   "bytes": 27800,
+   "sha256": "5aab38d62b2f000df6a913d2574ccd314300e350ccc76e71e16768ed5c73b706"
+  }
+ },
+ "sources": {
+  "v1c_corpus": {
+   "bytes": 503301148,
+   "sha256": "33577b70c819af76fe4737319e929f4c097b0c72f020e8b07b4baddbb1823439",
+   "per_window": {
+    "fc2": {
+     "bytes_covered_ge16": 145139,
+     "n_16gram_positions_shared": 71796,
+     "n_16grams_shared_unique": 38057,
+     "longest_covered_run_bytes": 419,
+     "pct_covered_ge16": 72.6451,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    },
+    "fresh_en": {
+     "bytes_covered_ge16": 93456,
+     "n_16gram_positions_shared": 34364,
+     "n_16grams_shared_unique": 32608,
+     "longest_covered_run_bytes": 338,
+     "pct_covered_ge16": 78.0341,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    },
+    "fresh_fr": {
+     "bytes_covered_ge16": 652302,
+     "n_16gram_positions_shared": 194384,
+     "n_16grams_shared_unique": 168701,
+     "longest_covered_run_bytes": 145,
+     "pct_covered_ge16": 59.9163,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    },
+    "test20k_old": {
+     "bytes_covered_ge16": 235956,
+     "n_16gram_positions_shared": 91729,
+     "n_16grams_shared_unique": 73892,
+     "longest_covered_run_bytes": 266,
+     "pct_covered_ge16": 73.7501,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    },
+    "test_v3": {
+     "bytes_covered_ge16": 2433706,
+     "n_16gram_positions_shared": 859632,
+     "n_16grams_shared_unique": 667154,
+     "longest_covered_run_bytes": 353,
+     "pct_covered_ge16": 69.8006,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    },
+    "queries": {
+     "bytes_covered_ge16": 13583,
+     "n_16gram_positions_shared": 3748,
+     "n_16grams_shared_unique": 75,
+     "longest_covered_run_bytes": 53,
+     "pct_covered_ge16": 48.8597,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    }
+   }
+  },
+  "v1c_sample": {
+   "bytes": 35948296,
+   "sha256": "3d8b6181cbffb27e360758267e1084197dccad82ae05b868691758e1cf72eed4",
+   "per_window": {
+    "fc2": {
+     "bytes_covered_ge16": 100440,
+     "n_16gram_positions_shared": 40436,
+     "n_16grams_shared_unique": 15136,
+     "longest_covered_run_bytes": 191,
+     "pct_covered_ge16": 50.2723,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    },
+    "fresh_en": {
+     "bytes_covered_ge16": 52239,
+     "n_16gram_positions_shared": 11634,
+     "n_16grams_shared_unique": 10573,
+     "longest_covered_run_bytes": 109,
+     "pct_covered_ge16": 43.6186,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    },
+    "fresh_fr": {
+     "bytes_covered_ge16": 286582,
+     "n_16gram_positions_shared": 55095,
+     "n_16grams_shared_unique": 41336,
+     "longest_covered_run_bytes": 88,
+     "pct_covered_ge16": 26.3236,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    },
+    "test20k_old": {
+     "bytes_covered_ge16": 128766,
+     "n_16gram_positions_shared": 36414,
+     "n_16grams_shared_unique": 23867,
+     "longest_covered_run_bytes": 205,
+     "pct_covered_ge16": 40.2469,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    },
+    "test_v3": {
+     "bytes_covered_ge16": 1316746,
+     "n_16gram_positions_shared": 295112,
+     "n_16grams_shared_unique": 185590,
+     "longest_covered_run_bytes": 129,
+     "pct_covered_ge16": 37.7653,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    },
+    "queries": {
+     "bytes_covered_ge16": 6200,
+     "n_16gram_positions_shared": 900,
+     "n_16grams_shared_unique": 18,
+     "longest_covered_run_bytes": 21,
+     "pct_covered_ge16": 22.3022,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    }
+   }
+  },
+  "v1b_corpus": {
+   "bytes": 520785883,
+   "sha256": "ddf67ffb4e9f340aca7a130639580211e1eaa4b4e850cec93fb19d222f63ddb3",
+   "per_window": {
+    "fc2": {
+     "bytes_covered_ge16": 145183,
+     "n_16gram_positions_shared": 71837,
+     "n_16grams_shared_unique": 38098,
+     "longest_covered_run_bytes": 419,
+     "pct_covered_ge16": 72.6671,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    },
+    "fresh_en": {
+     "bytes_covered_ge16": 94752,
+     "n_16gram_positions_shared": 35873,
+     "n_16grams_shared_unique": 34072,
+     "longest_covered_run_bytes": 338,
+     "pct_covered_ge16": 79.1163,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    },
+    "fresh_fr": {
+     "bytes_covered_ge16": 652322,
+     "n_16gram_positions_shared": 194413,
+     "n_16grams_shared_unique": 168730,
+     "longest_covered_run_bytes": 145,
+     "pct_covered_ge16": 59.9182,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    },
+    "test20k_old": {
+     "bytes_covered_ge16": 237099,
+     "n_16gram_positions_shared": 92792,
+     "n_16grams_shared_unique": 74894,
+     "longest_covered_run_bytes": 266,
+     "pct_covered_ge16": 74.1073,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    },
+    "test_v3": {
+     "bytes_covered_ge16": 2442104,
+     "n_16gram_positions_shared": 866084,
+     "n_16grams_shared_unique": 673440,
+     "longest_covered_run_bytes": 353,
+     "pct_covered_ge16": 70.0415,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    },
+    "queries": {
+     "bytes_covered_ge16": 15133,
+     "n_16gram_positions_shared": 4398,
+     "n_16grams_shared_unique": 88,
+     "longest_covered_run_bytes": 53,
+     "pct_covered_ge16": 54.4353,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    }
+   }
+  },
+  "v1b_sample": {
+   "bytes": 35128951,
+   "sha256": "9ea562e44f708adda470c53144dcae748796d77b98039b5fc52ac962d877370b",
+   "per_window": {
+    "fc2": {
+     "bytes_covered_ge16": 100880,
+     "n_16gram_positions_shared": 40698,
+     "n_16grams_shared_unique": 15264,
+     "longest_covered_run_bytes": 277,
+     "pct_covered_ge16": 50.4925,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    },
+    "fresh_en": {
+     "bytes_covered_ge16": 55107,
+     "n_16gram_positions_shared": 12710,
+     "n_16grams_shared_unique": 11534,
+     "longest_covered_run_bytes": 108,
+     "pct_covered_ge16": 46.0134,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    },
+    "fresh_fr": {
+     "bytes_covered_ge16": 283182,
+     "n_16gram_positions_shared": 55308,
+     "n_16grams_shared_unique": 41593,
+     "longest_covered_run_bytes": 76,
+     "pct_covered_ge16": 26.0113,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    },
+    "test20k_old": {
+     "bytes_covered_ge16": 128263,
+     "n_16gram_positions_shared": 36178,
+     "n_16grams_shared_unique": 23521,
+     "longest_covered_run_bytes": 166,
+     "pct_covered_ge16": 40.0897,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    },
+    "test_v3": {
+     "bytes_covered_ge16": 1189410,
+     "n_16gram_positions_shared": 260934,
+     "n_16grams_shared_unique": 174208,
+     "longest_covered_run_bytes": 105,
+     "pct_covered_ge16": 34.1132,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    },
+    "queries": {
+     "bytes_covered_ge16": 8500,
+     "n_16gram_positions_shared": 1800,
+     "n_16grams_shared_unique": 36,
+     "longest_covered_run_bytes": 36,
+     "pct_covered_ge16": 30.5755,
+     "matches_checker_covered": true,
+     "matches_checker_longest": true,
+     "matches_checker_n16_positions": true
+    }
+   }
+  }
+ },
+ "all_checks_match": true
+}

results/hashing/leak16_baseline_recount/leakcheck_hf.py

@@ -0,0 +1,24 @@
+# Leak check: for each file in hf_ready, longest byte run shared with any eval window (exact, binary search on k).
+import json,sys,os,numpy as np
+from numpy.lib.stride_tricks import sliding_window_view as swv
+R='/workspace/spira_shared/'; J=json.load(open(R+'checker/recount/m8_v1d_eval_prep/leak16_baseline_v1b_v1c.json'))
+D=sys.argv[1]; files=sorted(f for f in os.listdir(D) if f!='SHA256SUMS')
+W=[np.fromfile(R+m['path'],np.uint8) for m in J['windows'].values()]
+def grams(a,k): return np.ascontiguousarray(swv(a,k)).view(f'V{k}').ravel() if len(a)>=k else np.empty(0,f'V{k}')
+def shares(fb,k):
+    q=np.unique(grams(fb,k))
+    for a in W:
+        for s0 in range(0,max(len(a)-k+1,1),1<<18):
+            g=grams(a[s0:s0+(1<<18)+k-1],k); i=np.searchsorted(q,g); i=np.minimum(i,len(q)-1)
+            if len(q) and (q[i]==g).any(): return True
+    return False
+res={}
+for f in files:
+    fb=np.fromfile(os.path.join(D,f),np.uint8); lo,hi=0,min(len(fb),64)
+    if shares(fb,hi): lo=hi
+    while lo<hi:
+        m=(lo+hi+1)//2
+        if shares(fb,m): lo=m
+        else: hi=m-1
+    res[f]={'longest_shared_run_bytes':lo,'shares_16gram':lo>=16}
+print(json.dumps(res,indent=1)); print('LEAKCHECK','PASS' if all(not v['shares_16gram'] for v in res.values()) else 'FAIL')

results/hashing/leak16_baseline_recount/recount_leak16.py

@@ -0,0 +1,58 @@
+# Recount indépendant leak16 (clés exactes 2x8 o, sans hachage, corpus en blocs de 1 Mo). Lecture seule.
+import json,hashlib,sys,resource,time,numpy as np
+R='/workspace/spira_shared/'; T=R+'checker/recount/m8_v1d_eval_prep/leak16_baseline_v1b_v1c.json'
+def sha(p):
+    h=hashlib.sha256()
+    with open(p,'rb') as f:
+        for b in iter(lambda:f.read(1<<20),b''): h.update(b)
+    return h.hexdigest()
+def keys(a):  # a uint8 -> (hi,lo) big-endian u64 of each 16-gram
+    n=len(a)-15; hi=np.zeros(n,np.uint64); lo=np.zeros(n,np.uint64)
+    for i in range(8):
+        hi=(hi<<np.uint64(8))|a[i:i+n].astype(np.uint64); lo=(lo<<np.uint64(8))|a[i+8:i+8+n].astype(np.uint64)
+    return hi,lo
+J=json.load(open(T)); out={'target':T,'target_sha256':sha(T),'windows':{},'sources':{}}
+W={};allh=[];alll=[]
+for w,m in J['windows'].items():
+    p=R+m['path']; s=sha(p); a=np.fromfile(p,np.uint8)
+    out['windows'][w]={'path':p,'sha256':s,'sha_ok':s==m['sha256'],'bytes':len(a),'bytes_ok':len(a)==m['bytes']}
+    h,l=keys(a); W[w]=(h,l); allh.append(h); alll.append(l)
+H=np.concatenate(allh);L=np.concatenate(alll); o=np.lexsort((L,H)); H=H[o];L=L[o]
+k=np.ones(len(H),bool); k[1:]=(H[1:]!=H[:-1])|(L[1:]!=L[:-1]); UH=H[k];UL=L[k]; del H,L,o,k,allh,alll
+idx={w:None for w in W}
+for w,(h,l) in W.items():
+    i=np.searchsorted(UH,h,'left'); r=np.searchsorted(UH,h,'right')
+    while True:  # exact match on lo within hi group
+        bad=UL[np.minimum(i,len(UL)-1)]!=l
+        if not bad.any(): break
+        i[bad]+=1
+    assert (UH[i]==h).all() and (UL[i]==l).all(); idx[w]=i
+def member(h,l):
+    i=np.searchsorted(UH,h,'left'); r=np.searchsorted(UH,h,'right'); f=np.zeros(len(h),bool); g=np.full(len(h),-1)
+    c=np.nonzero(r>i)[0]; ii=i[c].copy(); rr=r[c]
+    while len(c):
+        m=UL[ii]==l[c]; f[c[m]]=True; g[c[m]]=ii[m]
+        keep=(~m)&(ii+1<rr); c=c[keep]; ii=ii[keep]+1; rr=rr[keep]
+    return g[f]
+for s,m in J['sources'].items():
+    t=time.time(); p=m['path']; sh=sha(p); found=np.zeros(len(UH),bool); n=0; prev=b''
+    with open(p,'rb') as f:
+        for b in iter(lambda:f.read(1<<20),b''):
+            a=np.frombuffer(prev+b,np.uint8); n+=len(b); prev=bytes(a[-15:])
+            if len(a)>=16: found[member(*keys(a))]=True
+    pw={}
+    for w,(h,l) in W.items():
+        hit=found[idx[w]]; N=len(hit)+15; d=np.zeros(N+1,np.int32); pos=np.nonzero(hit)[0]
+        np.add.at(d,pos,1); np.add.at(d,pos+16,-1); cov=np.cumsum(d[:N])>0
+        e=np.diff(np.concatenate(([0],cov.astype(np.int8),[0]))); runs=np.nonzero(e==-1)[0]-np.nonzero(e==1)[0]
+        c=m['per_window'][w]; mine={'bytes_covered_ge16':int(cov.sum()),'n_16gram_positions_shared':int(hit.sum()),
+          'n_16grams_shared_unique':int(len(np.unique(idx[w][hit]))),'longest_covered_run_bytes':int(runs.max() if len(runs) else 0)}
+        mine['frac_covered_ge16']=mine['bytes_covered_ge16']/N
+        mine['checker']=c; mine['covered_match']=mine['bytes_covered_ge16']==c['bytes_covered_ge16']
+        mine['longest_match']=mine['longest_covered_run_bytes']==c['longest_covered_run_bytes']
+        mine['n16_match_positions']=mine['n_16gram_positions_shared']==c['n_16grams_shared']
+        mine['n16_match_unique']=mine['n_16grams_shared_unique']==c['n_16grams_shared']; pw[w]=mine
+    out['sources'][s]={'path':p,'sha256':sh,'sha_ok':sh==m['sha256'],'bytes':n,'bytes_ok':n==m['bytes'],'per_window':pw,'seconds':round(time.time()-t,1)}
+    print(s,round(time.time()-t),flush=True)
+out['peak_rss_mb']=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+json.dump(out,open('results.json','w'),indent=1,ensure_ascii=False); print(out['peak_rss_mb'])

results/hashing/leak16_baseline_recount/recount_leak16_stream.py

@@ -0,0 +1,59 @@
+# Recount leak16 en streaming (< 300 Mo) : version de recount_leak16.py (9b870e4a).
+# Exact : clés = 16 octets bruts (dtype V16, ordre memcmp), aucun hachage ; tableau trié unique + searchsorted ;
+# corpus lu en blocs de 256 Kio avec recouvrement de 15 octets ; n_16grams_shared compté en positions.
+# Lecture seule hors de ce dossier. Sortie : argv[1] (défaut results_stream.json).
+import json,hashlib,sys,resource,time,numpy as np
+from numpy.lib.stride_tricks import sliding_window_view as swv
+R='/workspace/spira_shared/'; T=R+'checker/recount/m8_v1d_eval_prep/leak16_baseline_v1b_v1c.json'
+OUT=sys.argv[1] if len(sys.argv)>1 else 'results_stream.json'; BLK=1<<18
+def sha(p):
+    h=hashlib.sha256()
+    with open(p,'rb') as f:
+        for b in iter(lambda:f.read(1<<20),b''): h.update(b)
+    return h.hexdigest()
+def keys(a): return np.ascontiguousarray(swv(a,16)).view('V16').ravel()  # a uint8 -> 16-grammes (copie n*16 o)
+J=json.load(open(T)); out={'target':T,'target_sha256':sha(T),'windows':{},'sources':{}}
+NW={}
+for w,m in J['windows'].items():
+    p=R+m['path']; s=sha(p); NW[w]=n=int(np.fromfile(p,np.uint8).size)
+    out['windows'][w]={'path':p,'sha256':s,'sha_ok':s==m['sha256'],'bytes':n,'bytes_ok':n==m['bytes']}
+U=np.empty(sum(n-15 for n in NW.values()),'V16'); k=0   # un seul tableau préalloué, rempli par blocs
+for w,m in J['windows'].items():
+    a=np.fromfile(R+m['path'],np.uint8)
+    for s0 in range(0,len(a)-15,BLK):
+        q=keys(a[s0:s0+BLK+15]); U[k:k+len(q)]=q; k+=len(q)
+    del a
+assert k==len(U); U.sort()                                 # tri en place (ordre memcmp)
+keep=np.ones(len(U),bool); keep[1:]=U[1:]!=U[:-1]; nu=int(keep.sum())
+U[:nu]=U[keep]; U=U[:nu].copy(); del keep               # dédoublonnage exact
+def lookup(q):  # -> (indices dans U, masque présent), exact octet à octet
+    i=np.searchsorted(U,q); j=np.minimum(i,len(U)-1); f=U[j]==q; return j,f
+idx={}
+for w,m in J['windows'].items():
+    a=np.fromfile(R+m['path'],np.uint8); ii=[]
+    for s0 in range(0,len(a)-15,BLK):
+        j,f=lookup(keys(a[s0:s0+BLK+15])); assert f.all(); ii.append(j.astype(np.int32))
+    idx[w]=np.concatenate(ii); assert len(idx[w])==NW[w]-15
+for s,m in J['sources'].items():
+    t=time.time(); p=m['path']; sh=sha(p); found=np.zeros(len(U),bool); n=0; prev=b''
+    with open(p,'rb') as f:
+        for b in iter(lambda:f.read(BLK),b''):
+            a=np.frombuffer(prev+b,np.uint8); n+=len(b); prev=bytes(a[-15:])
+            if len(a)>=16:
+                j,fm=lookup(keys(a)); found[j[fm]]=True
+    pw={}
+    for w in idx:
+        hit=found[idx[w]]; N=len(hit)+15; d=np.zeros(N+1,np.int32); pos=np.nonzero(hit)[0]
+        np.add.at(d,pos,1); np.add.at(d,pos+16,-1); cov=np.cumsum(d[:N])>0
+        e=np.diff(np.concatenate(([0],cov.astype(np.int8),[0]))); runs=np.nonzero(e==-1)[0]-np.nonzero(e==1)[0]
+        c=m['per_window'][w]; mine={'bytes_covered_ge16':int(cov.sum()),'n_16gram_positions_shared':int(hit.sum()),
+          'n_16grams_shared_unique':int(len(np.unique(idx[w][hit]))),'longest_covered_run_bytes':int(runs.max() if len(runs) else 0)}
+        mine['frac_covered_ge16']=mine['bytes_covered_ge16']/N
+        mine['checker']=c; mine['covered_match']=mine['bytes_covered_ge16']==c['bytes_covered_ge16']
+        mine['longest_match']=mine['longest_covered_run_bytes']==c['longest_covered_run_bytes']
+        mine['n16_match_positions']=mine['n_16gram_positions_shared']==c['n_16grams_shared']
+        mine['n16_match_unique']=mine['n_16grams_shared_unique']==c['n_16grams_shared']; pw[w]=mine
+    out['sources'][s]={'path':p,'sha256':sh,'sha_ok':sh==m['sha256'],'bytes':n,'bytes_ok':n==m['bytes'],'per_window':pw,'seconds':round(time.time()-t,1)}
+    print(s,round(time.time()-t),flush=True)
+out['peak_rss_mb']=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+json.dump(out,open(OUT,'w'),indent=1,ensure_ascii=False); print(out['peak_rss_mb'])

results/hashing/leak16_baseline_recount/results.json

@@ -0,0 +1,523 @@
+{
+ "target": "/workspace/spira_shared/checker/recount/m8_v1d_eval_prep/leak16_baseline_v1b_v1c.json",
+ "target_sha256": "a9ee65d4ad394ada0af487a0cb7eec66073f5440fb405ec19d5df641b5334598",
+ "windows": {
+  "fc2": {
+   "path": "/workspace/spira_shared/broad/freshcode_v2.bin",
+   "sha256": "b052096b0c1e2b301cd79921502f85e61912f209fc0027151bbf82237de8c2e3",
+   "sha_ok": true,
+   "bytes": 199792,
+   "bytes_ok": true
+  },
+  "fresh_en": {
+   "path": "/workspace/spira_shared/spira_m7/windows/fresh_en.bin",
+   "sha256": "67ff20b26ce5e3401cea57c817b1a8417e95f6cd131fd8fb631cd5cd8625331a",
+   "sha_ok": true,
+   "bytes": 119763,
+   "bytes_ok": true
+  },
+  "fresh_fr": {
+   "path": "/workspace/spira_shared/spira_m7/windows/fresh_fr.bin",
+   "sha256": "875f0906a320e3f7b87e446aab646cf49c74786cb1a0db78b54fa4c62e872715",
+   "sha_ok": true,
+   "bytes": 1088688,
+   "bytes_ok": true
+  },
+  "test20k_old": {
+   "path": "/workspace/spira_shared/spira_m7/windows/test20k.bin",
+   "sha256": "ce0526c62048f8e590a96cca5393a87c2e631f194eb4a2e974c584fd443b6a22",
+   "sha_ok": true,
+   "bytes": 319940,
+   "bytes_ok": true
+  },
+  "test_v3": {
+   "path": "/workspace/spira_shared/corpus_v3/test.bin",
+   "sha256": "a6730bbe877e7b28a5529793b27a0b59f16d04db05ea895f0ee6d4f91807ca84",
+   "sha_ok": true,
+   "bytes": 3486653,
+   "bytes_ok": true
+  },
+  "queries": {
+   "path": "/workspace/spira_shared/spira_m7/windows/queries.bin",
+   "sha256": "5aab38d62b2f000df6a913d2574ccd314300e350ccc76e71e16768ed5c73b706",
+   "sha_ok": true,
+   "bytes": 27800,
+   "bytes_ok": true
+  }
+ },
+ "sources": {
+  "v1c_corpus": {
+   "path": "/workspace/spira_shared/hashing_gen/corpus_diverse_v1c/train.bin",
+   "sha256": "33577b70c819af76fe4737319e929f4c097b0c72f020e8b07b4baddbb1823439",
+   "sha_ok": true,
+   "bytes": 503301148,
+   "bytes_ok": true,
+   "per_window": {
+    "fc2": {
+     "bytes_covered_ge16": 145139,
+     "n_16gram_positions_shared": 71796,
+     "n_16grams_shared_unique": 38057,
+     "longest_covered_run_bytes": 419,
+     "frac_covered_ge16": 0.72645050852887,
+     "checker": {
+      "bytes": 199792,
+      "bytes_covered_ge16": 145139,
+      "frac_covered_ge16": 0.72645050852887,
+      "n_16grams_shared": 71796,
+      "longest_covered_run_bytes": 419
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "fresh_en": {
+     "bytes_covered_ge16": 93456,
+     "n_16gram_positions_shared": 34364,
+     "n_16grams_shared_unique": 32608,
+     "longest_covered_run_bytes": 338,
+     "frac_covered_ge16": 0.7803411738182912,
+     "checker": {
+      "bytes": 119763,
+      "bytes_covered_ge16": 93456,
+      "frac_covered_ge16": 0.7803411738182912,
+      "n_16grams_shared": 34364,
+      "longest_covered_run_bytes": 338
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "fresh_fr": {
+     "bytes_covered_ge16": 652302,
+     "n_16gram_positions_shared": 194384,
+     "n_16grams_shared_unique": 168701,
+     "longest_covered_run_bytes": 145,
+     "frac_covered_ge16": 0.5991633966756316,
+     "checker": {
+      "bytes": 1088688,
+      "bytes_covered_ge16": 652302,
+      "frac_covered_ge16": 0.5991633966756316,
+      "n_16grams_shared": 194384,
+      "longest_covered_run_bytes": 145
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "test20k_old": {
+     "bytes_covered_ge16": 235956,
+     "n_16gram_positions_shared": 91729,
+     "n_16grams_shared_unique": 73892,
+     "longest_covered_run_bytes": 266,
+     "frac_covered_ge16": 0.7375007813965119,
+     "checker": {
+      "bytes": 319940,
+      "bytes_covered_ge16": 235956,
+      "frac_covered_ge16": 0.7375007813965119,
+      "n_16grams_shared": 91729,
+      "longest_covered_run_bytes": 266
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "test_v3": {
+     "bytes_covered_ge16": 2433706,
+     "n_16gram_positions_shared": 859632,
+     "n_16grams_shared_unique": 667154,
+     "longest_covered_run_bytes": 353,
+     "frac_covered_ge16": 0.698006368858616,
+     "checker": {
+      "bytes": 3486653,
+      "bytes_covered_ge16": 2433706,
+      "frac_covered_ge16": 0.698006368858616,
+      "n_16grams_shared": 859632,
+      "longest_covered_run_bytes": 353
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "queries": {
+     "bytes_covered_ge16": 13583,
+     "n_16gram_positions_shared": 3748,
+     "n_16grams_shared_unique": 75,
+     "longest_covered_run_bytes": 53,
+     "frac_covered_ge16": 0.4885971223021583,
+     "checker": {
+      "bytes": 27800,
+      "bytes_covered_ge16": 13583,
+      "frac_covered_ge16": 0.4885971223021583,
+      "n_16grams_shared": 3748,
+      "longest_covered_run_bytes": 53
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    }
+   },
+   "seconds": 351.0
+  },
+  "v1c_sample": {
+   "path": "/workspace/spira_shared/spira_m8/v1c/sample/v1c_sample.bin",
+   "sha256": "3d8b6181cbffb27e360758267e1084197dccad82ae05b868691758e1cf72eed4",
+   "sha_ok": true,
+   "bytes": 35948296,
+   "bytes_ok": true,
+   "per_window": {
+    "fc2": {
+     "bytes_covered_ge16": 100440,
+     "n_16gram_positions_shared": 40436,
+     "n_16grams_shared_unique": 15136,
+     "longest_covered_run_bytes": 191,
+     "frac_covered_ge16": 0.5027228317450149,
+     "checker": {
+      "bytes": 199792,
+      "bytes_covered_ge16": 100440,
+      "frac_covered_ge16": 0.5027228317450149,
+      "n_16grams_shared": 40436,
+      "longest_covered_run_bytes": 191
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "fresh_en": {
+     "bytes_covered_ge16": 52239,
+     "n_16gram_positions_shared": 11634,
+     "n_16grams_shared_unique": 10573,
+     "longest_covered_run_bytes": 109,
+     "frac_covered_ge16": 0.4361864682748428,
+     "checker": {
+      "bytes": 119763,
+      "bytes_covered_ge16": 52239,
+      "frac_covered_ge16": 0.4361864682748428,
+      "n_16grams_shared": 11634,
+      "longest_covered_run_bytes": 109
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "fresh_fr": {
+     "bytes_covered_ge16": 286582,
+     "n_16gram_positions_shared": 55095,
+     "n_16grams_shared_unique": 41336,
+     "longest_covered_run_bytes": 88,
+     "frac_covered_ge16": 0.2632361153976162,
+     "checker": {
+      "bytes": 1088688,
+      "bytes_covered_ge16": 286582,
+      "frac_covered_ge16": 0.2632361153976162,
+      "n_16grams_shared": 55095,
+      "longest_covered_run_bytes": 88
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "test20k_old": {
+     "bytes_covered_ge16": 128766,
+     "n_16gram_positions_shared": 36414,
+     "n_16grams_shared_unique": 23867,
+     "longest_covered_run_bytes": 205,
+     "frac_covered_ge16": 0.40246921297743327,
+     "checker": {
+      "bytes": 319940,
+      "bytes_covered_ge16": 128766,
+      "frac_covered_ge16": 0.40246921297743327,
+      "n_16grams_shared": 36414,
+      "longest_covered_run_bytes": 205
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "test_v3": {
+     "bytes_covered_ge16": 1316746,
+     "n_16gram_positions_shared": 295112,
+     "n_16grams_shared_unique": 185590,
+     "longest_covered_run_bytes": 129,
+     "frac_covered_ge16": 0.37765329672898335,
+     "checker": {
+      "bytes": 3486653,
+      "bytes_covered_ge16": 1316746,
+      "frac_covered_ge16": 0.37765329672898335,
+      "n_16grams_shared": 295112,
+      "longest_covered_run_bytes": 129
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "queries": {
+     "bytes_covered_ge16": 6200,
+     "n_16gram_positions_shared": 900,
+     "n_16grams_shared_unique": 18,
+     "longest_covered_run_bytes": 21,
+     "frac_covered_ge16": 0.22302158273381295,
+     "checker": {
+      "bytes": 27800,
+      "bytes_covered_ge16": 6200,
+      "frac_covered_ge16": 0.22302158273381295,
+      "n_16grams_shared": 900,
+      "longest_covered_run_bytes": 21
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    }
+   },
+   "seconds": 22.5
+  },
+  "v1b_corpus": {
+   "path": "/workspace/spira_shared/hashing_gen/corpus_v1b/train.bin",
+   "sha256": "ddf67ffb4e9f340aca7a130639580211e1eaa4b4e850cec93fb19d222f63ddb3",
+   "sha_ok": true,
+   "bytes": 520785883,
+   "bytes_ok": true,
+   "per_window": {
+    "fc2": {
+     "bytes_covered_ge16": 145183,
+     "n_16gram_positions_shared": 71837,
+     "n_16grams_shared_unique": 38098,
+     "longest_covered_run_bytes": 419,
+     "frac_covered_ge16": 0.7266707375670698,
+     "checker": {
+      "bytes": 199792,
+      "bytes_covered_ge16": 145183,
+      "frac_covered_ge16": 0.7266707375670698,
+      "n_16grams_shared": 71837,
+      "longest_covered_run_bytes": 419
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "fresh_en": {
+     "bytes_covered_ge16": 94752,
+     "n_16gram_positions_shared": 35873,
+     "n_16grams_shared_unique": 34072,
+     "longest_covered_run_bytes": 338,
+     "frac_covered_ge16": 0.7911625460284061,
+     "checker": {
+      "bytes": 119763,
+      "bytes_covered_ge16": 94752,
+      "frac_covered_ge16": 0.7911625460284061,
+      "n_16grams_shared": 35873,
+      "longest_covered_run_bytes": 338
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "fresh_fr": {
+     "bytes_covered_ge16": 652322,
+     "n_16gram_positions_shared": 194413,
+     "n_16grams_shared_unique": 168730,
+     "longest_covered_run_bytes": 145,
+     "frac_covered_ge16": 0.5991817674117837,
+     "checker": {
+      "bytes": 1088688,
+      "bytes_covered_ge16": 652322,
+      "frac_covered_ge16": 0.5991817674117837,
+      "n_16grams_shared": 194413,
+      "longest_covered_run_bytes": 145
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "test20k_old": {
+     "bytes_covered_ge16": 237099,
+     "n_16gram_positions_shared": 92792,
+     "n_16grams_shared_unique": 74894,
+     "longest_covered_run_bytes": 266,
+     "frac_covered_ge16": 0.7410733262486716,
+     "checker": {
+      "bytes": 319940,
+      "bytes_covered_ge16": 237099,
+      "frac_covered_ge16": 0.7410733262486716,
+      "n_16grams_shared": 92792,
+      "longest_covered_run_bytes": 266
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "test_v3": {
+     "bytes_covered_ge16": 2442104,
+     "n_16gram_positions_shared": 866084,
+     "n_16grams_shared_unique": 673440,
+     "longest_covered_run_bytes": 353,
+     "frac_covered_ge16": 0.7004149825061455,
+     "checker": {
+      "bytes": 3486653,
+      "bytes_covered_ge16": 2442104,
+      "frac_covered_ge16": 0.7004149825061455,
+      "n_16grams_shared": 866084,
+      "longest_covered_run_bytes": 353
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "queries": {
+     "bytes_covered_ge16": 15133,
+     "n_16gram_positions_shared": 4398,
+     "n_16grams_shared_unique": 88,
+     "longest_covered_run_bytes": 53,
+     "frac_covered_ge16": 0.5443525179856115,
+     "checker": {
+      "bytes": 27800,
+      "bytes_covered_ge16": 15133,
+      "frac_covered_ge16": 0.5443525179856115,
+      "n_16grams_shared": 4398,
+      "longest_covered_run_bytes": 53
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    }
+   },
+   "seconds": 358.2
+  },
+  "v1b_sample": {
+   "path": "/workspace/spira_shared/hashing_gen/m8_v1b/sample/v1b_sample.bin",
+   "sha256": "9ea562e44f708adda470c53144dcae748796d77b98039b5fc52ac962d877370b",
+   "sha_ok": true,
+   "bytes": 35128951,
+   "bytes_ok": true,
+   "per_window": {
+    "fc2": {
+     "bytes_covered_ge16": 100880,
+     "n_16gram_positions_shared": 40698,
+     "n_16grams_shared_unique": 15264,
+     "longest_covered_run_bytes": 277,
+     "frac_covered_ge16": 0.5049251221270121,
+     "checker": {
+      "bytes": 199792,
+      "bytes_covered_ge16": 100880,
+      "frac_covered_ge16": 0.5049251221270121,
+      "n_16grams_shared": 40698,
+      "longest_covered_run_bytes": 277
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "fresh_en": {
+     "bytes_covered_ge16": 55107,
+     "n_16gram_positions_shared": 12710,
+     "n_16grams_shared_unique": 11534,
+     "longest_covered_run_bytes": 108,
+     "frac_covered_ge16": 0.46013376418426394,
+     "checker": {
+      "bytes": 119763,
+      "bytes_covered_ge16": 55107,
+      "frac_covered_ge16": 0.46013376418426394,
+      "n_16grams_shared": 12710,
+      "longest_covered_run_bytes": 108
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "fresh_fr": {
+     "bytes_covered_ge16": 283182,
+     "n_16gram_positions_shared": 55308,
+     "n_16grams_shared_unique": 41593,
+     "longest_covered_run_bytes": 76,
+     "frac_covered_ge16": 0.26011309025175255,
+     "checker": {
+      "bytes": 1088688,
+      "bytes_covered_ge16": 283182,
+      "frac_covered_ge16": 0.26011309025175255,
+      "n_16grams_shared": 55308,
+      "longest_covered_run_bytes": 76
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "test20k_old": {
+     "bytes_covered_ge16": 128263,
+     "n_16gram_positions_shared": 36178,
+     "n_16grams_shared_unique": 23521,
+     "longest_covered_run_bytes": 166,
+     "frac_covered_ge16": 0.40089704319559916,
+     "checker": {
+      "bytes": 319940,
+      "bytes_covered_ge16": 128263,
+      "frac_covered_ge16": 0.40089704319559916,
+      "n_16grams_shared": 36178,
+      "longest_covered_run_bytes": 166
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "test_v3": {
+     "bytes_covered_ge16": 1189410,
+     "n_16gram_positions_shared": 260934,
+     "n_16grams_shared_unique": 174208,
+     "longest_covered_run_bytes": 105,
+     "frac_covered_ge16": 0.34113231227770585,
+     "checker": {
+      "bytes": 3486653,
+      "bytes_covered_ge16": 1189410,
+      "frac_covered_ge16": 0.34113231227770585,
+      "n_16grams_shared": 260934,
+      "longest_covered_run_bytes": 105
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "queries": {
+     "bytes_covered_ge16": 8500,
+     "n_16gram_positions_shared": 1800,
+     "n_16grams_shared_unique": 36,
+     "longest_covered_run_bytes": 36,
+     "frac_covered_ge16": 0.3057553956834532,
+     "checker": {
+      "bytes": 27800,
+      "bytes_covered_ge16": 8500,
+      "frac_covered_ge16": 0.3057553956834532,
+      "n_16grams_shared": 1800,
+      "longest_covered_run_bytes": 36
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    }
+   },
+   "seconds": 22.3
+  }
+ },
+ "peak_rss_mb": 364.83984375
+}

results/hashing/leak16_baseline_recount/results_stream.json

@@ -0,0 +1,523 @@
+{
+ "target": "/workspace/spira_shared/checker/recount/m8_v1d_eval_prep/leak16_baseline_v1b_v1c.json",
+ "target_sha256": "a9ee65d4ad394ada0af487a0cb7eec66073f5440fb405ec19d5df641b5334598",
+ "windows": {
+  "fc2": {
+   "path": "/workspace/spira_shared/broad/freshcode_v2.bin",
+   "sha256": "b052096b0c1e2b301cd79921502f85e61912f209fc0027151bbf82237de8c2e3",
+   "sha_ok": true,
+   "bytes": 199792,
+   "bytes_ok": true
+  },
+  "fresh_en": {
+   "path": "/workspace/spira_shared/spira_m7/windows/fresh_en.bin",
+   "sha256": "67ff20b26ce5e3401cea57c817b1a8417e95f6cd131fd8fb631cd5cd8625331a",
+   "sha_ok": true,
+   "bytes": 119763,
+   "bytes_ok": true
+  },
+  "fresh_fr": {
+   "path": "/workspace/spira_shared/spira_m7/windows/fresh_fr.bin",
+   "sha256": "875f0906a320e3f7b87e446aab646cf49c74786cb1a0db78b54fa4c62e872715",
+   "sha_ok": true,
+   "bytes": 1088688,
+   "bytes_ok": true
+  },
+  "test20k_old": {
+   "path": "/workspace/spira_shared/spira_m7/windows/test20k.bin",
+   "sha256": "ce0526c62048f8e590a96cca5393a87c2e631f194eb4a2e974c584fd443b6a22",
+   "sha_ok": true,
+   "bytes": 319940,
+   "bytes_ok": true
+  },
+  "test_v3": {
+   "path": "/workspace/spira_shared/corpus_v3/test.bin",
+   "sha256": "a6730bbe877e7b28a5529793b27a0b59f16d04db05ea895f0ee6d4f91807ca84",
+   "sha_ok": true,
+   "bytes": 3486653,
+   "bytes_ok": true
+  },
+  "queries": {
+   "path": "/workspace/spira_shared/spira_m7/windows/queries.bin",
+   "sha256": "5aab38d62b2f000df6a913d2574ccd314300e350ccc76e71e16768ed5c73b706",
+   "sha_ok": true,
+   "bytes": 27800,
+   "bytes_ok": true
+  }
+ },
+ "sources": {
+  "v1c_corpus": {
+   "path": "/workspace/spira_shared/hashing_gen/corpus_diverse_v1c/train.bin",
+   "sha256": "33577b70c819af76fe4737319e929f4c097b0c72f020e8b07b4baddbb1823439",
+   "sha_ok": true,
+   "bytes": 503301148,
+   "bytes_ok": true,
+   "per_window": {
+    "fc2": {
+     "bytes_covered_ge16": 145139,
+     "n_16gram_positions_shared": 71796,
+     "n_16grams_shared_unique": 38057,
+     "longest_covered_run_bytes": 419,
+     "frac_covered_ge16": 0.72645050852887,
+     "checker": {
+      "bytes": 199792,
+      "bytes_covered_ge16": 145139,
+      "frac_covered_ge16": 0.72645050852887,
+      "n_16grams_shared": 71796,
+      "longest_covered_run_bytes": 419
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "fresh_en": {
+     "bytes_covered_ge16": 93456,
+     "n_16gram_positions_shared": 34364,
+     "n_16grams_shared_unique": 32608,
+     "longest_covered_run_bytes": 338,
+     "frac_covered_ge16": 0.7803411738182912,
+     "checker": {
+      "bytes": 119763,
+      "bytes_covered_ge16": 93456,
+      "frac_covered_ge16": 0.7803411738182912,
+      "n_16grams_shared": 34364,
+      "longest_covered_run_bytes": 338
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "fresh_fr": {
+     "bytes_covered_ge16": 652302,
+     "n_16gram_positions_shared": 194384,
+     "n_16grams_shared_unique": 168701,
+     "longest_covered_run_bytes": 145,
+     "frac_covered_ge16": 0.5991633966756316,
+     "checker": {
+      "bytes": 1088688,
+      "bytes_covered_ge16": 652302,
+      "frac_covered_ge16": 0.5991633966756316,
+      "n_16grams_shared": 194384,
+      "longest_covered_run_bytes": 145
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "test20k_old": {
+     "bytes_covered_ge16": 235956,
+     "n_16gram_positions_shared": 91729,
+     "n_16grams_shared_unique": 73892,
+     "longest_covered_run_bytes": 266,
+     "frac_covered_ge16": 0.7375007813965119,
+     "checker": {
+      "bytes": 319940,
+      "bytes_covered_ge16": 235956,
+      "frac_covered_ge16": 0.7375007813965119,
+      "n_16grams_shared": 91729,
+      "longest_covered_run_bytes": 266
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "test_v3": {
+     "bytes_covered_ge16": 2433706,
+     "n_16gram_positions_shared": 859632,
+     "n_16grams_shared_unique": 667154,
+     "longest_covered_run_bytes": 353,
+     "frac_covered_ge16": 0.698006368858616,
+     "checker": {
+      "bytes": 3486653,
+      "bytes_covered_ge16": 2433706,
+      "frac_covered_ge16": 0.698006368858616,
+      "n_16grams_shared": 859632,
+      "longest_covered_run_bytes": 353
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "queries": {
+     "bytes_covered_ge16": 13583,
+     "n_16gram_positions_shared": 3748,
+     "n_16grams_shared_unique": 75,
+     "longest_covered_run_bytes": 53,
+     "frac_covered_ge16": 0.4885971223021583,
+     "checker": {
+      "bytes": 27800,
+      "bytes_covered_ge16": 13583,
+      "frac_covered_ge16": 0.4885971223021583,
+      "n_16grams_shared": 3748,
+      "longest_covered_run_bytes": 53
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    }
+   },
+   "seconds": 247.4
+  },
+  "v1c_sample": {
+   "path": "/workspace/spira_shared/spira_m8/v1c/sample/v1c_sample.bin",
+   "sha256": "3d8b6181cbffb27e360758267e1084197dccad82ae05b868691758e1cf72eed4",
+   "sha_ok": true,
+   "bytes": 35948296,
+   "bytes_ok": true,
+   "per_window": {
+    "fc2": {
+     "bytes_covered_ge16": 100440,
+     "n_16gram_positions_shared": 40436,
+     "n_16grams_shared_unique": 15136,
+     "longest_covered_run_bytes": 191,
+     "frac_covered_ge16": 0.5027228317450149,
+     "checker": {
+      "bytes": 199792,
+      "bytes_covered_ge16": 100440,
+      "frac_covered_ge16": 0.5027228317450149,
+      "n_16grams_shared": 40436,
+      "longest_covered_run_bytes": 191
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "fresh_en": {
+     "bytes_covered_ge16": 52239,
+     "n_16gram_positions_shared": 11634,
+     "n_16grams_shared_unique": 10573,
+     "longest_covered_run_bytes": 109,
+     "frac_covered_ge16": 0.4361864682748428,
+     "checker": {
+      "bytes": 119763,
+      "bytes_covered_ge16": 52239,
+      "frac_covered_ge16": 0.4361864682748428,
+      "n_16grams_shared": 11634,
+      "longest_covered_run_bytes": 109
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "fresh_fr": {
+     "bytes_covered_ge16": 286582,
+     "n_16gram_positions_shared": 55095,
+     "n_16grams_shared_unique": 41336,
+     "longest_covered_run_bytes": 88,
+     "frac_covered_ge16": 0.2632361153976162,
+     "checker": {
+      "bytes": 1088688,
+      "bytes_covered_ge16": 286582,
+      "frac_covered_ge16": 0.2632361153976162,
+      "n_16grams_shared": 55095,
+      "longest_covered_run_bytes": 88
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "test20k_old": {
+     "bytes_covered_ge16": 128766,
+     "n_16gram_positions_shared": 36414,
+     "n_16grams_shared_unique": 23867,
+     "longest_covered_run_bytes": 205,
+     "frac_covered_ge16": 0.40246921297743327,
+     "checker": {
+      "bytes": 319940,
+      "bytes_covered_ge16": 128766,
+      "frac_covered_ge16": 0.40246921297743327,
+      "n_16grams_shared": 36414,
+      "longest_covered_run_bytes": 205
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "test_v3": {
+     "bytes_covered_ge16": 1316746,
+     "n_16gram_positions_shared": 295112,
+     "n_16grams_shared_unique": 185590,
+     "longest_covered_run_bytes": 129,
+     "frac_covered_ge16": 0.37765329672898335,
+     "checker": {
+      "bytes": 3486653,
+      "bytes_covered_ge16": 1316746,
+      "frac_covered_ge16": 0.37765329672898335,
+      "n_16grams_shared": 295112,
+      "longest_covered_run_bytes": 129
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "queries": {
+     "bytes_covered_ge16": 6200,
+     "n_16gram_positions_shared": 900,
+     "n_16grams_shared_unique": 18,
+     "longest_covered_run_bytes": 21,
+     "frac_covered_ge16": 0.22302158273381295,
+     "checker": {
+      "bytes": 27800,
+      "bytes_covered_ge16": 6200,
+      "frac_covered_ge16": 0.22302158273381295,
+      "n_16grams_shared": 900,
+      "longest_covered_run_bytes": 21
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    }
+   },
+   "seconds": 17.9
+  },
+  "v1b_corpus": {
+   "path": "/workspace/spira_shared/hashing_gen/corpus_v1b/train.bin",
+   "sha256": "ddf67ffb4e9f340aca7a130639580211e1eaa4b4e850cec93fb19d222f63ddb3",
+   "sha_ok": true,
+   "bytes": 520785883,
+   "bytes_ok": true,
+   "per_window": {
+    "fc2": {
+     "bytes_covered_ge16": 145183,
+     "n_16gram_positions_shared": 71837,
+     "n_16grams_shared_unique": 38098,
+     "longest_covered_run_bytes": 419,
+     "frac_covered_ge16": 0.7266707375670698,
+     "checker": {
+      "bytes": 199792,
+      "bytes_covered_ge16": 145183,
+      "frac_covered_ge16": 0.7266707375670698,
+      "n_16grams_shared": 71837,
+      "longest_covered_run_bytes": 419
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "fresh_en": {
+     "bytes_covered_ge16": 94752,
+     "n_16gram_positions_shared": 35873,
+     "n_16grams_shared_unique": 34072,
+     "longest_covered_run_bytes": 338,
+     "frac_covered_ge16": 0.7911625460284061,
+     "checker": {
+      "bytes": 119763,
+      "bytes_covered_ge16": 94752,
+      "frac_covered_ge16": 0.7911625460284061,
+      "n_16grams_shared": 35873,
+      "longest_covered_run_bytes": 338
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "fresh_fr": {
+     "bytes_covered_ge16": 652322,
+     "n_16gram_positions_shared": 194413,
+     "n_16grams_shared_unique": 168730,
+     "longest_covered_run_bytes": 145,
+     "frac_covered_ge16": 0.5991817674117837,
+     "checker": {
+      "bytes": 1088688,
+      "bytes_covered_ge16": 652322,
+      "frac_covered_ge16": 0.5991817674117837,
+      "n_16grams_shared": 194413,
+      "longest_covered_run_bytes": 145
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "test20k_old": {
+     "bytes_covered_ge16": 237099,
+     "n_16gram_positions_shared": 92792,
+     "n_16grams_shared_unique": 74894,
+     "longest_covered_run_bytes": 266,
+     "frac_covered_ge16": 0.7410733262486716,
+     "checker": {
+      "bytes": 319940,
+      "bytes_covered_ge16": 237099,
+      "frac_covered_ge16": 0.7410733262486716,
+      "n_16grams_shared": 92792,
+      "longest_covered_run_bytes": 266
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "test_v3": {
+     "bytes_covered_ge16": 2442104,
+     "n_16gram_positions_shared": 866084,
+     "n_16grams_shared_unique": 673440,
+     "longest_covered_run_bytes": 353,
+     "frac_covered_ge16": 0.7004149825061455,
+     "checker": {
+      "bytes": 3486653,
+      "bytes_covered_ge16": 2442104,
+      "frac_covered_ge16": 0.7004149825061455,
+      "n_16grams_shared": 866084,
+      "longest_covered_run_bytes": 353
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "queries": {
+     "bytes_covered_ge16": 15133,
+     "n_16gram_positions_shared": 4398,
+     "n_16grams_shared_unique": 88,
+     "longest_covered_run_bytes": 53,
+     "frac_covered_ge16": 0.5443525179856115,
+     "checker": {
+      "bytes": 27800,
+      "bytes_covered_ge16": 15133,
+      "frac_covered_ge16": 0.5443525179856115,
+      "n_16grams_shared": 4398,
+      "longest_covered_run_bytes": 53
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    }
+   },
+   "seconds": 266.2
+  },
+  "v1b_sample": {
+   "path": "/workspace/spira_shared/hashing_gen/m8_v1b/sample/v1b_sample.bin",
+   "sha256": "9ea562e44f708adda470c53144dcae748796d77b98039b5fc52ac962d877370b",
+   "sha_ok": true,
+   "bytes": 35128951,
+   "bytes_ok": true,
+   "per_window": {
+    "fc2": {
+     "bytes_covered_ge16": 100880,
+     "n_16gram_positions_shared": 40698,
+     "n_16grams_shared_unique": 15264,
+     "longest_covered_run_bytes": 277,
+     "frac_covered_ge16": 0.5049251221270121,
+     "checker": {
+      "bytes": 199792,
+      "bytes_covered_ge16": 100880,
+      "frac_covered_ge16": 0.5049251221270121,
+      "n_16grams_shared": 40698,
+      "longest_covered_run_bytes": 277
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "fresh_en": {
+     "bytes_covered_ge16": 55107,
+     "n_16gram_positions_shared": 12710,
+     "n_16grams_shared_unique": 11534,
+     "longest_covered_run_bytes": 108,
+     "frac_covered_ge16": 0.46013376418426394,
+     "checker": {
+      "bytes": 119763,
+      "bytes_covered_ge16": 55107,
+      "frac_covered_ge16": 0.46013376418426394,
+      "n_16grams_shared": 12710,
+      "longest_covered_run_bytes": 108
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "fresh_fr": {
+     "bytes_covered_ge16": 283182,
+     "n_16gram_positions_shared": 55308,
+     "n_16grams_shared_unique": 41593,
+     "longest_covered_run_bytes": 76,
+     "frac_covered_ge16": 0.26011309025175255,
+     "checker": {
+      "bytes": 1088688,
+      "bytes_covered_ge16": 283182,
+      "frac_covered_ge16": 0.26011309025175255,
+      "n_16grams_shared": 55308,
+      "longest_covered_run_bytes": 76
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "test20k_old": {
+     "bytes_covered_ge16": 128263,
+     "n_16gram_positions_shared": 36178,
+     "n_16grams_shared_unique": 23521,
+     "longest_covered_run_bytes": 166,
+     "frac_covered_ge16": 0.40089704319559916,
+     "checker": {
+      "bytes": 319940,
+      "bytes_covered_ge16": 128263,
+      "frac_covered_ge16": 0.40089704319559916,
+      "n_16grams_shared": 36178,
+      "longest_covered_run_bytes": 166
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "test_v3": {
+     "bytes_covered_ge16": 1189410,
+     "n_16gram_positions_shared": 260934,
+     "n_16grams_shared_unique": 174208,
+     "longest_covered_run_bytes": 105,
+     "frac_covered_ge16": 0.34113231227770585,
+     "checker": {
+      "bytes": 3486653,
+      "bytes_covered_ge16": 1189410,
+      "frac_covered_ge16": 0.34113231227770585,
+      "n_16grams_shared": 260934,
+      "longest_covered_run_bytes": 105
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    },
+    "queries": {
+     "bytes_covered_ge16": 8500,
+     "n_16gram_positions_shared": 1800,
+     "n_16grams_shared_unique": 36,
+     "longest_covered_run_bytes": 36,
+     "frac_covered_ge16": 0.3057553956834532,
+     "checker": {
+      "bytes": 27800,
+      "bytes_covered_ge16": 8500,
+      "frac_covered_ge16": 0.3057553956834532,
+      "n_16grams_shared": 1800,
+      "longest_covered_run_bytes": 36
+     },
+     "covered_match": true,
+     "longest_match": true,
+     "n16_match_positions": true,
+     "n16_match_unique": false
+    }
+   },
+   "seconds": 18.5
+  }
+ },
+ "peak_rss_mb": 239.453125
+}

results/hashing/leak16_reconcile_r3/REPORT.md

@@ -0,0 +1,27 @@
+# Recompte indépendant leak16 sur hf_ready_run_iii_r3 (SPIRA Hashing)
+Référence Checker : checker/recount/leak16_reconcile_r2/REPORT.md sha 9d7d8383… (lue seulement). **Checker n'a publié aucun chiffre pour r3** (son rapport porte sur r2) : pas de MATCH/MISMATCH possible sur r3, seuls nos chiffres sont donnés.
+Outil : recount.py (octets bruts, sans normalisation, plus long run exact par recherche dichotomique, fenêtres lues en memmap par blocs de 256 Ko, sha de chaque fenêtre vérifié contre ref_8w.json = copie de ref_checker_8w.json 01cb94f6…). Pic RSS 86,8 Mo.
+
+## r3 (plus long run partagé, octets)
+| fichier | sha256 | 6 fenêtres | 8 fenêtres | ≥16 ? |
+|:-:|:-:|:-:|:-:|:-:|
+| ADDENDUM.md | 581e7c2f… | 12 | 12 | non |
+| README.md | 54461b93… | 14 | 14 | non |
+| REPORT.md | fe293479… | 14 | 14 | non |
+| res.json | 23751f0b… | 6 | 6 | non |
+| res_add.json | 63986056… | 5 | 5 | non |
+| rss.py | 4dfb1c2e… | 13 | 13 | non |
+r3 : PASS strict (max 14 o) avec 6 et 8 fenêtres. Comparaison Checker r3 : NON PUBLIÉE.
+
+## Auto-contrôle de l'outil sur r2 contre le tableau Checker 9d7d8383
+| fichier | nous 6 | Checker 6 | nous 8 | Checker 8 | statut |
+|:-:|:-:|:-:|:-:|:-:|:-:|
+| ADDENDUM.md | 12 | 12 | 32 | 32 | MATCH |
+| README.md | 12 | 12 | 12 | 12 | MATCH |
+| REPORT.md | 13 | 13 | 32 | 32 | MATCH |
+| res.json | 6 | 6 | 6 | 6 | MATCH |
+| res_add.json | 5 | 5 | 5 | 5 | MATCH |
+| rss.py | 13 | 13 | 13 | 13 | MATCH |
+| runner.sh | 11 | 11 | 17 | 17 | MATCH |
+7/7 MATCH, écart 0 o (colonnes « Hashing réf. 6 » et « Hashing réf. 8 » du rapport Checker).
+Sorties : recount_r3.json, selfcheck_r2.json. Rien poussé.

results/hashing/leak16_reconcile_r3/SHA256SUMS

@@ -0,0 +1,5 @@
+41b6bd9eceff955a366c6bd49aa384470e184bde87cc6c509d21c531c6a6c927  REPORT.md
+9454d2178cec2f56e28c6c686d8128ba14ae6bcf954e249e26000dc1525aa083  recount.py
+a94c0833168ea8cf74b7bcc023300b6ad7d66aa564c9fbeac4f8289e91612061  recount_r3.json
+01cb94f62f931a423fbbf190f2ed10d30e5d306069a164c24cbce6d6768fb113  ref_8w.json
+7cb45f9e469e4779993b224b4f75cd73679c3ca324fd85d9741c511b93081a90  selfcheck_r2.json

results/hashing/leak16_reconcile_r3/recount.py

@@ -0,0 +1,35 @@
+# Recompte indépendant leak16 : plus long run d'octets bruts partagé (fichier x fenêtres), streaming par blocs de 256 Ko via memmap.
+import json,sys,os,hashlib,resource,numpy as np
+from numpy.lib.stride_tricks import sliding_window_view as swv
+S='/workspace/spira_shared/'; D=sys.argv[1]; ref=json.load(open(sys.argv[2]))['windows']
+def wp(m): p=m['path']; return p if p.startswith('/') else S+p
+W={}
+for k,m in ref.items():
+    h=hashlib.sha256(); a=np.memmap(wp(m),np.uint8,'r')
+    for s in range(0,len(a),1<<20): h.update(a[s:s+(1<<20)].tobytes())
+    assert h.hexdigest()==m['sha256'],k; W[k]=a
+def g(a,k): return np.ascontiguousarray(swv(a,k)).view(f'V{k}').ravel() if len(a)>=k else np.empty(0,f'V{k}')
+def hit(fb,k,names):
+    q=np.unique(g(fb,k)); out=[]
+    if not len(q): return out
+    for n in names:
+        a=W[n]
+        for s in range(0,max(len(a)-k+1,1),1<<18):
+            x=g(np.asarray(a[s:s+(1<<18)+k-1]),k); i=np.minimum(np.searchsorted(q,x),len(q)-1)
+            if (q[i]==x).any(): out.append(n); break
+    return out
+def longest(fb,names):
+    lo,hi=0,min(len(fb),128)
+    while lo<hi:
+        m=(lo+hi+1)//2
+        if hit(fb,m,names): lo=m
+        else: hi=m-1
+    return lo,(hit(fb,lo,names) if lo else [])
+six=['fc2','fresh_en','fresh_fr','test20k_old','test_v3','queries']; eight=six+['bg1_test','neither_code']
+res={}
+for f in sorted(os.listdir(D)):
+    if f=='SHA256SUMS': continue
+    b=open(os.path.join(D,f),'rb').read(); fb=np.frombuffer(b,np.uint8)
+    l6,w6=longest(fb,six); l8,w8=longest(fb,eight)
+    res[f]={'sha256':hashlib.sha256(b).hexdigest(),'max6':l6,'win6':w6,'max8':l8,'win8':w8}
+print(json.dumps({'dir':D,'files':res,'peak_rss_mb':resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024},indent=1))

results/hashing/leak16_reconcile_r3/recount_r3.json

@@ -0,0 +1,90 @@
+{
+ "dir": "/workspace/spira_shared/hashing_gen/precal_ratio_ckpt/hf_ready_run_iii_r3",
+ "files": {
+  "ADDENDUM.md": {
+   "sha256": "581e7c2ffd9ddda09253b4e879b1fac36e7c40e6879d8ed3330f8f4e344764e4",
+   "max6": 12,
+   "win6": [
+    "fresh_fr",
+    "test_v3"
+   ],
+   "max8": 12,
+   "win8": [
+    "fresh_fr",
+    "test_v3"
+   ]
+  },
+  "README.md": {
+   "sha256": "54461b939713662e96ba6394ebbe705233b9780806fb05207c9cea034e7e98ed",
+   "max6": 14,
+   "win6": [
+    "fresh_fr"
+   ],
+   "max8": 14,
+   "win8": [
+    "fresh_fr"
+   ]
+  },
+  "REPORT.md": {
+   "sha256": "fe2934795f726ddd8113e72267c52bfd6252128850a5c0f10cbfce9676e83576",
+   "max6": 14,
+   "win6": [
+    "fresh_fr"
+   ],
+   "max8": 14,
+   "win8": [
+    "fresh_fr"
+   ]
+  },
+  "res.json": {
+   "sha256": "23751f0b10d4568549f93c07214a36249e5950cddf07bd6f64049864d355c939",
+   "max6": 6,
+   "win6": [
+    "test20k_old",
+    "test_v3"
+   ],
+   "max8": 6,
+   "win8": [
+    "test20k_old",
+    "test_v3",
+    "bg1_test",
+    "neither_code"
+   ]
+  },
+  "res_add.json": {
+   "sha256": "6398605605d9d5c3950c74b055b7737befc91f0e9f0006a8ef718c300700d4eb",
+   "max6": 5,
+   "win6": [
+    "fc2",
+    "fresh_en",
+    "fresh_fr",
+    "test20k_old",
+    "test_v3"
+   ],
+   "max8": 5,
+   "win8": [
+    "fc2",
+    "fresh_en",
+    "fresh_fr",
+    "test20k_old",
+    "test_v3",
+    "bg1_test",
+    "neither_code"
+   ]
+  },
+  "rss.py": {
+   "sha256": "4dfb1c2ecf0264cad4629b7a6c684fd49b709bfffc244032aa877b025bc1bab7",
+   "max6": 13,
+   "win6": [
+    "test20k_old"
+   ],
+   "max8": 13,
+   "win8": [
+    "test20k_old",
+    "bg1_test",
+    "neither_code"
+   ]
+  }
+ },
+ "peak_rss_mb": 86.7890625
+}

results/hashing/leak16_reconcile_r3/ref_8w.json

@@ -0,0 +1,45 @@
+{
+ "windows": {
+  "fc2": {
+   "path": "broad/freshcode_v2.bin",
+   "bytes": 199792,
+   "sha256": "b052096b0c1e2b301cd79921502f85e61912f209fc0027151bbf82237de8c2e3"
+  },
+  "fresh_en": {
+   "path": "spira_m7/windows/fresh_en.bin",
+   "bytes": 119763,
+   "sha256": "67ff20b26ce5e3401cea57c817b1a8417e95f6cd131fd8fb631cd5cd8625331a"
+  },
+  "fresh_fr": {
+   "path": "spira_m7/windows/fresh_fr.bin",
+   "bytes": 1088688,
+   "sha256": "875f0906a320e3f7b87e446aab646cf49c74786cb1a0db78b54fa4c62e872715"
+  },
+  "test20k_old": {
+   "path": "spira_m7/windows/test20k.bin",
+   "bytes": 319940,
+   "sha256": "ce0526c62048f8e590a96cca5393a87c2e631f194eb4a2e974c584fd443b6a22"
+  },
+  "test_v3": {
+   "path": "corpus_v3/test.bin",
+   "bytes": 3486653,
+   "sha256": "a6730bbe877e7b28a5529793b27a0b59f16d04db05ea895f0ee6d4f91807ca84"
+  },
+  "queries": {
+   "path": "spira_m7/windows/queries.bin",
+   "bytes": 27800,
+   "sha256": "5aab38d62b2f000df6a913d2574ccd314300e350ccc76e71e16768ed5c73b706"
+  },
+  "bg1_test": {
+   "path": "broad/broad_gutenberg_v1/test.bin",
+   "bytes": 894718,
+   "sha256": "e89725bb94d9f167ffad1f68ae372343a14a1957ae0a1f0a67167aa974ec75a9"
+  },
+  "neither_code": {
+   "path": "checker/recount/leak16_reconcile_r2/neither_code_concat.bin",
+   "bytes": 546020,
+   "sha256": "9d07d5ed054b6fc1a327a8f08fceb9ff4f5a5f5b3525c1698b0e52454473acc6",
+   "source": "concat tri\u00e9 /workspace/kahnn/data/code/*"
+  }
+ }
+}

results/hashing/leak16_reconcile_r3/selfcheck_r2.json

@@ -0,0 +1,102 @@
+{
+ "dir": "/workspace/spira_shared/hashing_gen/precal_ratio_ckpt/hf_ready_run_iii_r2",
+ "files": {
+  "ADDENDUM.md": {
+   "sha256": "33041f014699ad988cb62c82c557b98d5ac3894be7d60f9869f1acd5a217f2a1",
+   "max6": 12,
+   "win6": [
+    "fresh_fr"
+   ],
+   "max8": 32,
+   "win8": [
+    "neither_code"
+   ]
+  },
+  "README.md": {
+   "sha256": "4735c694807ac56a5a58dac022fd543e5c85ba7a7825041749eb0a586829f977",
+   "max6": 12,
+   "win6": [
+    "fresh_fr",
+    "test_v3"
+   ],
+   "max8": 12,
+   "win8": [
+    "fresh_fr",
+    "test_v3"
+   ]
+  },
+  "REPORT.md": {
+   "sha256": "22086158e43e4dd3c430fbc361ba7db072fb5ed71a5e9019c308788257a02925",
+   "max6": 13,
+   "win6": [
+    "fresh_fr",
+    "test_v3"
+   ],
+   "max8": 32,
+   "win8": [
+    "neither_code"
+   ]
+  },
+  "res.json": {
+   "sha256": "23751f0b10d4568549f93c07214a36249e5950cddf07bd6f64049864d355c939",
+   "max6": 6,
+   "win6": [
+    "test20k_old",
+    "test_v3"
+   ],
+   "max8": 6,
+   "win8": [
+    "test20k_old",
+    "test_v3",
+    "bg1_test",
+    "neither_code"
+   ]
+  },
+  "res_add.json": {
+   "sha256": "6398605605d9d5c3950c74b055b7737befc91f0e9f0006a8ef718c300700d4eb",
+   "max6": 5,
+   "win6": [
+    "fc2",
+    "fresh_en",
+    "fresh_fr",
+    "test20k_old",
+    "test_v3"
+   ],
+   "max8": 5,
+   "win8": [
+    "fc2",
+    "fresh_en",
+    "fresh_fr",
+    "test20k_old",
+    "test_v3",
+    "bg1_test",
+    "neither_code"
+   ]
+  },
+  "rss.py": {
+   "sha256": "4dfb1c2ecf0264cad4629b7a6c684fd49b709bfffc244032aa877b025bc1bab7",
+   "max6": 13,
+   "win6": [
+    "test20k_old"
+   ],
+   "max8": 13,
+   "win8": [
+    "test20k_old",
+    "bg1_test",
+    "neither_code"
+   ]
+  },
+  "runner.sh": {
+   "sha256": "25854d01fa5b2793d75652d5a67753cda8d03160b973c005ce1b8bd82352a7ab",
+   "max6": 11,
+   "win6": [
+    "fc2"
+   ],
+   "max8": 17,
+   "win8": [
+    "neither_code"
+   ]
+  }
+ },
+ "peak_rss_mb": 86.71875
+}

results/hashing/lever1/STATUS

@@ -0,0 +1,2 @@
+2026-10-23 21:22 Hashing lever1: verification flux broad_gutenberg_v1 en cours (delta vs broad_local_v1)
+2026-10-23 21:32 Hashing lever1: verif OK delta=0 sur 8 fenetres (abs INVALID herite base); manifeste ecrit; M8 = gate Checker 661153 (attend fin 628595), pas de doublon

results/hashing/lever1/manifest_broad_gutenberg_v1.json

@@ -0,0 +1,126 @@
+{
+ "variant": "broad_gutenberg_v1",
+ "author": "SPIRA Hashing (independent; not inside broad/)",
+ "corpus_dir": "/workspace/spira_shared/broad/broad_gutenberg_v1",
+ "construction": "train.bin = broad_local_v1/train.bin + b'\\n\\n' + corpus_gutenberg_v2/train.bin ; test.bin et oneshot.bin = copies de broad_local_v1 (test20k / fresh_en inchangés)",
+ "sources": {
+  "broad_local_v1": {
+   "train_sha256": "eda07017a87f2b7bbd55c5c2ea23032bf70a38ae7ad75167523228738f2391a1",
+   "composition": {
+    "en": 28654965,
+    "fr": 214337,
+    "code": 6860045
+   }
+  },
+  "corpus_gutenberg_v2": {
+   "train_sha256": "75afd55a4e800715d6870ec88ed12e5c01a42b6caa53243ad267a35c55525fb3",
+   "n_books": 568,
+   "en_bytes": 186503118,
+   "fr_bytes": 69470991,
+   "source": "Project Gutenberg via aleph.pglaf.org (approuvé 23/10 16 h 04)"
+  }
+ },
+ "files": {
+  "train": {
+   "bytes": 290810502,
+   "sha256": "5af6398fc853bce270e430f8336fc2b136318c2e762ead19e92f5d180de083f2"
+  },
+  "test": {
+   "bytes": 894718,
+   "sha256": "e89725bb94d9f167ffad1f68ae372343a14a1957ae0a1f0a67167aa974ec75a9"
+  },
+  "oneshot": {
+   "bytes": 119763,
+   "sha256": "67ff20b26ce5e3401cea57c817b1a8417e95f6cd131fd8fb631cd5cd8625331a"
+  }
+ },
+ "base_prefix_check": {
+  "dir": "/workspace/spira_shared/broad/broad_local_v1",
+  "bytes": 34835257,
+  "sha256": "eda07017a87f2b7bbd55c5c2ea23032bf70a38ae7ad75167523228738f2391a1",
+  "is_exact_prefix": true
+ },
+ "verification": {
+  "script": "hashing_gen/lever1/verify_v3_chunked.py (verify_v3 semantics, streamed; sha256 d6d1905cca4c561b4a22d5bdaae939b8337696106dce71eadf9d0343f11b37d4)",
+  "k": 64,
+  "overlap64": {
+   "fresh_en": {
+    "window_bytes": 119763,
+    "n_64grams": 119700,
+    "hash_hits": 0,
+    "confirmed_positions": 0,
+    "confirmed_in_base_prefix": 0,
+    "delta_added_part": 0,
+    "delta_positions_first20": []
+   },
+   "fresh_fr": {
+    "window_bytes": 1088688,
+    "n_64grams": 1088625,
+    "hash_hits": 0,
+    "confirmed_positions": 0,
+    "confirmed_in_base_prefix": 0,
+    "delta_added_part": 0,
+    "delta_positions_first20": []
+   },
+   "neither_code": {
+    "window_bytes": 546036,
+    "n_64grams": 545973,
+    "hash_hits": 545558,
+    "confirmed_positions": 545558,
+    "confirmed_in_base_prefix": 545558,
+    "delta_added_part": 0,
+    "delta_positions_first20": []
+   },
+   "test20k": {
+    "window_bytes": 319940,
+    "n_64grams": 319877,
+    "hash_hits": 1405,
+    "confirmed_positions": 1405,
+    "confirmed_in_base_prefix": 1405,
+    "delta_added_part": 0,
+    "delta_positions_first20": []
+   },
+   "queries": {
+    "window_bytes": 27800,
+    "n_64grams": 27737,
+    "hash_hits": 0,
+    "confirmed_positions": 0,
+    "confirmed_in_base_prefix": 0,
+    "delta_added_part": 0,
+    "delta_positions_first20": []
+   },
+   "fc2": {
+    "window_bytes": 199792,
+    "n_64grams": 199729,
+    "hash_hits": 0,
+    "confirmed_positions": 0,
+    "confirmed_in_base_prefix": 0,
+    "delta_added_part": 0,
+    "delta_positions_first20": []
+   },
+   "corpus_test": {
+    "window_bytes": 894718,
+    "n_64grams": 894655,
+    "hash_hits": 33766,
+    "confirmed_positions": 33766,
+    "confirmed_in_base_prefix": 33766,
+    "delta_added_part": 0,
+    "delta_positions_first20": []
+   },
+   "corpus_oneshot": {
+    "window_bytes": 119763,
+    "n_64grams": 119700,
+    "hash_hits": 0,
+    "confirmed_positions": 0,
+    "confirmed_in_base_prefix": 0,
+    "delta_added_part": 0,
+    "delta_positions_first20": []
+   }
+  },
+  "valid_zero_overlap_absolute": false,
+  "delta_zero_all_windows": true,
+  "delta_zero_excl_neither_code": true,
+  "note": "Absolute overlap all inherited from broad_local_v1 prefix: neither_code 545558/545973, test20k 1405/319877, corpus_test 33766/894655. Gutenberg part adds 0 on every window."
+ },
+ "checker_manifest_sha256": "5ab579c5d8163a51e2b6dc85c28c2faf49ef4656e6e74d84421c76d9a635fd4c"
+}

results/hashing/lever1/verify_bg1.json

@@ -0,0 +1,101 @@
+{
+ "corpus": "/workspace/spira_shared/broad/broad_gutenberg_v1",
+ "method": "verify_v3 semantics, train streamed in 4M-position chunks",
+ "files": {
+  "train": {
+   "bytes": 290810502,
+   "sha256": "5af6398fc853bce270e430f8336fc2b136318c2e762ead19e92f5d180de083f2"
+  },
+  "test": {
+   "bytes": 894718,
+   "sha256": "e89725bb94d9f167ffad1f68ae372343a14a1957ae0a1f0a67167aa974ec75a9"
+  },
+  "oneshot": {
+   "bytes": 119763,
+   "sha256": "67ff20b26ce5e3401cea57c817b1a8417e95f6cd131fd8fb631cd5cd8625331a"
+  }
+ },
+ "overlap64": {
+  "fresh_en": {
+   "window_bytes": 119763,
+   "n_64grams": 119700,
+   "hash_hits": 0,
+   "confirmed_positions": 0,
+   "confirmed_in_base_prefix": 0,
+   "delta_added_part": 0,
+   "delta_positions_first20": []
+  },
+  "fresh_fr": {
+   "window_bytes": 1088688,
+   "n_64grams": 1088625,
+   "hash_hits": 0,
+   "confirmed_positions": 0,
+   "confirmed_in_base_prefix": 0,
+   "delta_added_part": 0,
+   "delta_positions_first20": []
+  },
+  "neither_code": {
+   "window_bytes": 546036,
+   "n_64grams": 545973,
+   "hash_hits": 545558,
+   "confirmed_positions": 545558,
+   "confirmed_in_base_prefix": 545558,
+   "delta_added_part": 0,
+   "delta_positions_first20": []
+  },
+  "test20k": {
+   "window_bytes": 319940,
+   "n_64grams": 319877,
+   "hash_hits": 1405,
+   "confirmed_positions": 1405,
+   "confirmed_in_base_prefix": 1405,
+   "delta_added_part": 0,
+   "delta_positions_first20": []
+  },
+  "queries": {
+   "window_bytes": 27800,
+   "n_64grams": 27737,
+   "hash_hits": 0,
+   "confirmed_positions": 0,
+   "confirmed_in_base_prefix": 0,
+   "delta_added_part": 0,
+   "delta_positions_first20": []
+  },
+  "fc2": {
+   "window_bytes": 199792,
+   "n_64grams": 199729,
+   "hash_hits": 0,
+   "confirmed_positions": 0,
+   "confirmed_in_base_prefix": 0,
+   "delta_added_part": 0,
+   "delta_positions_first20": []
+  },
+  "corpus_test": {
+   "window_bytes": 894718,
+   "n_64grams": 894655,
+   "hash_hits": 33766,
+   "confirmed_positions": 33766,
+   "confirmed_in_base_prefix": 33766,
+   "delta_added_part": 0,
+   "delta_positions_first20": []
+  },
+  "corpus_oneshot": {
+   "window_bytes": 119763,
+   "n_64grams": 119700,
+   "hash_hits": 0,
+   "confirmed_positions": 0,
+   "confirmed_in_base_prefix": 0,
+   "delta_added_part": 0,
+   "delta_positions_first20": []
+  }
+ },
+ "base": {
+  "dir": "/workspace/spira_shared/broad/broad_local_v1",
+  "bytes": 34835257,
+  "sha256": "eda07017a87f2b7bbd55c5c2ea23032bf70a38ae7ad75167523228738f2391a1",
+  "is_exact_prefix": true
+ },
+ "valid_zero_overlap": false,
+ "delta_zero_all_windows": true,
+ "delta_zero_excl_neither_code": true
+}

results/hashing/lever1/verify_v3.py

@@ -0,0 +1,37 @@
+"""SPIRA Hashing - verification independante (lecture seule) d'un corpus: tailles, sha256, overlap 64 o.
+Usage: verify_v3.py CORPUS_DIR OUT_JSON"""
+import sys, os, json, hashlib, numpy as np
+sys.path.insert(0, "/workspace/spira_shared/checker")
+C, OUT = sys.argv[1:3]; K = 64; P = np.uint64(1099511628211)
+def hashes(b):
+    a = np.frombuffer(b, np.uint8).astype(np.uint64); n = len(a) - K + 1
+    if n <= 0: return np.zeros(0, np.uint64)
+    h = np.zeros(n, np.uint64)
+    with np.errstate(over="ignore"):
+        for j in range(K): h = h * P + a[j:j+n] + np.uint64(1)
+    return h
+W = {k: open(f"/workspace/spira_shared/spira_m7/windows/{k}.bin","rb").read() for k in ("fresh_en","fresh_fr","neither_code","test20k","queries")}
+try:
+    from windows_v2 import freshcode_v2; W["fc2"] = freshcode_v2()
+except Exception as e: W["fc2_ERROR"] = b""; print("fc2 indisponible", e)
+F = {f: open(f"{C}/{f}.bin","rb").read() for f in ("train","test","oneshot") if os.path.exists(f"{C}/{f}.bin")}
+for _k in ("test","oneshot"):
+    if _k in F: W["corpus_"+_k] = F[_k]
+rep = {"corpus": C, "files": {f: {"bytes": len(b), "sha256": hashlib.sha256(b).hexdigest()} for f, b in F.items()}, "overlap64": {}}
+ht = hashes(F["train"]); hs = np.sort(ht)
+for k, b in W.items():
+    hw = hashes(b); u = np.unique(hw)
+    i = np.searchsorted(hs, u); i = np.minimum(i, len(hs)-1); cand = u[hs[i] == u]
+    # confirmation octet par octet (anti-collision)
+    tset = set(cand.tolist()); pos_t = {}
+    for p in np.nonzero(np.isin(ht, cand))[0]: pos_t.setdefault(int(ht[p]), []).append(int(p))
+    conf = 0
+    for p in np.nonzero(np.isin(hw, cand))[0]:
+        seg = b[p:p+K]
+        if any(F["train"][q:q+K] == seg for q in pos_t.get(int(hw[p]), ())): conf += 1
+    rep["overlap64"][k] = {"window_bytes": len(b), "n_64grams": int(len(hw)), "hash_hits": int(np.isin(hw, cand).sum()), "confirmed_positions": conf}
+    print(k, rep["overlap64"][k], flush=True)
+rep["valid_zero_overlap"] = all(v["confirmed_positions"] == 0 for v in rep["overlap64"].values()) and "fc2_ERROR" not in W
+m = f"{C}/corpus_manifest.json" if os.path.exists(f"{C}/corpus_manifest.json") else f"{C}/manifest.json"
+if os.path.exists(m): rep["manifest_checker"] = json.load(open(m))
+json.dump(rep, open(OUT, "w"), indent=1); print("VALID" if rep["valid_zero_overlap"] else "INVALID")