Hashing lever2_sparse: smoke SP + compare.py fix (Checker PASS 03h13, add-only)

results/hashing/lever2_sparse/code/chain_sp.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# lever2_sparse — chaîne SP1 puis SP2 (SPIRA Hashing, 2026-10-24). N'écrit JAMAIS dans spira_relay.md (brouillon seulement), ne tue/signale rien,
+# rien dans /tmp. Porte par bras : MemAvailable >= 5,2e9 o ET lourds < 4 (lourd = process python, tous agents, RSS >= 300 Mo), sondage 120 s.
+B=/workspace/spira_shared/hashing_gen/lever2_sparse; C=$B/code; ST=$B/STATUS; PY=/workspace/SPIRA-venv/bin/python
+ts(){ TZ=America/Toronto date '+%F %T %Z'; }
+memb(){ awk '/MemAvailable/{printf "%d", $2*1024}' /proc/meminfo; }
+heavy(){ ps -eo rss=,comm= | awk '$1>=300000 && $2 ~ /python/ {n++} END{print n+0}'; }
+heavy_list(){ ps -eo pid=,rss=,comm= | awk '$2>=300000 && $3 ~ /python/ {printf "%s ", $1}'; }
+export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 PYTHONHASHSEED=0
+SMOKE_PID=$1
+while [ -n "$SMOKE_PID" ] && [ -d /proc/$SMOKE_PID ]; do sleep 30; done
+if ! $PY -c "import json,sys; sys.exit(0 if json.load(open('$B/smoke/smoke_sp.json'))['PASS'] else 1)" 2>/dev/null; then
+  echo "$(ts) | smoke test NON PASS ou absent : chaîne ARRÊTÉE, aucun bras lancé" >> $ST; exit 2; fi
+echo "$(ts) | smoke PASS ; chaîne armée (PID $$), attente porte pour SP1" >> $ST
+for spec in "SP1 SP1" "SP2 SP2"; do set -- $spec; ARM=$1; MODE=$2; last=""
+  while :; do m=$(memb); h=$(heavy)
+    if [ "$m" -ge 5200000000 ] && [ "$h" -lt 4 ]; then break; fi
+    s="mem $(awk -v m=$m 'BEGIN{printf "%.2f", m/1e9}') Go, lourds $h ($(heavy_list))"
+    echo "$(ts) $ARM porte fermée : $s" >> $B/logs/gate.log
+    st="$([ "$m" -ge 5200000000 ] && echo memOK || echo memKO)/$h"; [ "$st" != "$last" ] && echo "$(ts) | $ARM attente porte : $s" >> $ST; last=$st
+    sleep 120; done
+  setsid nice -n 10 $PY -u $C/run_sp.py $ARM $MODE 24000 4000 $B/short_old > $B/short_old.$ARM.log 2>&1 < /dev/null &
+  P=$!; echo "$(ts) | $ARM lancé PID $P (mem $(awk -v m=$(memb) 'BEGIN{printf "%.2f", m/1e9}') Go, lourds avant lancement $h : $(heavy_list)), ETA ~50-80 min" >> $ST
+  while [ -d /proc/$P ]; do sleep 60; done
+  n=$(grep -c DONE $B/short_old.$ARM.log); echo "$(ts) | $ARM fini, DONE=$n" >> $ST
+  if [ "$n" = 1 ]; then
+    nice -n 10 $PY $C/verdict_SP.py $ARM > $B/logs/verdict_$ARM.log 2>&1; echo "$(ts) | verdict $ARM rc=$? : $(grep -m1 -o '\*\*[A-ZÉ]*\*\*.*' $B/verdict_$ARM/verdict_$ARM.md 2>/dev/null)" >> $ST
+  else echo "$(ts) | $ARM : run incomplet, pas de verdict (voir short_old.$ARM.log)" >> $ST; fi
+done
+echo "$(ts) | chaîne SP terminée ; relais NON écrit automatiquement (à rédiger à la main)" >> $ST

results/hashing/lever2_sparse/code/run_sp.py

@@ -0,0 +1,37 @@
+"""lever2_sparse (SPIRA Hashing 2026-10-24) : copie de lever2/code/run_l2.py ; SEULS changements : antihebb.install(core, eta) -> sparse.install(core, MODE)
+(MODE none|SP1|SP2, voir sparse.py) et mesure du taux de décharge (champ "SP" au lieu de "Q"). Tout le reste (données, cfg, snn_tokens, éval, porte mémoire) inchangé.
+Usage: run_sp.py ARM MODE SNN_TOKENS MAXPOS OUTDIR. Short lever-2 run on old_local_v1 (CMSPIRA = v3 + Count-Min, repo default cfg),
+only snn_tokens reduced (equal for all arms). Train in-process, calibrate_fusion(train[:60000]), eval_hits (exec'd verbatim from checker/eval_ref_v3.py)."""
+import sys, os, json, time, hashlib, resource, dataclasses
+L = os.path.dirname(os.path.abspath(__file__)); sys.path.insert(0, f"{L}/python"); sys.path.insert(0, L); sys.path.insert(0, "/workspace/spira_shared/checker")
+import numpy as np
+from spira.hashing import fnv1a64
+from countmin import CMSPIRA, SPIRAConfig
+import sparse
+arm, mode, SNN, MAXPOS, OD = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4]), sys.argv[5]; os.makedirs(OD, exist_ok=True)
+# porte mémoire (23 h 53) : marge utilisateur 4 Go + plancher Relay 4,5 Go ; RSS du bras ~0,65 Go -> attendre MemAvailable >= 5,2 Go (décimal) avant de charger
+def _mem(): return int([l for l in open("/proc/meminfo") if l.startswith("MemAvailable")][0].split()[1])*1024
+while _mem() < 5.2e9:
+    print(f"porte mémoire : MemAvailable {_mem()/1e9:.2f} Go < 5,2 Go, attente 60 s", flush=True); time.sleep(60)
+t0 = time.perf_counter(); log = lambda s: print(f"[{time.perf_counter()-t0:7.0f}s] {s}", flush=True)
+src = open("/workspace/spira_shared/checker/eval_ref_v3.py").read(); g = {"np": np, "fnv1a64": fnv1a64}
+exec(src[src.index("def eval_hits"):src.index("m = SPIRA.load(ckpt)")], g); eval_hits = g["eval_hits"]
+train = open("/workspace/spira_shared/broad/old_local_v1/train.bin", "rb").read()
+cfg = dataclasses.replace(SPIRAConfig(), snn_tokens=SNN)
+m = CMSPIRA.build_cm(train, cfg, log=log); sparse.install(m.core, mode)
+m.prepare(); h = m.train(log_fn=log)
+cal = m.calibrate_fusion(train[:60_000]); log(f"cal {cal} beta={m.core.beta}")
+sp_cal = sparse.stats(m.core, reset_eval=True); log(f"SP après train+calibration {json.dumps(sp_cal)}")
+W = "/workspace/spira_shared/spira_m7/windows"
+from windows_v2 import freshcode_v2
+wins = {"seen": train[:200_000], "fc2": freshcode_v2(), **{k: open(f"{W}/{k}.bin","rb").read() for k in ("fresh_en","fresh_fr","test20k")}}
+for name, data in wins.items():
+    t1 = time.perf_counter(); r, H = eval_hits(m, data, max_positions=MAXPOS); spw = sparse.stats(m.core, reset_eval=True)
+    pl = np.fromiter((len(p) for p in m.ept.segment(data)), np.int64); off = np.concatenate([[0], np.cumsum(pl)]); n = len(H["fused_hit"])
+    np.savez_compressed(f"{OD}/{arm}__{name}.npz", **H, tgt_start=off[1:n+1], tgt_len=pl[1:n+1], window_sha256=np.array(hashlib.sha256(data).hexdigest()))
+    rec = {"arm": arm, "mode": mode, "snn_tokens": SNN, "window": name, "n": n, "fused_invocab_pct": round(100*r["fused_acc_invocab"],3),
+           "fused_all_pct": round(100*r["fused_acc"],3), "snn_invocab_pct": round(100*r["snn_acc_invocab"],3), "cal": cal, "SP_train_cal": sp_cal, "SP_window": spw,
+           "train_acc": [e.get("accuracy") for e in h["epochs"]], "eval_s": round(time.perf_counter()-t1,1),
+           "peak_rss_mb": round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024,1)}
+    open(f"{OD}/results.jsonl","a").write(json.dumps(rec)+"\n"); log(json.dumps(rec))
+log("DONE")

results/hashing/lever2_sparse/code/smoke_sp.py

@@ -0,0 +1,56 @@
+"""Smoke test lever2_sparse (SPIRA Hashing 2026-10-24) : quelques centaines de ticks, mesure du taux de décharge.
+Même construction que run_sp.py (old_local_v1, CMSPIRA cfg repo, porte mémoire 5,2 Go), snn_tokens=700 (smoke seulement).
+Variantes sur des copies du MÊME cœur initial : ah0 (antihebb.install eta=0 = témoin), none, SP1, SP2.
+  - train_pass sur 400 ticks (apprentissage actif), puis 300 ticks de core.step hors train_pass (régime éval, theta figé).
+  - vérifie : none == ah0 bit à bit (spikes par tick, W_out, W_rec, thr) ; SP1 exactement k=51 spikes/tick si >=51 éligibles ;
+    SP2 theta bouge en train, inchangé en éval. Écrit smoke/smoke_sp.json."""
+import sys, os, json, time, copy, resource, dataclasses
+L = os.path.dirname(os.path.abspath(__file__)); sys.path.insert(0, f"{L}/python"); sys.path.insert(0, L)
+import numpy as np
+from spira.hashing import fnv1a64
+from countmin import CMSPIRA, SPIRAConfig
+import antihebb, sparse
+def _mem(): return int([l for l in open("/proc/meminfo") if l.startswith("MemAvailable")][0].split()[1])*1024
+while _mem() < 5.2e9:
+    print(f"porte mémoire : MemAvailable {_mem()/1e9:.2f} Go < 5,2 Go, attente 120 s", flush=True); time.sleep(120)
+print(f"porte ouverte : MemAvailable {_mem()/1e9:.2f} Go", flush=True)
+t0 = time.perf_counter(); log = lambda s: print(f"[{time.perf_counter()-t0:7.0f}s] {s}", flush=True)
+OD = sys.argv[1]; NT, NE = 400, 300
+train = open("/workspace/spira_shared/broad/old_local_v1/train.bin", "rb").read()
+cfg = dataclasses.replace(SPIRAConfig(), snn_tokens=NT + NE)
+m = CMSPIRA.build_cm(train, cfg, log=log); m.prepare(); del train
+neu, cols = m._neurons, m._cols; base = copy.deepcopy(m.core); del m
+out = {"NT_train_ticks": NT, "NE_eval_ticks": NE, "variants": {}}; ras = {}; fin = {}
+for var in ("ah0", "none", "SP1", "SP2"):
+    c = copy.deepcopy(base)
+    if var == "ah0": antihebb.install(c, 0.0)
+    else: sparse.install(c, var)
+    rec = []; st = c.step
+    def step(idx, st=st, rec=rec, c=c):
+        x = st(idx); rec.append((c.s > 0).copy()); return x
+    c.step = step
+    c.train_pass(neu[:NT], cols[1:NT + 1], log_every=10**9)
+    th_after_train = c.theta.copy() if var in ("none", "SP1", "SP2") else None
+    for t in range(NT, NT + NE): c.step(neu[t])
+    S = np.array(rec); ras[var] = S; fin[var] = (c.W_out.copy(), c.W_rec.copy(), c.thr.copy())
+    per = S.sum(1); tr, ev = per[:NT], per[NT:]
+    d = {"rate_train": float(tr.mean() / c.n_res), "rate_eval": float(ev.mean() / c.n_res),
+         "spikes_per_tick_train": [int(tr.min()), float(np.median(tr)), int(tr.max())], "spikes_per_tick_eval": [int(ev.min()), float(np.median(ev)), int(ev.max())],
+         "rate_train_last100": float(tr[-100:].mean() / c.n_res)}
+    if var != "ah0":
+        d["sp_stats"] = sparse.stats(c)
+        d["theta_changed_in_train"] = bool(np.any(th_after_train != 0)); d["theta_unchanged_in_eval"] = bool(np.array_equal(th_after_train, c.theta))
+        d["theta_after_train"] = [float(th_after_train.mean()), float(th_after_train.min()), float(th_after_train.max())]
+    if var == "SP1": d["ticks_not_exactly_k"] = int((per != c.sp_k).sum())
+    out["variants"][var] = d; log(f"{var} {json.dumps(d)}"); del c
+out["parity_none_vs_ah0"] = {"spikes_identical": bool(np.array_equal(ras["none"], ras["ah0"])),
+                             "W_out_identical": bool(np.array_equal(fin["none"][0], fin["ah0"][0])), "W_rec_identical": bool(np.array_equal(fin["none"][1], fin["ah0"][1])),
+                             "thr_identical": bool(np.array_equal(fin["none"][2], fin["ah0"][2]))}
+v = out["variants"]
+out["checks"] = {"parity": all(out["parity_none_vs_ah0"].values()),
+                 "SP1_exact_k_every_tick": v["SP1"]["ticks_not_exactly_k"] == 0,
+                 "SP2_theta_learns_in_train_frozen_in_eval": v["SP2"]["theta_changed_in_train"] and v["SP2"]["theta_unchanged_in_eval"],
+                 "none_theta_zero": v["none"]["theta_after_train"] == [0.0, 0.0, 0.0]}
+out["PASS"] = all(out["checks"].values()); out["peak_rss_mb"] = round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, 1)
+json.dump(out, open(f"{OD}/smoke_sp.json", "w"), indent=1); log(json.dumps({"checks": out["checks"], "PASS": out["PASS"], "peak_rss_mb": out["peak_rss_mb"]}))
+sys.exit(0 if out["PASS"] else 3)

results/hashing/lever2_sparse/code/sparse.py

@@ -0,0 +1,80 @@
+"""Lever 2 — bras SPARSITÉ (SPIRA Hashing, 2026-10-24). Règles LOCALES, zéro gradient, aucune perte globale.
+y = vecteur de spikes s (0/1) de la couche spiking (réservoir LIF, N = n_res = 512) à chaque tick.
+  mode "none" : step IDENTIQUE à antihebb.install(core, 0) (= témoin eta=0 du levier 2) — vérifié bit à bit au smoke test.
+  mode "SP1"  : k-WTA, k = round(0.1*N) = 51. Parmi les neurones non réfractaires (refr == 0), les k de plus grande marge v - thr
+                émettent un spike (exactement k si >= k éligibles ; tri stable, départage par indice), les autres n'émettent pas
+                (leur potentiel v n'est pas remis à zéro). Le seuil v >= thr n'est plus utilisé pour décider du spike (thr reste mis à jour
+                par l'homéostasie du dépôt, inchangée, et sert au classement). Actif en entraînement ET en éval (contrainte d'architecture,
+                aucun paramètre appris).
+  mode "SP2"  : seuil adaptatif homéostatique additionnel theta_i : spike ssi v >= thr + theta et non réfractaire ;
+                theta_i += eta_theta * (s_i - rho), rho = 0.1, eta_theta = 1e-3, theta init 0, non borné ; appris UNIQUEMENT dans
+                train_pass (comme Q des bras AH), figé en calibration/éval. L'homéostasie du dépôt (thr, cible 0,06) reste inchangée.
+Compteurs (mesure du taux de décharge) : spikes et ticks séparés entraînement / hors entraînement."""
+import numpy as np
+def install(core, mode, rho=0.1, eta_theta=1e-3):
+    assert mode in ("none", "SP1", "SP2")
+    core.sp_mode = mode; core.sp_rho = np.float32(rho); core.sp_eta = np.float32(eta_theta)
+    core.sp_k = int(round(rho * core.n_res)); core.theta = np.zeros(core.n_res, np.float32); core.sp_learn = False
+    core.sp_cnt = {"train": [0, 0], "eval": [0, 0]}          # [spikes, ticks]
+    core.sp_kmin = {"train": core.n_res, "eval": core.n_res}; core.sp_kmax = {"train": 0, "eval": 0}
+    orig_train_pass = core.train_pass
+    def train_pass(*a, **k):
+        core.sp_learn = True
+        try: return orig_train_pass(*a, **k)
+        finally: core.sp_learn = False
+    core.train_pass = train_pass
+    def step(input_idx):
+        self = core
+        I = self.W_in[input_idx].sum(axis=0, dtype=np.float32)
+        I += self.s @ self.W_rec
+        v = self.lam * self.v + I
+        can = self.refr == 0
+        v[~can] = 0.0
+        if self.sp_mode == "SP1":
+            marg = np.where(can, v - self.thr, -np.inf)
+            win = np.argsort(-marg, kind="stable")[:self.sp_k]
+            spikes = np.zeros(self.n_res, np.bool_); spikes[win] = True; spikes &= can
+        elif self.sp_mode == "SP2":
+            spikes = (v >= self.thr + self.theta) & can
+        else:
+            spikes = (v >= self.thr) & can
+        v[spikes] = 0.0
+        np.subtract(self.refr, 1, out=self.refr, where=self.refr > 0)
+        self.refr[spikes] = self.refrac
+        self.v = v
+        self.s = spikes.astype(np.float32)
+        self.rhat += 0.02 * (self.s - self.rhat)
+        self.thr += self.eta_homeo * (self.rhat - self.target_rate)
+        np.clip(self.thr, self.thr_min, self.thr_max, out=self.thr)
+        if self.sp_mode == "SP2" and self.sp_learn:
+            self.theta += self.sp_eta * (self.s - self.sp_rho)
+        ns = int(spikes.sum()); ph = "train" if self.sp_learn else "eval"; c = self.sp_cnt[ph]; c[0] += ns; c[1] += 1
+        if ns < self.sp_kmin[ph]: self.sp_kmin[ph] = ns
+        if ns > self.sp_kmax[ph]: self.sp_kmax[ph] = ns
+        self.x_prev = self._banks[0].copy()
+        for i, tau in enumerate(self.trace_taus):
+            if i == 0: self._banks[i] = tau * self._banks[i] + self.s
+            else: self._banks[i] = tau * self._banks[i] + (1.0 - tau) * self.s
+        if self.direct_input:
+            self._direct *= np.float32(self.direct_tau)
+            np.add.at(self._direct, input_idx, np.float32(self.direct_gain))
+            self.x = np.concatenate([*self._banks, self._direct])
+        elif self.n_banks > 1: self.x = np.concatenate([*self._banks])
+        else: self.x = self._banks[0]
+        self.x_mean = 0.999 * self.x_mean + 0.001 * self.x
+        self.x_eff = self.x - self.x_mean
+        self.tick += 1
+        if self.hier: self.win_spikes += self.s
+        return self.x
+    core.step = step
+def stats(core, reset_eval=False):
+    th = core.theta
+    out = {"mode": core.sp_mode, "k": core.sp_k, "N": int(core.n_res),
+           "rate_train": (core.sp_cnt["train"][0] / (core.sp_cnt["train"][1] * core.n_res)) if core.sp_cnt["train"][1] else None,
+           "ticks_train": core.sp_cnt["train"][1], "spikes_per_tick_train_min_max": [core.sp_kmin["train"], core.sp_kmax["train"]],
+           "rate_eval": (core.sp_cnt["eval"][0] / (core.sp_cnt["eval"][1] * core.n_res)) if core.sp_cnt["eval"][1] else None,
+           "ticks_eval": core.sp_cnt["eval"][1], "spikes_per_tick_eval_min_max": [core.sp_kmin["eval"], core.sp_kmax["eval"]],
+           "theta_mean": float(th.mean()), "theta_min": float(th.min()), "theta_max": float(th.max()),
+           "thr_mean": float(core.thr.mean()), "thr_at_max_frac": float((core.thr >= core.thr_max).mean()), "thr_at_min_frac": float((core.thr <= core.thr_min).mean())}
+    if reset_eval: core.sp_cnt["eval"] = [0, 0]; core.sp_kmin["eval"] = core.n_res; core.sp_kmax["eval"] = 0
+    return out

results/hashing/lever2_sparse/code/verdict_SP.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""SPIRA Hashing — verdict levier 2 SPARSITÉ (SP1/SP2, argument 1) — copie de lever2/verdict_AH1/verdict_AH1.py, SEULS changements :
+bras candidat lu dans lever2_sparse/short_old, sortie dans lever2_sparse/verdict_<ARM>, libellés. Témoin, masque et protocole identiques.
+(ex-docstring AH1) verdict levier 2 AH1 (eta_ah=1e-3) contre le témoin à budget égal ref (eta_ah=0), méthode banc v4.1
+(copie adaptée de m7_v41/strict/verdict_m7_v41_strict.py ; lecture seule des hits short_old/{ref,AH1}__*.npz, aucun modèle chargé).
+Fenêtres MESURÉES par run_l2.py (4000 positions chacune) : seen (= old_local_v1/train[:200000]), fc2, fresh_en, fresh_fr, test20k
+(= old_local_v1/test.bin, sha ce0526c6…, PAS broad_local_v1/test.bin ; masque strict propre recalculé : verdict_AH1/mask_oldtest_strict.npz,
+union old_local_v1 ∪ broad_local_v1). queries et test_v3 : NON évalués par les bras → absents (non mesurables). neither_code exclu.
+hit = fused_hit & invocab. Appariement par position si tgt_start et tgt_len identiques (vérifié). McNemar exact bilatéral = DÉCISION (p<0,05).
+Bootstrap par blocs de 512 o (bloc = tgt_start // 512), B = 10 000, un générateur seed 0, fenêtres dans l'ordre WIN, apparié ; stat = acc invocab.
+Écart seen − moyenne fraîche (fc2, fresh_en, fresh_fr, test20k strict ; test_v3 non mesuré). Bootstrap secondaire par positions, seed 25.
+Critère utilisateur : ADOPTÉ ssi gain significatif sur seen ET sur >=1 fenêtre fraîche ET aucune perte significative sur une fenêtre valide ; sinon REJETÉ."""
+import sys, os, json, math, hashlib, numpy as np
+D = "/workspace/spira_shared/hashing_gen/lever2/short_old"; RT, CT = "ref", sys.argv[1]; assert CT in ("SP1", "SP2")
+DC = "/workspace/spira_shared/hashing_gen/lever2_sparse/short_old"; OP = f"/workspace/spira_shared/hashing_gen/lever2_sparse/verdict_{CT}"
+MASKP = "/workspace/spira_shared/hashing_gen/lever2/verdict_AH1/mask_oldtest_strict.npz"
+DESC = {'SP1': 'SP1 (k-WTA k=51 sur les spikes, N=512, rho=0.1)', 'SP2': 'SP2 (seuil adaptatif theta += 1e-3*(s-0.1), theta0=0, figé en éval)'}[CT]
+WIN = ["seen", "fc2", "fresh_en", "fresh_fr", "test20k"]; FRESH = ["fc2", "fresh_en", "fresh_fr", "test20k"]; GAPW = FRESH
+NOT_MEASURED = ["queries", "test_v3"]; BLK, NB, SEED = 512, 10000, 0
+os.makedirs(OP, exist_ok=True); MS = np.load(MASKP)
+def binp(b, c):
+    n = b + c
+    return 1.0 if n == 0 else min(1.0, 2 * sum(math.comb(n, i) for i in range(min(b, c) + 1)) / 2 ** n)
+def blocksums(st, h, iv, nbk):
+    b = st // BLK; return np.bincount(b, weights=h & iv, minlength=nbk), np.bincount(b, weights=iv, minlength=nbk)
+R = {}; RNG = np.random.default_rng(SEED); BOOT = {}; KEEP = {}; EX = {}
+for w in WIN:
+    A = np.load(f"{D}/{RT}__{w}.npz"); B = np.load(f"{DC}/{CT}__{w}.npz")
+    assert str(A["window_sha256"]) == str(B["window_sha256"]), f"{w}: octets différents"
+    na, nb_ = len(A["fused_hit"]), len(B["fused_hit"])
+    same_tok = na == nb_ and np.array_equal(A["tgt_start"], B["tgt_start"]) and np.array_equal(A["tgt_len"], B["tgt_len"])
+    assert same_tok, f"{w}: tokenisation différente, appariement par position impossible"
+    ha, hb = A["fused_hit"].astype(bool) & A["invocab"].astype(bool), B["fused_hit"].astype(bool) & B["invocab"].astype(bool)
+    if w == "test20k":
+        assert str(MS["sha__test20k"]) == str(A["window_sha256"]), "masque : fenêtre différente"
+        bad = lambda cov: np.array([cov[max(0, int(s) - 64):int(s) + int(l)].any() for s, l in zip(A["tgt_start"], A["tgt_len"])])
+        bo, bb_ = bad(MS["old_local_v1__covered"]), bad(MS["broad_local_v1__covered"]); k = ~(bo | bb_)
+        EX = {"excluded_by_old_local_v1": int(bo.sum()), "excluded_by_broad_local_v1": int(bb_.sum()), "excluded_union": int((bo | bb_).sum()),
+              "max_byte_evaluated": int((A["tgt_start"] + A["tgt_len"]).max())}
+    else: k = np.ones(na, bool)
+    b = int((ha & ~hb & k).sum()); c = int((~ha & hb & k).sum()); npair = int(k.sum())
+    inv_same = bool(np.array_equal(A["invocab"][k], B["invocab"][k])); p = binp(b, c)
+    ia, ib = A["invocab"].astype(bool) & k, B["invocab"].astype(bool) & k
+    acc_a, acc_b = 100 * (ha & k).sum() / ia.sum(), 100 * (hb & k).sum() / ib.sum()
+    nbk = int(A["tgt_start"].max() // BLK + 1)
+    sa_h, sa_n = blocksums(A["tgt_start"][k], ha[k], A["invocab"].astype(bool)[k], nbk)
+    sb_h, sb_n = blocksums(B["tgt_start"][k], hb[k], B["invocab"].astype(bool)[k], nbk)
+    nz = (sa_n + sb_n) > 0; sa_h, sa_n, sb_h, sb_n = sa_h[nz], sa_n[nz], sb_h[nz], sb_n[nz]; m = len(sa_h)
+    idx = RNG.integers(0, m, (NB, m))
+    with np.errstate(invalid="ignore", divide="ignore"):
+        ba = 100 * sa_h[idx].sum(1) / sa_n[idx].sum(1); bb = 100 * sb_h[idx].sum(1) / sb_n[idx].sum(1)
+    d = bb - ba; BOOT[w] = (ba, bb); KEEP[w] = (k, ha, hb, A["invocab"].astype(bool))
+    lo, hi = np.percentile(d, [2.5, 97.5]); pboot = float(min(1.0, 2 * min((d <= 0).mean(), (d >= 0).mean())))
+    gain, loss = bool(p < 0.05 and c > b), bool(p < 0.05 and b > c)
+    R[w] = {"n_positions": na, "n_clean": npair, "n_invocab_clean_ref": int(ia.sum()), "n_invocab_clean_cand": int(ib.sum()), "invocab_identical": inv_same,
+            "pairing": "position (tokenizers identiques, tgt_start/tgt_len vérifiés)", "acc_ref_pct": float(acc_a), "acc_cand_pct": float(acc_b), "delta_pct": float(acc_b - acc_a),
+            "b_ref_only": b, "c_cand_only": c, "mcnemar_p": p, "boot_ci95_delta_pct": [float(lo), float(hi)], "boot_p": pboot, "n_blocks": m,
+            "sig_gain": gain, "sig_loss": loss, "boot_conflict": bool((gain and hi < 0) or (loss and lo > 0)), "window_sha256": str(A["window_sha256"])}
+    if w == "test20k": R[w]["strict_mask"] = EX
+    print(w, json.dumps(R[w]), flush=True)
+gap = {}
+for side, j in (("ref", 0), ("cand", 1)):
+    fm = np.mean([BOOT[w][j] for w in GAPW], axis=0); g = BOOT["seen"][j] - fm
+    pt = R["seen"]["acc_%s_pct" % side] - np.mean([R[w]["acc_%s_pct" % side] for w in GAPW])
+    gap[side] = {"fresh_mean_pct": float(np.mean([R[w]["acc_%s_pct" % side] for w in GAPW])), "seen_pct": R["seen"]["acc_%s_pct" % side], "gap_pct": float(pt), "ci95": [float(x) for x in np.nanpercentile(g, [2.5, 97.5])]}
+dg = (BOOT["seen"][1] - np.mean([BOOT[w][1] for w in GAPW], axis=0)) - (BOOT["seen"][0] - np.mean([BOOT[w][0] for w in GAPW], axis=0))
+gap["cand_minus_ref"] = {"gap_diff_pct": gap["cand"]["gap_pct"] - gap["ref"]["gap_pct"], "ci95": [float(x) for x in np.nanpercentile(dg, [2.5, 97.5])]}
+R2 = np.random.default_rng(25); BP = {}
+for w in WIN:
+    k, ha_, hb_, iv_ = KEEP[w]; m_ = k & iv_; r_ = ha_[m_].astype(np.float64); c_ = hb_[m_].astype(np.float64); n_ = len(r_); ar = np.empty(NB); ac = np.empty(NB)
+    for t in range(NB):
+        i = R2.integers(0, n_, n_); ar[t] = r_[i].mean(); ac[t] = c_[i].mean()
+    BP[w] = (100 * ar, 100 * ac)
+dgp = (BP["seen"][1] - np.mean([BP[w][1] for w in GAPW], axis=0)) - (BP["seen"][0] - np.mean([BP[w][0] for w in GAPW], axis=0))
+gap["cand_minus_ref"]["ci95_boot_positions_seed25"] = [float(x) for x in np.percentile(dgp, [2.5, 97.5])]
+gap["windows"] = GAPW; gap["note"] = "test_v3 non mesuré par les bras (absent de la moyenne) ; queries non mesuré ; neither_code exclu"
+L = [w for w in WIN if R[w]["sig_loss"]]; G = [w for w in WIN if R[w]["sig_gain"]]; GF = [w for w in FRESH if R[w]["sig_gain"]]
+dec = "ADOPTÉ" if (R["seen"]["sig_gain"] and GF and not L) else "REJETÉ"
+why = []
+if not R["seen"]["sig_gain"]: why.append("pas de gain significatif sur seen")
+if not GF: why.append("aucun gain significatif sur une fenêtre fraîche")
+if L: why.append("perte(s) significative(s) : " + ", ".join(L))
+out = {"cand": DESC, "ref": "ref (eta_ah=0, budget égal)", "corpus": "broad/old_local_v1", "snn_tokens": 24000, "epochs": 2, "max_positions": 4000,
+       "windows_measured": WIN, "windows_not_measured": NOT_MEASURED, "excluded": ["neither_code"], "decision": dec, "reasons": why, "losses": L, "gains": G, "fresh_gains": GF,
+       "rule": "ADOPTÉ ssi gain significatif (McNemar exact p<0,05) sur seen ET sur >=1 fenêtre fraîche (fc2, fresh_en, fresh_fr, test20k strict) ET aucune perte significative sur une fenêtre valide",
+       "bootstrap": {"block_bytes": BLK, "B": NB, "seed": SEED, "stat": "acc invocab fused, blocs = tgt_start//512, apparié"}, "per_window": R, "seen_fresh_gap": gap,
+       "script_sha256": hashlib.sha256(open(__file__, "rb").read()).hexdigest(),
+       "inputs_sha256": {f"{a}__{w}.npz": hashlib.sha256(open(f"{D if a == RT else DC}/{a}__{w}.npz", "rb").read()).hexdigest() for a in (RT, CT) for w in WIN},
+       "mask_sha256": hashlib.sha256(open(MASKP, "rb").read()).hexdigest(), "mask_path": MASKP}
+json.dump(out, open(f"{OP}/verdict_{CT}.json", "w"), indent=1, ensure_ascii=False)
+md = [f"# Verdict levier 2 — {DESC} contre ref (eta_ah = 0, budget égal), old_local_v1 court\n",
+      f"**{dec}** — " + ("; ".join(why) if why else "critère rempli"), "",
+      "Précision fused sur les cibles invocab (%), 4000 positions par fenêtre, appariement par position.", "",
+      f"| fenêtre | n apparié | {CT} % | réf % | Δ pts | b (réf seule) / c ({CT} seul) | p McNemar exact | IC95 Δ bootstrap (512 o, B=10 000, seed 0) | décision |", "|---|---|---|---|---|---|---|---|---|"]
+for w in WIN:
+    r = R[w]; dd = "GAIN" if r["sig_gain"] else ("PERTE" if r["sig_loss"] else "n.s.")
+    nm = "test20k strict (old_local_v1/test.bin, union old ∪ broad)" if w == "test20k" else w
+    md.append(f"| {nm} | {r['n_clean']} | {r['acc_cand_pct']:.3f} | {r['acc_ref_pct']:.3f} | {r['delta_pct']:+.3f} | {r['b_ref_only']}/{r['c_cand_only']} | {r['mcnemar_p']:.3g} | [{r['boot_ci95_delta_pct'][0]:+.3f} ; {r['boot_ci95_delta_pct'][1]:+.3f}] | {dd}{' ⚠ conflit bootstrap' if r['boot_conflict'] else ''} |")
+md += ["| queries | — | non mesuré | non mesuré | — | — | — | — | non mesurable (fenêtre absente du run) |", "| test_v3 | — | non mesuré | non mesuré | — | — | — | — | non mesurable (fenêtre absente du run) |", "",
+       f"test20k strict : {EX['excluded_union']} positions exclues sur 4000 ({EX['excluded_by_old_local_v1']} par old_local_v1, {EX['excluded_by_broad_local_v1']} par broad_local_v1), octets évalués jusqu'à {EX['max_byte_evaluated']}.", "",
+       f"Écart seen − moyenne fraîche ({', '.join(GAPW)} ; test_v3 non mesuré) : {CT} {gap['cand']['gap_pct']:+.3f} pts [IC95 {gap['cand']['ci95'][0]:+.3f} ; {gap['cand']['ci95'][1]:+.3f}] (fraîche {gap['cand']['fresh_mean_pct']:.3f} %) ; réf {gap['ref']['gap_pct']:+.3f} pts [{gap['ref']['ci95'][0]:+.3f} ; {gap['ref']['ci95'][1]:+.3f}] (fraîche {gap['ref']['fresh_mean_pct']:.3f} %) ; différence {CT} − réf {gap['cand_minus_ref']['gap_diff_pct']:+.3f} [blocs 512 o seed 0 : {gap['cand_minus_ref']['ci95'][0]:+.3f} ; {gap['cand_minus_ref']['ci95'][1]:+.3f}] [positions seed 25 : {gap['cand_minus_ref']['ci95_boot_positions_seed25'][0]:+.3f} ; {gap['cand_minus_ref']['ci95_boot_positions_seed25'][1]:+.3f}].", "",
+       f"Script sha256 {out['script_sha256']} ; masque sha256 {out['mask_sha256']}.", ""]
+open(f"{OP}/verdict_{CT}.md", "w").write("\n".join(md) + "\n"); print("\n".join(md))

results/hashing/lever2_sparse/compare_fix/compare.py

@@ -0,0 +1,31 @@
+"""McNemar v4.1 (fused_hit & invocab, exact binomial, same tokenizer => patch-level pairing) + bootstrap seed0 B=10000 over 512-byte blocks.
+Correctif 2026-10-24 (SPIRA Hashing) : scipy absent du venv -> scipy.stats.binomtest remplacé par un McNemar exact bilatéral en pur Python
+(math.comb). Méthode = celle de scipy.binomtest (alternative='two-sided') : p = somme des pmf(i) <= pmf(observé)*(1+1e-7), i=0..n, p=0,5,
+plafonné à 1. Pour p=0,5 la loi est symétrique : c'est numériquement identique à 2*min(queue gauche, queue droite) plafonné à 1
+(méthode de verdict_AH1.py) ; mcnemar_p_2tail est aussi écrit pour contrôle. Original conservé : compare.py.bak_scipy.
+NB : ce script NE fait PAS le masque strict test20k (voir verdict_AH1.py pour le protocole complet)."""
+import sys, json, math, numpy as np
+def mcnemar_exact(g, l, rtol=1e-7):
+    n = g + l
+    if n == 0: return 1.0
+    pmf = [math.comb(n, i) / 2 ** n for i in range(n + 1)]; thr = pmf[g] * (1 + rtol)
+    return min(1.0, math.fsum(x for x in pmf if x <= thr))
+def mcnemar_2tail(g, l):
+    n = g + l
+    return 1.0 if n == 0 else min(1.0, 2 * sum(math.comb(n, i) for i in range(min(g, l) + 1)) / 2 ** n)
+if __name__ == "__main__":
+    D, ref, new = sys.argv[1:4]; out = {}
+    for w in ("seen","fc2","fresh_en","fresh_fr","test20k"):
+        a = np.load(f"{D}/{ref}__{w}.npz"); b = np.load(f"{D}/{new}__{w}.npz")
+        assert str(a["window_sha256"]) == str(b["window_sha256"]) and (a["tgt_start"] == b["tgt_start"]).all()
+        ha = a["fused_hit"] & a["invocab"]; hb = b["fused_hit"] & b["invocab"]; g = int((hb & ~ha).sum()); l = int((ha & ~hb).sum())
+        blk = a["tgt_start"] // 512; ub, inv = np.unique(blk, return_inverse=True)
+        sa = np.bincount(inv, ha); sb = np.bincount(inv, hb); cn = np.bincount(inv); rng = np.random.default_rng(0); nb = len(ub)
+        bs = np.empty(10000)
+        for i in range(10000):
+            k = rng.integers(0, nb, nb); bs[i] = 100 * (sb[k].sum() - sa[k].sum()) / cn[k].sum()
+        out[w] = dict(n=len(ha), n_blocks=nb, ref_pct=round(100*ha.mean(),3), new_pct=round(100*hb.mean(),3), new_only=g, ref_only=l,
+                      p=mcnemar_exact(g, l), p_2tail=mcnemar_2tail(g, l), delta_pt=round(100*(hb.mean()-ha.mean()),3),
+                      ci95=[round(np.percentile(bs,2.5),3), round(np.percentile(bs,97.5),3)])
+        print(w, json.dumps(out[w]))
+    json.dump(out, open(f"{D}/mcnemar_{new}_vs_{ref}.json","w"), indent=1)

results/hashing/lever2_sparse/compare_fix/test_compare_mcnemar.py

@@ -0,0 +1,12 @@
+"""Test du correctif compare.py : p McNemar contre verdict_AH1.json / verdict_AH2.json (b,c,p) + cas connus."""
+import json, sys; sys.path.insert(0, "/workspace/spira_shared/hashing_gen/lever2/code"); from compare import mcnemar_exact, mcnemar_2tail
+ok = True
+for v in ("verdict_AH1.json", "verdict_AH2/verdict_AH2.json"):
+    R = json.load(open(f"/workspace/spira_shared/hashing_gen/lever2/{v}"))["per_window"]
+    for w, r in R.items():
+        b, c, p = r["b_ref_only"], r["c_cand_only"], r["mcnemar_p"]; p1 = mcnemar_exact(c, b); p2 = mcnemar_2tail(c, b)
+        good = abs(p1 - p) <= 1e-12 * max(1, p) and abs(p2 - p) <= 1e-12 * max(1, p); ok &= good
+        print(f"{v:30s} {w:9s} b/c={b}/{c}  verdict p={p:.10g}  compare p={p1:.10g}  2tail={p2:.10g}  {'OK' if good else 'ÉCART'}")
+for g, l, exp in ((10, 24, 0.0243), (0, 0, 1.0), (5, 5, 1.0), (0, 10, 2/1024)):
+    p = mcnemar_exact(g, l); good = abs(p - exp) < 5e-5; ok &= good; print(f"cas {g}/{l}: p={p:.6g} attendu {exp:.6g} {'OK' if good else 'ÉCART'}")
+print("TOUS OK" if ok else "ÉCHEC DU TEST"); sys.exit(0 if ok else 1)

results/hashing/lever2_sparse/smoke/smoke_sp.json

@@ -0,0 +1,171 @@
+{
+ "NT_train_ticks": 400,
+ "NE_eval_ticks": 300,
+ "variants": {
+  "ah0": {
+   "rate_train": 0.06556640625,
+   "rate_eval": 0.06859375,
+   "spikes_per_tick_train": [
+    9,
+    36.0,
+    82
+   ],
+   "spikes_per_tick_eval": [
+    11,
+    38.0,
+    66
+   ],
+   "rate_train_last100": 0.06296875
+  },
+  "none": {
+   "rate_train": 0.06556640625,
+   "rate_eval": 0.06859375,
+   "spikes_per_tick_train": [
+    9,
+    36.0,
+    82
+   ],
+   "spikes_per_tick_eval": [
+    11,
+    38.0,
+    66
+   ],
+   "rate_train_last100": 0.06296875,
+   "sp_stats": {
+    "mode": "none",
+    "k": 51,
+    "N": 512,
+    "rate_train": 0.06556640625,
+    "ticks_train": 400,
+    "spikes_per_tick_train_min_max": [
+     9,
+     82
+    ],
+    "rate_eval": 0.06859375,
+    "ticks_eval": 300,
+    "spikes_per_tick_eval_min_max": [
+     11,
+     66
+    ],
+    "theta_mean": 0.0,
+    "theta_min": 0.0,
+    "theta_max": 0.0,
+    "thr_mean": 1.0085546970367432,
+    "thr_at_max_frac": 0.0,
+    "thr_at_min_frac": 0.0
+   },
+   "theta_changed_in_train": false,
+   "theta_unchanged_in_eval": true,
+   "theta_after_train": [
+    0.0,
+    0.0,
+    0.0
+   ]
+  },
+  "SP1": {
+   "rate_train": 0.099609375,
+   "rate_eval": 0.099609375,
+   "spikes_per_tick_train": [
+    51,
+    51.0,
+    51
+   ],
+   "spikes_per_tick_eval": [
+    51,
+    51.0,
+    51
+   ],
+   "rate_train_last100": 0.099609375,
+   "sp_stats": {
+    "mode": "SP1",
+    "k": 51,
+    "N": 512,
+    "rate_train": 0.099609375,
+    "ticks_train": 400,
+    "spikes_per_tick_train_min_max": [
+     51,
+     51
+    ],
+    "rate_eval": 0.099609375,
+    "ticks_eval": 300,
+    "spikes_per_tick_eval_min_max": [
+     51,
+     51
+    ],
+    "theta_mean": 0.0,
+    "theta_min": 0.0,
+    "theta_max": 0.0,
+    "thr_mean": 1.0515716075897217,
+    "thr_at_max_frac": 0.0,
+    "thr_at_min_frac": 0.0
+   },
+   "theta_changed_in_train": false,
+   "theta_unchanged_in_eval": true,
+   "theta_after_train": [
+    0.0,
+    0.0,
+    0.0
+   ],
+   "ticks_not_exactly_k": 0
+  },
+  "SP2": {
+   "rate_train": 0.0657861328125,
+   "rate_eval": 0.06897786458333334,
+   "spikes_per_tick_train": [
+    10,
+    36.0,
+    82
+   ],
+   "spikes_per_tick_eval": [
+    11,
+    38.0,
+    66
+   ],
+   "rate_train_last100": 0.0632421875,
+   "sp_stats": {
+    "mode": "SP2",
+    "k": 51,
+    "N": 512,
+    "rate_train": 0.0657861328125,
+    "ticks_train": 400,
+    "spikes_per_tick_train_min_max": [
+     10,
+     82
+    ],
+    "rate_eval": 0.06897786458333334,
+    "ticks_eval": 300,
+    "spikes_per_tick_eval_min_max": [
+     11,
+     66
+    ],
+    "theta_mean": -0.01368551142513752,
+    "theta_min": -0.03900008648633957,
+    "theta_max": 0.01400004792958498,
+    "thr_mean": 1.0089516639709473,
+    "thr_at_max_frac": 0.0,
+    "thr_at_min_frac": 0.0
+   },
+   "theta_changed_in_train": true,
+   "theta_unchanged_in_eval": true,
+   "theta_after_train": [
+    -0.01368551142513752,
+    -0.03900008648633957,
+    0.01400004792958498
+   ]
+  }
+ },
+ "parity_none_vs_ah0": {
+  "spikes_identical": true,
+  "W_out_identical": true,
+  "W_rec_identical": true,
+  "thr_identical": true
+ },
+ "checks": {
+  "parity": true,
+  "SP1_exact_k_every_tick": true,
+  "SP2_theta_learns_in_train_frozen_in_eval": true,
+  "none_theta_zero": true
+ },
+ "PASS": true,
+ "peak_rss_mb": 4028.2
+}

results/hashing/lever2_sparse/smoke/smoke_sp.log.txt

@@ -0,0 +1,14 @@
+porte ouverte : MemAvailable 5.33 Go
+[      5s] [spira] fitting EPT byte-level Markov model ...
+[      6s] [spira] calibrated surprisal budget = 15.37 bits (target avg patch len 4.5)
+[      6s] [spira] segmenting full train stream ...
+[     12s] [spira] 2656631 patches, avg len 4.69 B (12.5 MB segmented in 5.8s, 2.16 MB/s)
+[     12s] [spira] distinct patches in train: 210582
+[     12s] [spira] building patch registry (137261 ids) ...
+[spira] precomputing projections for 700 SNN + 1200000 AM tokens ...
+    done in 0.1s
+[     33s] ah0 {"rate_train": 0.06556640625, "rate_eval": 0.06859375, "spikes_per_tick_train": [9, 36.0, 82], "spikes_per_tick_eval": [11, 38.0, 66], "rate_train_last100": 0.06296875}
+[     50s] none {"rate_train": 0.06556640625, "rate_eval": 0.06859375, "spikes_per_tick_train": [9, 36.0, 82], "spikes_per_tick_eval": [11, 38.0, 66], "rate_train_last100": 0.06296875, "sp_stats": {"mode": "none", "k": 51, "N": 512, "rate_train": 0.06556640625, "ticks_train": 400, "spikes_per_tick_train_min_max": [9, 82], "rate_eval": 0.06859375, "ticks_eval": 300, "spikes_per_tick_eval_min_max": [11, 66], "theta_mean": 0.0, "theta_min": 0.0, "theta_max": 0.0, "thr_mean": 1.0085546970367432, "thr_at_max_frac": 0.0, "thr_at_min_frac": 0.0}, "theta_changed_in_train": false, "theta_unchanged_in_eval": true, "theta_after_train": [0.0, 0.0, 0.0]}
+[     66s] SP1 {"rate_train": 0.099609375, "rate_eval": 0.099609375, "spikes_per_tick_train": [51, 51.0, 51], "spikes_per_tick_eval": [51, 51.0, 51], "rate_train_last100": 0.099609375, "sp_stats": {"mode": "SP1", "k": 51, "N": 512, "rate_train": 0.099609375, "ticks_train": 400, "spikes_per_tick_train_min_max": [51, 51], "rate_eval": 0.099609375, "ticks_eval": 300, "spikes_per_tick_eval_min_max": [51, 51], "theta_mean": 0.0, "theta_min": 0.0, "theta_max": 0.0, "thr_mean": 1.0515716075897217, "thr_at_max_frac": 0.0, "thr_at_min_frac": 0.0}, "theta_changed_in_train": false, "theta_unchanged_in_eval": true, "theta_after_train": [0.0, 0.0, 0.0], "ticks_not_exactly_k": 0}
+[     84s] SP2 {"rate_train": 0.0657861328125, "rate_eval": 0.06897786458333334, "spikes_per_tick_train": [10, 36.0, 82], "spikes_per_tick_eval": [11, 38.0, 66], "rate_train_last100": 0.0632421875, "sp_stats": {"mode": "SP2", "k": 51, "N": 512, "rate_train": 0.0657861328125, "ticks_train": 400, "spikes_per_tick_train_min_max": [10, 82], "rate_eval": 0.06897786458333334, "ticks_eval": 300, "spikes_per_tick_eval_min_max": [11, 66], "theta_mean": -0.01368551142513752, "theta_min": -0.03900008648633957, "theta_max": 0.01400004792958498, "thr_mean": 1.0089516639709473, "thr_at_max_frac": 0.0, "thr_at_min_frac": 0.0}, "theta_changed_in_train": true, "theta_unchanged_in_eval": true, "theta_after_train": [-0.01368551142513752, -0.03900008648633957, 0.01400004792958498]}
+[     84s] {"checks": {"parity": true, "SP1_exact_k_every_tick": true, "SP2_theta_learns_in_train_frozen_in_eval": true, "none_theta_zero": true}, "PASS": true, "peak_rss_mb": 4028.2}