Hashing: verify_v3 (fixed, all positions per hash), pipeline_v3, train_cov
Browse files
results/hashing/code/SHA256SUMS
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
a135948fc8c2464844c6ce3b3b0bf174ce7131b452ca000c40a9b6d4b71bb9e0 code/pipeline_v3.sh
|
| 2 |
+
73fa1d199c1b41304b9de5ef8b120aceeaffc2cb1d6455b50de4a1b888417de5 code/train_cov.py
|
| 3 |
+
26d598eebf6763d3d28b274e1385477d162ef99abae7ef4f019f78e231710349 code/verify_v3.py
|
results/hashing/code/pipeline_v3.sh
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
# SPIRA Hashing : attente corpus_v3 (10 min x 12 = 2 h max), verif, puis si VALID : train v2b concat cov (bras b) + eval + McNemar.
|
| 3 |
+
export OMP_NUM_THREADS=1 PYTHONHASHSEED=0 TZ=America/Toronto
|
| 4 |
+
H=/workspace/spira_shared/spira_v3verify; PY=/workspace/SPIRA-venv/bin/python; cd $H
|
| 5 |
+
log(){ echo "$(date '+%F %H:%M:%S %Z') $*" >> $H/pipeline.log; }
|
| 6 |
+
C=""; prev=""
|
| 7 |
+
for i in $(seq 1 481); do
|
| 8 |
+
for d in /workspace/spira_shared/corpus_v3 /workspace/spira_shared/broad/broad_gutenberg_v1 /workspace/spira_shared/checker/corpus_v3; do
|
| 9 |
+
[ -f $d/train.bin ] && { [ -f $d/manifest.json ] || [ -f $d/corpus_manifest.json ]; } && C=$d && break
|
| 10 |
+
done
|
| 11 |
+
if [ -n "$C" ]; then cur=$(stat -c '%s%Y' $C/*.bin $C/*manifest*.json | tr -d '\n'); [ "$cur" = "$prev" ] && break; prev=$cur; log "corpus vu $C, attente stabilite"; else log "check $i : corpus_v3 absent"; fi
|
| 12 |
+
[ $i -eq 481 ] && { log "TIMEOUT 8h : corpus_v3 non pret"; exit 3; }
|
| 13 |
+
sleep 60
|
| 14 |
+
done
|
| 15 |
+
log "verif $C"; $PY verify_v3.py $C verify_corpus_v3.json > verify.log 2>&1
|
| 16 |
+
grep -q '^VALID' verify.log || { log "INVALIDE ou erreur : pas d'entrainement (voir verify.log)"; exit 4; }
|
| 17 |
+
av=$(awk '/MemAvailable/{print int($2/1048576)}' /proc/meminfo); until [ $(awk '/MemAvailable/{print int($2/1048576)}' /proc/meminfo) -ge 5 ]; do log "RAM < 5G, attente"; sleep 60; done
|
| 18 |
+
log "VALID, lancement train (RAM dispo ${av}G)"
|
| 19 |
+
cd /workspace/spira_shared/spira_cov && $PY train_cov.py $C/train.bin $H/cm_v3_cov_b.npz b > $H/train_v3_cov_b.log 2>&1 || { log "train ECHEC"; exit 1; }
|
| 20 |
+
cd $H; log "train fini, eval"
|
| 21 |
+
$PY eval_win.py cm_v3_cov_b.npz $C/train.bin $C/test.bin v3cov hits > eval_v3.log 2>&1 || log "eval new ECHEC"
|
| 22 |
+
for r in "/workspace/spira_shared/spira_m8/cm_broad_local_v1.npz m8broad" "/workspace/spira_shared/spira_cov/cm_broad_cov_b.npz covb"; do
|
| 23 |
+
set -- $r; for k in $(seq 1 72); do [ -f $1 ] && break; sleep 600; done
|
| 24 |
+
[ -f $1 ] || { log "ref $2 absente apres 12h"; continue; }
|
| 25 |
+
$PY eval_win.py $1 /workspace/spira_shared/broad/broad_local_v1/train.bin $C/test.bin $2 hits > eval_$2.log 2>&1 || log "eval $2 ECHEC"
|
| 26 |
+
$PY mcnemar_win.py hits v3cov $2 mcnemar_v3cov_vs_$2.json > mcnemar_$2.log 2>&1; log "mcnemar vs $2 fait"
|
| 27 |
+
done; log "FIN"
|
results/hashing/code/train_cov.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Copie de checker/train_scale_v2.py (recette fixed2+cal, config repo par defaut, d_in=1024, vocab 16384)
|
| 2 |
+
ou m.prepare()/m.train() sont remplaces par cov_prepare.prepare_cov/train_cov (bras a|b).
|
| 3 |
+
Usage: train_cov.py DATA OUT ARM"""
|
| 4 |
+
import json, sys, time, os
|
| 5 |
+
sys.path.insert(0, "/workspace/SPIRA/python"); sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| 6 |
+
import numpy as np
|
| 7 |
+
from spira.model import SPIRA, SPIRAConfig
|
| 8 |
+
from cov_prepare import prepare_cov, train_cov
|
| 9 |
+
data_p, out, arm = sys.argv[1:4]
|
| 10 |
+
cfg = SPIRAConfig(vocab_size=16384, d_in=1024, snn_tokens=600_000, am_tokens=1_200_000, epochs=2)
|
| 11 |
+
data = open(data_p, "rb").read()
|
| 12 |
+
t0 = time.perf_counter(); log = lambda s: print(f"[{time.perf_counter()-t0:7.0f}s] {s}", flush=True)
|
| 13 |
+
m = SPIRA.build(data, cfg, log=log); cov = prepare_cov(m, arm=arm, log=log); m._patches_train = None
|
| 14 |
+
hist = train_cov(m, log_fn=log, ckpt_path=None)
|
| 15 |
+
cal = m.calibrate_fusion(data[:60_000]); log(f"calibration {cal} beta={m.core.beta}")
|
| 16 |
+
m.save(out)
|
| 17 |
+
chk = SPIRA.load(out).fusion; assert chk["scale"] == cal["scale"], (chk, cal); log(f"ckpt fusion verifie {chk}")
|
| 18 |
+
json.dump({"script": "spira_cov/train_cov", "arm": arm, "config": vars(cfg), "cov": {k: cov[k] for k in ("arm","N","seed","k")},
|
| 19 |
+
"snn_slices": cov["snn_slices"], "am_slices": cov["am_slices"], "fusion": cal, "meta": m.meta,
|
| 20 |
+
"wall_s": time.perf_counter()-t0, "history_epochs": hist.get("epochs"), "am": hist.get("am")},
|
| 21 |
+
open(out + ".cov.json", "w"), indent=1, default=str)
|
| 22 |
+
log(f"saved {out}")
|
results/hashing/code/verify_v3.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""SPIRA Hashing - verification independante (lecture seule) d'un corpus: tailles, sha256, overlap 64 o.
|
| 2 |
+
Usage: verify_v3.py CORPUS_DIR OUT_JSON"""
|
| 3 |
+
import sys, os, json, hashlib, numpy as np
|
| 4 |
+
sys.path.insert(0, "/workspace/spira_shared/checker")
|
| 5 |
+
C, OUT = sys.argv[1:3]; K = 64; P = np.uint64(1099511628211)
|
| 6 |
+
def hashes(b):
|
| 7 |
+
a = np.frombuffer(b, np.uint8).astype(np.uint64); n = len(a) - K + 1
|
| 8 |
+
if n <= 0: return np.zeros(0, np.uint64)
|
| 9 |
+
h = np.zeros(n, np.uint64)
|
| 10 |
+
with np.errstate(over="ignore"):
|
| 11 |
+
for j in range(K): h = h * P + a[j:j+n] + np.uint64(1)
|
| 12 |
+
return h
|
| 13 |
+
W = {k: open(f"/workspace/spira_shared/spira_m7/windows/{k}.bin","rb").read() for k in ("fresh_en","fresh_fr","neither_code","test20k","queries")}
|
| 14 |
+
try:
|
| 15 |
+
from windows_v2 import freshcode_v2; W["fc2"] = freshcode_v2()
|
| 16 |
+
except Exception as e: W["fc2_ERROR"] = b""; print("fc2 indisponible", e)
|
| 17 |
+
F = {f: open(f"{C}/{f}.bin","rb").read() for f in ("train","test","oneshot") if os.path.exists(f"{C}/{f}.bin")}
|
| 18 |
+
for _k in ("test","oneshot"):
|
| 19 |
+
if _k in F: W["corpus_"+_k] = F[_k]
|
| 20 |
+
rep = {"corpus": C, "files": {f: {"bytes": len(b), "sha256": hashlib.sha256(b).hexdigest()} for f, b in F.items()}, "overlap64": {}}
|
| 21 |
+
ht = hashes(F["train"]); hs = np.sort(ht)
|
| 22 |
+
for k, b in W.items():
|
| 23 |
+
hw = hashes(b); u = np.unique(hw)
|
| 24 |
+
i = np.searchsorted(hs, u); i = np.minimum(i, len(hs)-1); cand = u[hs[i] == u]
|
| 25 |
+
# confirmation octet par octet (anti-collision)
|
| 26 |
+
tset = set(cand.tolist()); pos_t = {}
|
| 27 |
+
for p in np.nonzero(np.isin(ht, cand))[0]: pos_t.setdefault(int(ht[p]), []).append(int(p))
|
| 28 |
+
conf = 0
|
| 29 |
+
for p in np.nonzero(np.isin(hw, cand))[0]:
|
| 30 |
+
seg = b[p:p+K]
|
| 31 |
+
if any(F["train"][q:q+K] == seg for q in pos_t.get(int(hw[p]), ())): conf += 1
|
| 32 |
+
rep["overlap64"][k] = {"window_bytes": len(b), "n_64grams": int(len(hw)), "hash_hits": int(np.isin(hw, cand).sum()), "confirmed_positions": conf}
|
| 33 |
+
print(k, rep["overlap64"][k], flush=True)
|
| 34 |
+
rep["valid_zero_overlap"] = all(v["confirmed_positions"] == 0 for v in rep["overlap64"].values()) and "fc2_ERROR" not in W
|
| 35 |
+
m = f"{C}/corpus_manifest.json" if os.path.exists(f"{C}/corpus_manifest.json") else f"{C}/manifest.json"
|
| 36 |
+
if os.path.exists(m): rep["manifest_checker"] = json.load(open(m))
|
| 37 |
+
json.dump(rep, open(OUT, "w"), indent=1); print("VALID" if rep["valid_zero_overlap"] else "INVALID")
|