thefinalboss commited on
Commit
8a557c0
·
verified ·
1 Parent(s): d82240a

Hashing: verify_v3 (fixed, all positions per hash), pipeline_v3, train_cov

Browse files
results/hashing/code/SHA256SUMS ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ a135948fc8c2464844c6ce3b3b0bf174ce7131b452ca000c40a9b6d4b71bb9e0 code/pipeline_v3.sh
2
+ 73fa1d199c1b41304b9de5ef8b120aceeaffc2cb1d6455b50de4a1b888417de5 code/train_cov.py
3
+ 26d598eebf6763d3d28b274e1385477d162ef99abae7ef4f019f78e231710349 code/verify_v3.py
results/hashing/code/pipeline_v3.sh ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # SPIRA Hashing : attente corpus_v3 (10 min x 12 = 2 h max), verif, puis si VALID : train v2b concat cov (bras b) + eval + McNemar.
3
+ export OMP_NUM_THREADS=1 PYTHONHASHSEED=0 TZ=America/Toronto
4
+ H=/workspace/spira_shared/spira_v3verify; PY=/workspace/SPIRA-venv/bin/python; cd $H
5
+ log(){ echo "$(date '+%F %H:%M:%S %Z') $*" >> $H/pipeline.log; }
6
+ C=""; prev=""
7
+ for i in $(seq 1 481); do
8
+ for d in /workspace/spira_shared/corpus_v3 /workspace/spira_shared/broad/broad_gutenberg_v1 /workspace/spira_shared/checker/corpus_v3; do
9
+ [ -f $d/train.bin ] && { [ -f $d/manifest.json ] || [ -f $d/corpus_manifest.json ]; } && C=$d && break
10
+ done
11
+ if [ -n "$C" ]; then cur=$(stat -c '%s%Y' $C/*.bin $C/*manifest*.json | tr -d '\n'); [ "$cur" = "$prev" ] && break; prev=$cur; log "corpus vu $C, attente stabilite"; else log "check $i : corpus_v3 absent"; fi
12
+ [ $i -eq 481 ] && { log "TIMEOUT 8h : corpus_v3 non pret"; exit 3; }
13
+ sleep 60
14
+ done
15
+ log "verif $C"; $PY verify_v3.py $C verify_corpus_v3.json > verify.log 2>&1
16
+ grep -q '^VALID' verify.log || { log "INVALIDE ou erreur : pas d'entrainement (voir verify.log)"; exit 4; }
17
+ av=$(awk '/MemAvailable/{print int($2/1048576)}' /proc/meminfo); until [ $(awk '/MemAvailable/{print int($2/1048576)}' /proc/meminfo) -ge 5 ]; do log "RAM < 5G, attente"; sleep 60; done
18
+ log "VALID, lancement train (RAM dispo ${av}G)"
19
+ cd /workspace/spira_shared/spira_cov && $PY train_cov.py $C/train.bin $H/cm_v3_cov_b.npz b > $H/train_v3_cov_b.log 2>&1 || { log "train ECHEC"; exit 1; }
20
+ cd $H; log "train fini, eval"
21
+ $PY eval_win.py cm_v3_cov_b.npz $C/train.bin $C/test.bin v3cov hits > eval_v3.log 2>&1 || log "eval new ECHEC"
22
+ for r in "/workspace/spira_shared/spira_m8/cm_broad_local_v1.npz m8broad" "/workspace/spira_shared/spira_cov/cm_broad_cov_b.npz covb"; do
23
+ set -- $r; for k in $(seq 1 72); do [ -f $1 ] && break; sleep 600; done
24
+ [ -f $1 ] || { log "ref $2 absente apres 12h"; continue; }
25
+ $PY eval_win.py $1 /workspace/spira_shared/broad/broad_local_v1/train.bin $C/test.bin $2 hits > eval_$2.log 2>&1 || log "eval $2 ECHEC"
26
+ $PY mcnemar_win.py hits v3cov $2 mcnemar_v3cov_vs_$2.json > mcnemar_$2.log 2>&1; log "mcnemar vs $2 fait"
27
+ done; log "FIN"
results/hashing/code/train_cov.py ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Copie de checker/train_scale_v2.py (recette fixed2+cal, config repo par defaut, d_in=1024, vocab 16384)
2
+ ou m.prepare()/m.train() sont remplaces par cov_prepare.prepare_cov/train_cov (bras a|b).
3
+ Usage: train_cov.py DATA OUT ARM"""
4
+ import json, sys, time, os
5
+ sys.path.insert(0, "/workspace/SPIRA/python"); sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
6
+ import numpy as np
7
+ from spira.model import SPIRA, SPIRAConfig
8
+ from cov_prepare import prepare_cov, train_cov
9
+ data_p, out, arm = sys.argv[1:4]
10
+ cfg = SPIRAConfig(vocab_size=16384, d_in=1024, snn_tokens=600_000, am_tokens=1_200_000, epochs=2)
11
+ data = open(data_p, "rb").read()
12
+ t0 = time.perf_counter(); log = lambda s: print(f"[{time.perf_counter()-t0:7.0f}s] {s}", flush=True)
13
+ m = SPIRA.build(data, cfg, log=log); cov = prepare_cov(m, arm=arm, log=log); m._patches_train = None
14
+ hist = train_cov(m, log_fn=log, ckpt_path=None)
15
+ cal = m.calibrate_fusion(data[:60_000]); log(f"calibration {cal} beta={m.core.beta}")
16
+ m.save(out)
17
+ chk = SPIRA.load(out).fusion; assert chk["scale"] == cal["scale"], (chk, cal); log(f"ckpt fusion verifie {chk}")
18
+ json.dump({"script": "spira_cov/train_cov", "arm": arm, "config": vars(cfg), "cov": {k: cov[k] for k in ("arm","N","seed","k")},
19
+ "snn_slices": cov["snn_slices"], "am_slices": cov["am_slices"], "fusion": cal, "meta": m.meta,
20
+ "wall_s": time.perf_counter()-t0, "history_epochs": hist.get("epochs"), "am": hist.get("am")},
21
+ open(out + ".cov.json", "w"), indent=1, default=str)
22
+ log(f"saved {out}")
results/hashing/code/verify_v3.py ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """SPIRA Hashing - verification independante (lecture seule) d'un corpus: tailles, sha256, overlap 64 o.
2
+ Usage: verify_v3.py CORPUS_DIR OUT_JSON"""
3
+ import sys, os, json, hashlib, numpy as np
4
+ sys.path.insert(0, "/workspace/spira_shared/checker")
5
+ C, OUT = sys.argv[1:3]; K = 64; P = np.uint64(1099511628211)
6
+ def hashes(b):
7
+ a = np.frombuffer(b, np.uint8).astype(np.uint64); n = len(a) - K + 1
8
+ if n <= 0: return np.zeros(0, np.uint64)
9
+ h = np.zeros(n, np.uint64)
10
+ with np.errstate(over="ignore"):
11
+ for j in range(K): h = h * P + a[j:j+n] + np.uint64(1)
12
+ return h
13
+ W = {k: open(f"/workspace/spira_shared/spira_m7/windows/{k}.bin","rb").read() for k in ("fresh_en","fresh_fr","neither_code","test20k","queries")}
14
+ try:
15
+ from windows_v2 import freshcode_v2; W["fc2"] = freshcode_v2()
16
+ except Exception as e: W["fc2_ERROR"] = b""; print("fc2 indisponible", e)
17
+ F = {f: open(f"{C}/{f}.bin","rb").read() for f in ("train","test","oneshot") if os.path.exists(f"{C}/{f}.bin")}
18
+ for _k in ("test","oneshot"):
19
+ if _k in F: W["corpus_"+_k] = F[_k]
20
+ rep = {"corpus": C, "files": {f: {"bytes": len(b), "sha256": hashlib.sha256(b).hexdigest()} for f, b in F.items()}, "overlap64": {}}
21
+ ht = hashes(F["train"]); hs = np.sort(ht)
22
+ for k, b in W.items():
23
+ hw = hashes(b); u = np.unique(hw)
24
+ i = np.searchsorted(hs, u); i = np.minimum(i, len(hs)-1); cand = u[hs[i] == u]
25
+ # confirmation octet par octet (anti-collision)
26
+ tset = set(cand.tolist()); pos_t = {}
27
+ for p in np.nonzero(np.isin(ht, cand))[0]: pos_t.setdefault(int(ht[p]), []).append(int(p))
28
+ conf = 0
29
+ for p in np.nonzero(np.isin(hw, cand))[0]:
30
+ seg = b[p:p+K]
31
+ if any(F["train"][q:q+K] == seg for q in pos_t.get(int(hw[p]), ())): conf += 1
32
+ rep["overlap64"][k] = {"window_bytes": len(b), "n_64grams": int(len(hw)), "hash_hits": int(np.isin(hw, cand).sum()), "confirmed_positions": conf}
33
+ print(k, rep["overlap64"][k], flush=True)
34
+ rep["valid_zero_overlap"] = all(v["confirmed_positions"] == 0 for v in rep["overlap64"].values()) and "fc2_ERROR" not in W
35
+ m = f"{C}/corpus_manifest.json" if os.path.exists(f"{C}/corpus_manifest.json") else f"{C}/manifest.json"
36
+ if os.path.exists(m): rep["manifest_checker"] = json.load(open(m))
37
+ json.dump(rep, open(OUT, "w"), indent=1); print("VALID" if rep["valid_zero_overlap"] else "INVALID")