v3_scale: code banc v4/gen/v3, rapports validés, corpus_v3 (sha256 vérifié), README
519e2ef verified | # Checker — watcher scale v2 (remplace v1, sauvegardé logs/watch_scale.v1.sh.bak). 1 cœur, nice 10, OMP=1, PYTHONHASHSEED=0. | |
| # Boucle : pour chaque ligne de jobs_v4.tsv (tag ckpt corpus), quand le .npz existe, est stable ≥120 s et se charge : | |
| # vérifie fusion.scale ∈ [0,03 ; 0,11] (enregistré), puis eval_bench_v4.py (8 fenêtres fixes) → scale_bench_v4.py, puis gen/gen_eval.py. | |
| # Un seul job à la fois ; seulement si < 4 lourds Checker ET MemAvailable ≥ 4 Go. (Bloc M8 FULL de v1 conservé, déjà déclenché.) | |
| export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 PYTHONHASHSEED=0 | |
| D=/workspace/spira_shared/checker/scale; G=/workspace/spira_shared/checker/gen; PY=/workspace/SPIRA-venv/bin/python; cd $D | |
| log(){ echo "$(TZ=America/Toronto date '+%F %H:%M') $*" >> $D/watch_scale.log; } | |
| slot(){ [ $(bash $D/heavy_checker.sh) -lt 4 ] && [ $(awk '/MemAvailable/{print int($2/1048576)}' /proc/meminfo) -ge 4 ]; } | |
| log "démarrage watcher v2 pid $$" | |
| while true; do | |
| while IFS=$'\t' read -r tag ck corp; do | |
| [ -z "$tag" ] && continue; [ -f logs/v4_$tag.done ] && continue | |
| [ -f "$ck" ] && [ $(( $(date +%s) - $(stat -c %Y "$ck") )) -ge 120 ] || continue | |
| slot || continue | |
| sc=$($PY -c "import numpy as np,json,sys;z=np.load(sys.argv[1],allow_pickle=False);print(json.loads(str(z['fusion']))['scale'] if 'fusion' in z.files else 'NA')" "$ck" 2>/dev/null) || continue | |
| ok=$($PY -c "import sys;s=sys.argv[1];print('OK' if s!='NA' and 0.03<=float(s)<=0.11 else 'HORS_PLAGE')" "$sc" 2>/dev/null) | |
| log "$tag ckpt prêt, fusion.scale(sauvé)=$sc → $ok ; banc v4" | |
| nice -n 10 timeout 10800 $PY $D/eval_bench_v4.py "$ck" "$corp" $tag > logs/v4_$tag.log 2>&1; rc=$?; log "$tag banc v4 EXIT $rc" | |
| $PY $D/scale_bench_v4.py > logs/bench_v4_last.log 2>&1; log "banc v4 mis à jour" | |
| if [ $rc = 0 ]; then | |
| [ -f $G/logs/gen_$tag.log ] || (cd $G && nice -n 10 timeout 7200 $PY -u $G/gen_eval.py "$ck" "$corp" $tag > $G/logs/gen_$tag.log 2>&1); log "$tag génération EXIT $?" | |
| echo "$sc $ok" > logs/v4_$tag.done | |
| fi | |
| done < jobs_v4.tsv | |
| sleep 120 | |
| done | |