SPIRA / v3_scale /code /scale /hf_push_new.sh
thefinalboss's picture
v3_scale: watcher HF léger hf_push_new.sh
6c4984a verified
Raw History Blame Contribute Delete
1.47 kB
#!/bin/bash
# Checker — watcher léger HF (nice 10, 1 cœur). Toutes les 5 min : pour chaque tag de jobs_v4.tsv dont logs/v4_<tag>.done
# indique "OK" (fusion.scale ∈ [0,03;0,11], banc v4 EXIT 0, posé par watch_scale.sh) et pas encore poussé (hf_pushed.list),
# pousse le checkpoint (+ .summary.json/.v2.json) et les rapports banc v4 à jour sous v3_scale/ via hf_push_new.py.
export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1
D=/workspace/spira_shared/checker/scale; PY=/workspace/SPIRA-venv/bin/python; cd $D; touch hf_pushed.list
echo "$(TZ=America/Toronto date '+%F %H:%M') watcher HF démarré pid $$" >> hf_push.log
echo -e "old_ref\nbroad14_ref" >> hf_pushed.list; sort -u -o hf_pushed.list hf_pushed.list
while true; do
while IFS=$'\t' read -r tag ck corp; do
[ -z "$tag" ] && continue; grep -qx "$tag" hf_pushed.list && continue
[ -f logs/v4_$tag.done ] && grep -q " OK" logs/v4_$tag.done || continue
b=$(basename "$ck"); A="$ck=v3_scale/checkpoints/$b"
for s in "${ck%.npz}.summary.json" "$ck.v2.json"; do [ -f "$s" ] && A="$A $s=v3_scale/checkpoints/$(basename $s)"; done
for r in scale_bench_v4.md scale_bench_v4.json results_v4.jsonl; do A="$A $D/$r=v3_scale/reports/$r"; done
if nice -n 10 $PY $D/hf_push_new.py "v3_scale: checkpoint $tag terminé et calibré ($(cat logs/v4_$tag.done)) + banc v4 à jour" $A >> hf_push.out 2>&1; then echo "$tag" >> hf_pushed.list; fi
done < jobs_v4.tsv
sleep 300
done