scripts_full/classify_all.sh

#!/bin/bash
# classify_all.sh: full classify slate pipeline, self-contained; uploads logs + artifacts to HF as it goes
cd $HOME/mt92
export TOKENIZERS_PARALLELISM=false OMP_NUM_THREADS=8
set -a; . ./.env; set +a
export CAL=ex/cl_train0.jsonl
LOG=$HOME/mt92/classify_all.log
up() { ./venv/bin/python - <<PY
import os
from huggingface_hub import HfApi
HfApi(token=os.environ["HF_TOKEN"]).upload_file(path_or_fileobj="$LOG", path_in_repo="scripts_full/classify_all.log", repo_id="dasLOL/mt92-extract-work", commit_message="classify log")
PY
}
say() { echo "[$(date -u +%H:%M:%S)] $*" | tee -a $LOG; }
python3 - <<PY
import json, random
rows = [json.loads(l) for l in open("ex/cl_full100.jsonl")]
keep = [r for r in rows if not r["ref"].startswith("clinc-validation") and not r["ref"].startswith("clinc-test")]
random.seed(3); random.shuffle(keep)
open("ex/cl_train0.jsonl", "w").write("".join(json.dumps(r) + "\n" for r in keep))
PY
acc() { ./venv/bin/python evalcl.py $1 ex/$2.jsonl 2>&1 | grep -oE "ACCURACY [0-9.]+" | awk '{print $2}'; }
tr() { # name base epochs set -> prints "val test"
  [ -d $1 ] || OUT=$1 BASE=$2 EPOCHS=$3 LR=8e-5 TRAIN=$4 ./venv/bin/python sft.py > train_$1.log 2>&1
  V=$(acc $1 cl_validation_all); T=$(acc $1 cl_test_all); say "$1 ($2, $3 ep, $4): val $V test $T"; echo "$V $T"; }
say "phase A: base models"
tr cm28 "" 4 ex/cl_train0.jsonl > /dev/null
for k in 2 4 8 16; do [ -d p${k}c ] || ./venv/bin/python prune.py cm28 p${k}c $k > prune_$k.log 2>&1; done
U2=$(tr u2 p2c 8 ex/cl_train0.jsonl | tail -1); U4=$(tr u4 p4c 4 ex/cl_train0.jsonl | tail -1); U8=$(tr u8 p8c 4 ex/cl_train0.jsonl | tail -1); U16=$(tr u16 p16c 4 ex/cl_train0.jsonl | tail -1)
up
pick() { # u target -> nearest available coverage file pct
  python3 - "$1" "$2" <<PY
import sys
u = sum(map(float, sys.argv[1].split()))/2; t = float(sys.argv[2])
c = 0.0 if u >= t else min(0.95, (t - u) / max(1e-6, 1.0 - u))
avail = [0,5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,86,90]
print(min(avail, key=lambda a: abs(a - 100*c)))
PY
}
say "phase B: coverage-tuned slate (targets D 0.950, B 0.960, C 0.968; A = plain 4L)"
PD=$(pick "$U2" 0.950); PB=$(pick "$U8" 0.960); PC=$(pick "$U16" 0.968)
say "coverages: D(2L)=$PD% B(8L)=$PB% C(16L)=$PC%"
setfile() { [ "$1" = "0" ] && echo ex/cl_train0.jsonl || echo ex/cl_full$1.jsonl; }
RD=$(tr cld2 p2c 16 $(setfile $PD) | tail -1); RB=$(tr clb8 p8c 8 $(setfile $PB) | tail -1); RC=$(tr clc16 p16c 4 $(setfile $PC) | tail -1)
cp -r u4 cla4 2>/dev/null || true
up
say "phase C: ONNX build + profile + stage"
for M in cld2 cla4 clb8 clc16; do
  bash ./build_final.sh $M 32 > build_$M.log 2>&1 || say "build $M FAILED"
  cp manifest_template.json onnx_$M/final/manifest.json
  ./venv/bin/python calib.py onnx_$M/final 60 > calib_$M.log 2>&1; say "$M calib: $(grep -v '^probe' calib_$M.log | tr -d '\n' | head -c 200)"
  ./venv/bin/python up_art.py onnx_$M/final onnx_$M > up_$M.log 2>&1 && say "staged onnx_$M"
  ./venv/bin/python backup_now.py $M > backup_$M.log 2>&1 && say "$M backed up"
  up
done
say "CLASSIFY_ALL_DONE"; up