fbfhbfbfbv / day35 /scripts /gl_switchover.sh

Duplicate from td-builder/RSI

77f690c 2 months ago

4.39 kB

#!/bin/bash
# gl_switchover.sh — day-35 hot-swap watcher (runs pod-side in tmux 'sw').
# Waits for the gen-1 record (the decisive datapoint, collected under the original
# high-fidelity settings), then kills the slow loop and relaunches run_gl.sh with the
# fast code (adaptive probe, no base probe, K=4 harvest + solved-cache). The probe/stage
# caches make the restart re-pay NOTHING: gen0+gen1 probes cache-HIT, gen1 grown dir,
# corpus and adapter are reused, so the new loop lands directly at gen-2 under fast settings.
set -u
OUT=/workspace/RSI/outputs/gen_loop.jsonl
WD=/workspace/RSI/expanded_models/gen_loop
echo "[sw] watching $OUT for the gen-1 record ..."
while true; do
  if [ -f "$OUT" ] && grep -q '"gen": 1' "$OUT" && grep '"gen": 1' "$OUT" | grep -q '"ceiling_avg"'; then
    echo "[sw] gen-1 record landed:"; grep '"gen": 1' "$OUT" | tail -1
    break
  fi
  # if the old loop died before logging gen-1, still switch over (restart resumes stages)
  if ! tmux has-session -t cr 2>/dev/null; then
    echo "[sw] 'cr' session GONE before gen-1 record — switching over anyway (resume via caches)"
    break
  fi
  sleep 60
done
echo "[sw] killing old loop + seeding probe cache from banked results ..."
tmux kill-session -t cr 2>/dev/null; sleep 5
/venv/main/bin/python3 - <<'PY'
import json, os
WD = "/workspace/RSI/expanded_models/gen_loop"
OUT = "/workspace/RSI/outputs/gen_loop.jsonl"
cp = os.path.join(WD, "probe_cache.json")
try:
    cache = json.load(open(cp))
except Exception:
    cache = {}
rows = [json.loads(l) for l in open(OUT) if l.strip()] if os.path.exists(OUT) else []
for r in rows:
    if "ceiling_avg" not in r:
        continue
    g = r["gen"]
    if g == 0:
        key = f"gen0|{r['model']}|"
    else:
        key = f"gen{g}|{r['grown_dir']}|{r.get('fill_adapter','')}"
    cache.setdefault(key, {"avg": r["ceiling_avg"], "probes": r.get("probes", [])})
    # also seed any grown-base probes so a re-audit run would cache-hit too
    if r.get("grown_base_ceiling") is not None:
        cache.setdefault(f"gen{g}_base|{r['grown_dir']}|",
                         {"avg": r["grown_base_ceiling"], "probes": r.get("grown_base_probes", [])})
json.dump(cache, open(cp, "w"), indent=1)
print("[sw] probe cache seeded:", list(cache.keys()))
PY
# ---- RULER CALIBRATION (GPU is free right now — the one cheap window) --------------
# Rerun a KNOWN probe (gen1_grown base, salt=0, previously 15/60 at util 0.45) at util 0.80.
# VLLM_DETERMINISTIC=1 is batch-invariant by design; this PROVES it on our exact stack.
# Exact match -> gl_flags.env flips GL_PROBE_UTIL=0.80 (~2x faster sealed-ruler probes).
# Mismatch -> ruler stays at 0.45 and we have learned the ruler is batch-sensitive (logged).
EXPECT=15
if [ -d "$WD/gen1_grown" ]; then
  echo "[sw] calibration: probe gen1_grown salt=0 at GPU_UTIL=0.80 (expect exactly $EXPECT) ..."
  cd /workspace/RSI
  export HF_HOME=/workspace/.hf_home
  export LD_LIBRARY_PATH=/venv/main/lib/python3.10/site-packages/nvidia/cu13/lib:$LD_LIBRARY_PATH
  export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
  export VLLM_DETERMINISTIC=1 VLLM_USE_FLASHINFER_SAMPLER=0 PYTHONPATH=/workspace/RSI
  MODEL="$WD/gen1_grown" ADAPTER= SALT=0 CEIL_K=8 HOLD_N=60 CR_TEMP=0.8 MAXTOK=700 \
    MAX_MODEL_LEN=4096 GEN_CHUNK=240 GPU_UTIL=0.80 HOLD_SET=hard_holdout \
    VERIFY_TC=15 VERIFY_TO=5 GRADE_WORKERS=32 \
    PO_OUT=/workspace/RSI/outputs/calib_util080.jsonl \
    /venv/main/bin/python3 scripts/probe_once.py 2>&1 | tail -3
  GOT=$(/venv/main/bin/python3 -c "import json;print(json.loads(open('/workspace/RSI/outputs/calib_util080.jsonl').read().strip().splitlines()[-1])['solved'])" 2>/dev/null || echo "ERR")
  if [ "$GOT" = "$EXPECT" ]; then
    echo "export GL_PROBE_UTIL=0.80  # calibrated $(date -u +%FT%TZ): util-0.80 probe == util-0.45 probe ($GOT/$EXPECT) exact" \
      > /workspace/RSI/outputs/gl_flags.env
    echo "[sw] CALIBRATION PASS ($GOT == $EXPECT) — fast sealed-ruler probes UNLOCKED"
  else
    echo "[sw] CALIBRATION MISMATCH (got $GOT expected $EXPECT) — ruler stays at 0.45 (batch-sensitive!)"
  fi
else
  echo "[sw] calibration skipped: $WD/gen1_grown missing"
fi

echo "[sw] relaunching fast loop in tmux 'cr' ..."
tmux new-session -d -s cr 'bash /workspace/RSI/scripts/run_gl.sh 2>&1 | tee -a /workspace/RSI/outputs/gen_loop.log'
echo "[sw] DONE — fast loop live. This watcher exits."

	#!/bin/bash
	# gl_switchover.sh — day-35 hot-swap watcher (runs pod-side in tmux 'sw').
	# Waits for the gen-1 record (the decisive datapoint, collected under the original
	# high-fidelity settings), then kills the slow loop and relaunches run_gl.sh with the
	# fast code (adaptive probe, no base probe, K=4 harvest + solved-cache). The probe/stage
	# caches make the restart re-pay NOTHING: gen0+gen1 probes cache-HIT, gen1 grown dir,
	# corpus and adapter are reused, so the new loop lands directly at gen-2 under fast settings.
	set -u
	OUT=/workspace/RSI/outputs/gen_loop.jsonl
	WD=/workspace/RSI/expanded_models/gen_loop
	echo "[sw] watching $OUT for the gen-1 record ..."
	while true; do
	if [ -f "$OUT" ] && grep -q '"gen": 1' "$OUT" && grep '"gen": 1' "$OUT" \| grep -q '"ceiling_avg"'; then
	echo "[sw] gen-1 record landed:"; grep '"gen": 1' "$OUT" \| tail -1
	break
	fi
	# if the old loop died before logging gen-1, still switch over (restart resumes stages)
	if ! tmux has-session -t cr 2>/dev/null; then
	echo "[sw] 'cr' session GONE before gen-1 record — switching over anyway (resume via caches)"
	break
	fi
	sleep 60
	done
	echo "[sw] killing old loop + seeding probe cache from banked results ..."
	tmux kill-session -t cr 2>/dev/null; sleep 5
	/venv/main/bin/python3 - <<'PY'
	import json, os
	WD = "/workspace/RSI/expanded_models/gen_loop"
	OUT = "/workspace/RSI/outputs/gen_loop.jsonl"
	cp = os.path.join(WD, "probe_cache.json")
	try:
	cache = json.load(open(cp))
	except Exception:
	cache = {}
	rows = [json.loads(l) for l in open(OUT) if l.strip()] if os.path.exists(OUT) else []
	for r in rows:
	if "ceiling_avg" not in r:
	continue
	g = r["gen"]
	if g == 0:
	key = f"gen0\|{r['model']}\|"
	else:
	key = f"gen{g}\|{r['grown_dir']}\|{r.get('fill_adapter','')}"
	cache.setdefault(key, {"avg": r["ceiling_avg"], "probes": r.get("probes", [])})
	# also seed any grown-base probes so a re-audit run would cache-hit too
	if r.get("grown_base_ceiling") is not None:
	cache.setdefault(f"gen{g}_base\|{r['grown_dir']}\|",
	{"avg": r["grown_base_ceiling"], "probes": r.get("grown_base_probes", [])})
	json.dump(cache, open(cp, "w"), indent=1)
	print("[sw] probe cache seeded:", list(cache.keys()))
	PY
	# ---- RULER CALIBRATION (GPU is free right now — the one cheap window) --------------
	# Rerun a KNOWN probe (gen1_grown base, salt=0, previously 15/60 at util 0.45) at util 0.80.
	# VLLM_DETERMINISTIC=1 is batch-invariant by design; this PROVES it on our exact stack.
	# Exact match -> gl_flags.env flips GL_PROBE_UTIL=0.80 (~2x faster sealed-ruler probes).
	# Mismatch -> ruler stays at 0.45 and we have learned the ruler is batch-sensitive (logged).
	EXPECT=15
	if [ -d "$WD/gen1_grown" ]; then
	echo "[sw] calibration: probe gen1_grown salt=0 at GPU_UTIL=0.80 (expect exactly $EXPECT) ..."
	cd /workspace/RSI
	export HF_HOME=/workspace/.hf_home
	export LD_LIBRARY_PATH=/venv/main/lib/python3.10/site-packages/nvidia/cu13/lib:$LD_LIBRARY_PATH
	export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
	export VLLM_DETERMINISTIC=1 VLLM_USE_FLASHINFER_SAMPLER=0 PYTHONPATH=/workspace/RSI
	MODEL="$WD/gen1_grown" ADAPTER= SALT=0 CEIL_K=8 HOLD_N=60 CR_TEMP=0.8 MAXTOK=700 \
	MAX_MODEL_LEN=4096 GEN_CHUNK=240 GPU_UTIL=0.80 HOLD_SET=hard_holdout \
	VERIFY_TC=15 VERIFY_TO=5 GRADE_WORKERS=32 \
	PO_OUT=/workspace/RSI/outputs/calib_util080.jsonl \
	/venv/main/bin/python3 scripts/probe_once.py 2>&1 \| tail -3
	GOT=$(/venv/main/bin/python3 -c "import json;print(json.loads(open('/workspace/RSI/outputs/calib_util080.jsonl').read().strip().splitlines()[-1])['solved'])" 2>/dev/null \|\| echo "ERR")
	if [ "$GOT" = "$EXPECT" ]; then
	echo "export GL_PROBE_UTIL=0.80 # calibrated $(date -u +%FT%TZ): util-0.80 probe == util-0.45 probe ($GOT/$EXPECT) exact" \
	> /workspace/RSI/outputs/gl_flags.env
	echo "[sw] CALIBRATION PASS ($GOT == $EXPECT) — fast sealed-ruler probes UNLOCKED"
	else
	echo "[sw] CALIBRATION MISMATCH (got $GOT expected $EXPECT) — ruler stays at 0.45 (batch-sensitive!)"
	fi
	else
	echo "[sw] calibration skipped: $WD/gen1_grown missing"
	fi

	echo "[sw] relaunching fast loop in tmux 'cr' ..."
	tmux new-session -d -s cr 'bash /workspace/RSI/scripts/run_gl.sh 2>&1 \| tee -a /workspace/RSI/outputs/gen_loop.log'
	echo "[sw] DONE — fast loop live. This watcher exits."