File size: 1,718 Bytes
33c6b95 2dd02cc 33c6b95 2dd02cc 33c6b95 2dd02cc 33c6b95 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | """Remote watchdog (v4): auto-run the control eval right after v4 real_c finishes.
Same contract as watchdog_eval_v3.py, pointed at the v4 checkpoint
(vision-aware pooling fix, recovered via v4_recover_bootstrap.py).
Polls /content/qwenjev_work_v3/v3_status.json every 5 min. When current ==
ALL_DONE, launches run_eval on ar25+bp35 (same params as the v3 control and
the zero-shot baseline: 150 actions, hf backend) and writes
/content/eval_v4_control.json.
"""
import json
import os
import subprocess
import time
STATUS = "/content/qwenjev_work_v3/v3_status.json"
OUT = "/content/eval_v4_control.json"
# v0.4.10: namespace-driven so the same watchdog serves v4 and v4b reruns
CKPT = "checkpoints/" + os.environ.get("QWENJEV_OUT", "qwenjev-s1-hf-v4")
CMD = ("cd /content/qwenjev && python3 -u ops/run_eval.py "
"--games ar25 bp35 --max-actions 150 "
f"--checkpoint {CKPT} --backend hf "
f"--out {OUT} >> /content/eval_v4.log 2>&1")
while True:
try:
st = json.load(open(STATUS))
cur = st.get("current", "")
print(f"[watchdog-v4] current={cur}", flush=True)
if cur == "ALL_DONE":
print("[watchdog-v4] training ALL_DONE -> launching control eval",
flush=True)
subprocess.run(["bash", "-c", CMD])
print("[watchdog-v4] control eval finished -> " + OUT, flush=True)
break
if cur == "FAILED":
print("[watchdog-v4] training reported FAILED, aborting", flush=True)
break
except FileNotFoundError:
print("[watchdog-v4] no status file yet", flush=True)
except Exception as exc:
print(f"[watchdog-v4] err: {exc}", flush=True)
time.sleep(300)
|