File size: 1,406 Bytes
c8e52d0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 | """Remote watchdog: auto-run the control eval right after v3 training finishes.
Polls /content/qwenjev_work_v3/v3_status.json every 5 min. When current ==
ALL_DONE, launches run_eval on ar25+bp35 (same params as the zero-shot
baseline: 150 actions, hf backend) and writes /content/eval_v3_control.json.
"""
import json
import subprocess
import time
STATUS = "/content/qwenjev_work_v3/v3_status.json"
OUT = "/content/eval_v3_control.json"
CMD = ("cd /content/qwenjev && python3 -u ops/run_eval.py "
"--games ar25 bp35 --max-actions 150 "
"--checkpoint checkpoints/qwenjev-s1-hf-v3 --backend hf "
f"--out {OUT} >> /content/eval_v3.log 2>&1")
while True:
try:
st = json.load(open(STATUS))
cur = st.get("current", "")
print(f"[watchdog] current={cur}", flush=True)
if cur == "ALL_DONE":
print("[watchdog] training ALL_DONE -> launching control eval",
flush=True)
subprocess.run(["bash", "-c", CMD])
print("[watchdog] control eval finished -> " + OUT, flush=True)
break
if cur == "FAILED":
print("[watchdog] training reported FAILED, aborting", flush=True)
break
except FileNotFoundError:
print("[watchdog] no status file yet", flush=True)
except Exception as exc:
print(f"[watchdog] err: {exc}", flush=True)
time.sleep(300)
|