"""Remote watchdog: auto-run the control eval right after v3 training finishes. Polls /content/qwenjev_work_v3/v3_status.json every 5 min. When current == ALL_DONE, launches run_eval on ar25+bp35 (same params as the zero-shot baseline: 150 actions, hf backend) and writes /content/eval_v3_control.json. """ import json import subprocess import time STATUS = "/content/qwenjev_work_v3/v3_status.json" OUT = "/content/eval_v3_control.json" CMD = ("cd /content/qwenjev && python3 -u ops/run_eval.py " "--games ar25 bp35 --max-actions 150 " "--checkpoint checkpoints/qwenjev-s1-hf-v3 --backend hf " f"--out {OUT} >> /content/eval_v3.log 2>&1") while True: try: st = json.load(open(STATUS)) cur = st.get("current", "") print(f"[watchdog] current={cur}", flush=True) if cur == "ALL_DONE": print("[watchdog] training ALL_DONE -> launching control eval", flush=True) subprocess.run(["bash", "-c", CMD]) print("[watchdog] control eval finished -> " + OUT, flush=True) break if cur == "FAILED": print("[watchdog] training reported FAILED, aborting", flush=True) break except FileNotFoundError: print("[watchdog] no status file yet", flush=True) except Exception as exc: print(f"[watchdog] err: {exc}", flush=True) time.sleep(300)