"""Remote watchdog (v4): auto-run the control eval right after v4 real_c finishes. Same contract as watchdog_eval_v3.py, pointed at the v4 checkpoint (vision-aware pooling fix, recovered via v4_recover_bootstrap.py). Polls /content/qwenjev_work_v3/v3_status.json every 5 min. When current == ALL_DONE, launches run_eval on ar25+bp35 (same params as the v3 control and the zero-shot baseline: 150 actions, hf backend) and writes /content/eval_v4_control.json. """ import json import os import subprocess import time STATUS = "/content/qwenjev_work_v3/v3_status.json" OUT = "/content/eval_v4_control.json" # v0.4.10: namespace-driven so the same watchdog serves v4 and v4b reruns CKPT = "checkpoints/" + os.environ.get("QWENJEV_OUT", "qwenjev-s1-hf-v4") CMD = ("cd /content/qwenjev && python3 -u ops/run_eval.py " "--games ar25 bp35 --max-actions 150 " f"--checkpoint {CKPT} --backend hf " f"--out {OUT} >> /content/eval_v4.log 2>&1") while True: try: st = json.load(open(STATUS)) cur = st.get("current", "") print(f"[watchdog-v4] current={cur}", flush=True) if cur == "ALL_DONE": print("[watchdog-v4] training ALL_DONE -> launching control eval", flush=True) subprocess.run(["bash", "-c", CMD]) print("[watchdog-v4] control eval finished -> " + OUT, flush=True) break if cur == "FAILED": print("[watchdog-v4] training reported FAILED, aborting", flush=True) break except FileNotFoundError: print("[watchdog-v4] no status file yet", flush=True) except Exception as exc: print(f"[watchdog-v4] err: {exc}", flush=True) time.sleep(300)