| """Remote watchdog (v4): auto-run the control eval right after v4 real_c finishes. |
| |
| Same contract as watchdog_eval_v3.py, pointed at the v4 checkpoint |
| (vision-aware pooling fix, recovered via v4_recover_bootstrap.py). |
| |
| Polls /content/qwenjev_work_v3/v3_status.json every 5 min. When current == |
| ALL_DONE, launches run_eval on ar25+bp35 (same params as the v3 control and |
| the zero-shot baseline: 150 actions, hf backend) and writes |
| /content/eval_v4_control.json. |
| """ |
| import json |
| import os |
| import subprocess |
| import time |
|
|
| STATUS = "/content/qwenjev_work_v3/v3_status.json" |
| OUT = "/content/eval_v4_control.json" |
| |
| CKPT = "checkpoints/" + os.environ.get("QWENJEV_OUT", "qwenjev-s1-hf-v4") |
| CMD = ("cd /content/qwenjev && python3 -u ops/run_eval.py " |
| "--games ar25 bp35 --max-actions 150 " |
| f"--checkpoint {CKPT} --backend hf " |
| f"--out {OUT} >> /content/eval_v4.log 2>&1") |
|
|
| while True: |
| try: |
| st = json.load(open(STATUS)) |
| cur = st.get("current", "") |
| print(f"[watchdog-v4] current={cur}", flush=True) |
| if cur == "ALL_DONE": |
| print("[watchdog-v4] training ALL_DONE -> launching control eval", |
| flush=True) |
| subprocess.run(["bash", "-c", CMD]) |
| print("[watchdog-v4] control eval finished -> " + OUT, flush=True) |
| break |
| if cur == "FAILED": |
| print("[watchdog-v4] training reported FAILED, aborting", flush=True) |
| break |
| except FileNotFoundError: |
| print("[watchdog-v4] no status file yet", flush=True) |
| except Exception as exc: |
| print(f"[watchdog-v4] err: {exc}", flush=True) |
| time.sleep(300) |
|
|
