| """Remote watchdog: auto-run the control eval right after v3 training finishes. |
| |
| Polls /content/qwenjev_work_v3/v3_status.json every 5 min. When current == |
| ALL_DONE, launches run_eval on ar25+bp35 (same params as the zero-shot |
| baseline: 150 actions, hf backend) and writes /content/eval_v3_control.json. |
| """ |
| import json |
| import subprocess |
| import time |
|
|
| STATUS = "/content/qwenjev_work_v3/v3_status.json" |
| OUT = "/content/eval_v3_control.json" |
| CMD = ("cd /content/qwenjev && python3 -u ops/run_eval.py " |
| "--games ar25 bp35 --max-actions 150 " |
| "--checkpoint checkpoints/qwenjev-s1-hf-v3 --backend hf " |
| f"--out {OUT} >> /content/eval_v3.log 2>&1") |
|
|
| while True: |
| try: |
| st = json.load(open(STATUS)) |
| cur = st.get("current", "") |
| print(f"[watchdog] current={cur}", flush=True) |
| if cur == "ALL_DONE": |
| print("[watchdog] training ALL_DONE -> launching control eval", |
| flush=True) |
| subprocess.run(["bash", "-c", CMD]) |
| print("[watchdog] control eval finished -> " + OUT, flush=True) |
| break |
| if cur == "FAILED": |
| print("[watchdog] training reported FAILED, aborting", flush=True) |
| break |
| except FileNotFoundError: |
| print("[watchdog] no status file yet", flush=True) |
| except Exception as exc: |
| print(f"[watchdog] err: {exc}", flush=True) |
| time.sleep(300) |
|
|
