Add per-checkpoint eval worker (eval_watch.py + run_eval_watch.sh); ONLY_MODEL mode in eval_adapter.py

eval_adapter.py

@@ -17,6 +17,7 @@ Config via env:
     N_TASKS        how many test tasks (default 50)
     CONCURRENCY    parallel rollouts (default 8; keep <= MAX_CONCURRENT_ENVS)
     OUT            results JSON path (default /app/eval_results.json)
+    ONLY_MODEL     evaluate just this one served model name (per-checkpoint eval)
 """
 
 import json
@@ -31,6 +32,7 @@ VLLM_URL = os.environ.get("VLLM_URL", "http://127.0.0.1:8001")
 TEST_SPLIT = os.environ.get("TEST_SPLIT", "/app/prepared/datasets/test")
 BASE_MODEL = os.environ.get("BASE_MODEL", "LiquidAI/LFM2.5-2.6B")
 ADAPTER_NAME = os.environ.get("ADAPTER_NAME", "trained")
+ONLY_MODEL = os.environ.get("ONLY_MODEL", "")
 N_TASKS = int(os.environ.get("N_TASKS", "50"))
 CONCURRENCY = int(os.environ.get("CONCURRENCY", "8"))
 OUT = os.environ.get("OUT", "/app/eval_results.json")
@@ -69,7 +71,10 @@ def run_one(task_index, model):
 
 
 def main():
-    models = [BASE_MODEL] + ([ADAPTER_NAME] if ADAPTER_NAME else [])
+    if ONLY_MODEL:
+        models = [ONLY_MODEL]
+    else:
+        models = [BASE_MODEL] + ([ADAPTER_NAME] if ADAPTER_NAME else [])
     work = [(i, m) for i in range(N_TASKS) for m in models]
     cells = {}  # (task_index, model) -> {"correct": bool|None, "failed": bool, "error": str|None}
     with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool:

eval_watch.py

@@ -0,0 +1,110 @@
+"""Per-checkpoint eval worker: watch the adapter repo, eval each new checkpoint.
+
+Loop: poll ADAPTER_REPO for checkpoint-N dirs without eval results, hot-load each
+as a LoRA into the already-running vLLM (runtime LoRA updates — no server restart),
+run the held-out eval on a fixed task subset, and upload per-checkpoint results
+plus a rolling eval/summary.json (the live learning curve).
+
+Base-model cells are computed once and reused for every checkpoint.
+
+Env: ADAPTER_REPO (required), BASE_MODEL, N_TASKS (default 100), POLL_S (600).
+Stops when eval/final_results.json appears (the training job's own final eval).
+"""
+
+import json
+import os
+import subprocess
+import time
+import urllib.request
+from pathlib import Path
+
+from huggingface_hub import HfApi, snapshot_download
+
+REPO = os.environ["ADAPTER_REPO"]
+BASE_MODEL = os.environ.get("BASE_MODEL", "LiquidAI/LFM2.5-2.6B")
+POLL_S = int(os.environ.get("POLL_S", "600"))
+VLLM = "http://127.0.0.1:8001"
+
+api = HfApi()
+
+
+def repo_files():
+    return set(api.list_repo_files(REPO))
+
+
+def pending_checkpoints(fs):
+    ckpts = sorted(
+        {f.split("/")[0] for f in fs if f.startswith("checkpoint-") and f.endswith("adapter_config.json")},
+        key=lambda s: int(s.rsplit("-", 1)[1]),
+    )
+    done = {f[len("eval/"):-len("_results.json")] for f in fs if f.startswith("eval/") and f.endswith("_results.json")}
+    return [c for c in ckpts if c not in done]
+
+
+def run_eval(model_name, out_path):
+    env = dict(os.environ, ONLY_MODEL=model_name, OUT=out_path)
+    subprocess.run(["python3", "/app/eval_adapter.py"], env=env, check=True)
+
+
+def upload(local, remote):
+    api.upload_file(path_or_fileobj=str(local), path_in_repo=remote, repo_id=REPO)
+
+
+def lora_rpc(route, name, path=None):
+    payload = {"lora_name": name}
+    if path is not None:
+        payload["lora_path"] = str(path)
+    req = urllib.request.Request(
+        f"{VLLM}/v1/{route}",
+        data=json.dumps(payload).encode(),
+        headers={"Content-Type": "application/json"},
+    )
+    urllib.request.urlopen(req, timeout=900)
+
+
+def pass_at_1(cells_path):
+    cells = json.loads(Path(cells_path).read_text())["cells"]
+    scored = [c["correct"] for c in cells.values() if not c["failed"]]
+    return (sum(scored) / len(scored)) if scored else None, len(scored)
+
+
+def main():
+    fs = repo_files()
+    # 1. Base-model cells, once. Every checkpoint compares against these.
+    if "eval/base_cells.json" not in fs:
+        print("[watch] evaluating base model first", flush=True)
+        run_eval(BASE_MODEL, "/tmp/base_cells.json")
+        upload("/tmp/base_cells.json", "eval/base_cells.json")
+
+    summary = {}
+    if "eval/summary.json" in fs:
+        summary = json.loads(Path(api.hf_hub_download(REPO, "eval/summary.json")).read_text())
+
+    loaded = []
+    while True:
+        fs = repo_files()
+        if "eval/final_results.json" in fs:
+            print("[watch] training job's final eval landed; stopping", flush=True)
+            return
+        for ckpt in pending_checkpoints(fs):
+            print(f"[watch] new checkpoint: {ckpt}", flush=True)
+            local = Path(snapshot_download(REPO, allow_patterns=[f"{ckpt}/*"])) / ckpt
+            lora_rpc("load_lora_adapter", ckpt, local)
+            loaded.append(ckpt)
+            if len(loaded) > 4:  # stay under --max-loras
+                lora_rpc("unload_lora_adapter", loaded.pop(0))
+            run_eval(ckpt, f"/tmp/{ckpt}_cells.json")
+            upload(f"/tmp/{ckpt}_cells.json", f"eval/{ckpt}_results.json")
+            p1, n = pass_at_1(f"/tmp/{ckpt}_cells.json")
+            pb, nb = pass_at_1("/tmp/base_cells.json")
+            summary[ckpt] = {"pass_at_1": p1, "scorable": n,
+                             "base_pass_at_1": pb,
+                             "delta": (p1 - pb) if p1 is not None and pb is not None else None}
+            Path("/tmp/summary.json").write_text(json.dumps(summary, indent=2))
+            upload("/tmp/summary.json", "eval/summary.json")
+            print(f"[watch] {ckpt}: pass@1={p1} (base {pb})", flush=True)
+        time.sleep(POLL_S)
+
+
+if __name__ == "__main__":
+    main()

run_eval_watch.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+# Per-checkpoint eval worker (1 GPU). Polls ADAPTER_REPO for new checkpoint-N
+# dirs and evaluates each on a fixed N_TASKS subset of the held-out test split,
+# writing eval/<checkpoint-N>_results.json + a rolling eval/summary.json.
+#
+# Expected env:
+#   HF_TOKEN       (secret)  hub access + sandbox Jobs
+#   ADAPTER_REPO   (env)     repo the training job pushes checkpoints to
+#   N_TASKS        (env)     test tasks per eval (default 100; same subset every time)
+set -euo pipefail
+cd "$(dirname "$0")"
+
+echo "=== [setup] ==="
+apt-get update -qq && apt-get install -y -qq --no-install-recommends git ca-certificates
+python3 -m pip install --quiet \
+    'openenv[harbor]' \
+    'harbor[hf-sandbox]>=0.24.0' \
+    math-verify tomli-w \
+    aiohttp
+git clone --depth 1 https://github.com/huggingface/OpenEnv.git /opt/OpenEnv
+SITE=$(python3 -c "import site; print(site.getsitepackages()[0])")
+echo "/opt/OpenEnv/envs" > "$SITE/openenv_examples.pth"
+
+echo "=== [patch] opencode seam: session id as x-api-key (Jobs proxy strips Authorization) ==="
+python3 - <<'EOF'
+import openenv.harbor.seams as seams
+
+path = seams.__file__
+src = open(path).read()
+old = '"apiKey": session,\n                        "timeout": 600_000,'
+new = ('"apiKey": session,\n'
+       '                        "headers": {"x-api-key": session},\n'
+       '                        "timeout": 600_000,')
+assert src.count(old) == 1, f"opencode seam not found in {path}"
+open(path, "w").write(src.replace(old, new))
+print("patched", path)
+EOF
+
+echo "=== [setup] pinned test tasks ==="
+python3 prepare_tasks.py --output /app/prepared --splits test
+
+echo "=== [start] vLLM (base; LoRAs hot-loaded per checkpoint) ==="
+CUDA_VISIBLE_DEVICES=0 VLLM_ALLOW_RUNTIME_LORA_UPDATING=1 \
+vllm serve "${BASE_MODEL:-LiquidAI/LFM2.5-2.6B}" \
+    --port 8001 \
+    --max-model-len 32768 \
+    --return-tokens-as-token-ids \
+    --logprobs-mode processed_logprobs \
+    --enable-auto-tool-choice \
+    --tool-call-parser lfm2 \
+    --enable-lora --max-lora-rank 32 --max-loras 8 &
+VLLM_PID=$!
+
+echo "=== [start] env server (test split; tunnel-free via Jobs proxy) ==="
+export SPACE_HOST="${JOB_ID}--8000.hf.jobs"
+PREPARED=/app/prepared OPENENV_DATASETS=/app/prepared/datasets/test bash serve_env.sh &
+ENV_PID=$!
+
+python3 - <<'EOF'
+import time, urllib.request
+for name, url in [("vllm", "http://127.0.0.1:8001/v1/models"),
+                  ("env",  "http://127.0.0.1:8000/health")]:
+    for i in range(360):
+        try:
+            urllib.request.urlopen(url, timeout=5)
+            print(name, "ready")
+            break
+        except Exception:
+            time.sleep(5)
+    else:
+        raise SystemExit(f"{name} server never became ready")
+EOF
+
+echo "=== [watch] evaluating every new checkpoint in ${ADAPTER_REPO} ==="
+python3 eval_watch.py
+
+kill $VLLM_PID $ENV_PID 2>/dev/null || true
+echo "=== done ==="