lfm-opencode-lora-rl-code / eval_watch.py
SaylorTwift's picture
SaylorTwift HF Staff
Eval robustness: fresh HarborEnv client per rollout (openenv reconnect bug); failed checkpoint evals retried next poll
75a9ca0 verified
Raw History Blame
4.47 kB
"""Per-checkpoint eval worker: watch the adapter repo, eval each new checkpoint.
Loop: poll ADAPTER_REPO for checkpoint-N dirs without eval results, hot-load each
as a LoRA into the already-running vLLM (runtime LoRA updates — no server restart),
run the held-out eval on a fixed task subset, and upload per-checkpoint results
plus a rolling eval/summary.json (the live learning curve).
Base-model cells are computed once and reused for every checkpoint.
Env: ADAPTER_REPO (required), BASE_MODEL, N_TASKS (default 100), POLL_S (600).
Stops when eval/final_results.json appears (the training job's own final eval).
"""
import json
import os
import subprocess
import time
import urllib.request
from pathlib import Path
from huggingface_hub import HfApi, snapshot_download
REPO = os.environ["ADAPTER_REPO"]
BASE_MODEL = os.environ.get("BASE_MODEL", "LiquidAI/LFM2.5-2.6B")
POLL_S = int(os.environ.get("POLL_S", "600"))
VLLM = "http://127.0.0.1:8001"
api = HfApi()
def repo_files():
return set(api.list_repo_files(REPO))
def pending_checkpoints(fs):
ckpts = sorted(
{f.split("/")[0] for f in fs if f.startswith("checkpoint-") and f.endswith("adapter_config.json")},
key=lambda s: int(s.rsplit("-", 1)[1]),
)
done = {f[len("eval/"):-len("_results.json")] for f in fs if f.startswith("eval/") and f.endswith("_results.json")}
return [c for c in ckpts if c not in done]
def run_eval(model_name, out_path):
env = dict(os.environ, ONLY_MODEL=model_name, OUT=out_path)
subprocess.run(["python3", "/app/eval_adapter.py"], env=env, check=True)
def upload(local, remote):
api.upload_file(path_or_fileobj=str(local), path_in_repo=remote, repo_id=REPO)
def lora_rpc(route, name, path=None):
payload = {"lora_name": name}
if path is not None:
payload["lora_path"] = str(path)
req = urllib.request.Request(
f"{VLLM}/v1/{route}",
data=json.dumps(payload).encode(),
headers={"Content-Type": "application/json"},
)
urllib.request.urlopen(req, timeout=900)
def pass_at_1(cells_path):
cells = json.loads(Path(cells_path).read_text())["cells"]
scored = [c["correct"] for c in cells.values() if not c["failed"]]
return (sum(scored) / len(scored)) if scored else None, len(scored)
def main():
fs = repo_files()
# 1. Base-model cells, once. Every checkpoint compares against these.
if "eval/base_cells.json" not in fs:
print("[watch] evaluating base model first", flush=True)
run_eval(BASE_MODEL, "/tmp/base_cells.json")
upload("/tmp/base_cells.json", "eval/base_cells.json")
summary = {}
if "eval/summary.json" in fs:
summary = json.loads(Path(api.hf_hub_download(REPO, "eval/summary.json")).read_text())
loaded = []
while True:
fs = repo_files()
if "eval/final_results.json" in fs:
print("[watch] training job's final eval landed; stopping", flush=True)
return
for ckpt in pending_checkpoints(fs):
print(f"[watch] new checkpoint: {ckpt}", flush=True)
local = Path(snapshot_download(REPO, allow_patterns=[f"{ckpt}/*"])) / ckpt
lora_rpc("load_lora_adapter", ckpt, local)
loaded.append(ckpt)
if len(loaded) > 4: # stay under --max-loras
lora_rpc("unload_lora_adapter", loaded.pop(0))
try:
run_eval(ckpt, f"/tmp/{ckpt}_cells.json")
except subprocess.CalledProcessError as exc:
# Transient eval failure (client reconnect flake, sandbox hiccup):
# skip it — the next poll recomputes pending and retries it.
print(f"[watch] {ckpt} eval failed ({exc}); will retry next poll", flush=True)
continue
upload(f"/tmp/{ckpt}_cells.json", f"eval/{ckpt}_results.json")
p1, n = pass_at_1(f"/tmp/{ckpt}_cells.json")
pb, nb = pass_at_1("/tmp/base_cells.json")
summary[ckpt] = {"pass_at_1": p1, "scorable": n,
"base_pass_at_1": pb,
"delta": (p1 - pb) if p1 is not None and pb is not None else None}
Path("/tmp/summary.json").write_text(json.dumps(summary, indent=2))
upload("/tmp/summary.json", "eval/summary.json")
print(f"[watch] {ckpt}: pass@1={p1} (base {pb})", flush=True)
time.sleep(POLL_S)
if __name__ == "__main__":
main()