SaylorTwift HF Staff
Eval robustness: fresh HarborEnv client per rollout (openenv reconnect bug); failed checkpoint evals retried next poll
75a9ca0 verified Download eval_watch.py from SaylorTwift/lfm-opencode-lora-rl-code: direct link, hf CLI and curl.
- Browser
- Download file 4.47 kB
-
https://huggingface.co/SaylorTwift/lfm-opencode-lora-rl-code/resolve/75a9ca074fbc2e1b856733cb452e5b5d2cae5a68/eval_watch.py
- Command line
-
hf download hf://SaylorTwift/lfm-opencode-lora-rl-code@75a9ca074fbc2e1b856733cb452e5b5d2cae5a68/eval_watch.py
-
curl -L -o eval_watch.py https://huggingface.co/SaylorTwift/lfm-opencode-lora-rl-code/resolve/75a9ca074fbc2e1b856733cb452e5b5d2cae5a68/eval_watch.py
4.47 kB
| """Per-checkpoint eval worker: watch the adapter repo, eval each new checkpoint. | |
| Loop: poll ADAPTER_REPO for checkpoint-N dirs without eval results, hot-load each | |
| as a LoRA into the already-running vLLM (runtime LoRA updates — no server restart), | |
| run the held-out eval on a fixed task subset, and upload per-checkpoint results | |
| plus a rolling eval/summary.json (the live learning curve). | |
| Base-model cells are computed once and reused for every checkpoint. | |
| Env: ADAPTER_REPO (required), BASE_MODEL, N_TASKS (default 100), POLL_S (600). | |
| Stops when eval/final_results.json appears (the training job's own final eval). | |
| """ | |
| import json | |
| import os | |
| import subprocess | |
| import time | |
| import urllib.request | |
| from pathlib import Path | |
| from huggingface_hub import HfApi, snapshot_download | |
| REPO = os.environ["ADAPTER_REPO"] | |
| BASE_MODEL = os.environ.get("BASE_MODEL", "LiquidAI/LFM2.5-2.6B") | |
| POLL_S = int(os.environ.get("POLL_S", "600")) | |
| VLLM = "http://127.0.0.1:8001" | |
| api = HfApi() | |
| def repo_files(): | |
| return set(api.list_repo_files(REPO)) | |
| def pending_checkpoints(fs): | |
| ckpts = sorted( | |
| {f.split("/")[0] for f in fs if f.startswith("checkpoint-") and f.endswith("adapter_config.json")}, | |
| key=lambda s: int(s.rsplit("-", 1)[1]), | |
| ) | |
| done = {f[len("eval/"):-len("_results.json")] for f in fs if f.startswith("eval/") and f.endswith("_results.json")} | |
| return [c for c in ckpts if c not in done] | |
| def run_eval(model_name, out_path): | |
| env = dict(os.environ, ONLY_MODEL=model_name, OUT=out_path) | |
| subprocess.run(["python3", "/app/eval_adapter.py"], env=env, check=True) | |
| def upload(local, remote): | |
| api.upload_file(path_or_fileobj=str(local), path_in_repo=remote, repo_id=REPO) | |
| def lora_rpc(route, name, path=None): | |
| payload = {"lora_name": name} | |
| if path is not None: | |
| payload["lora_path"] = str(path) | |
| req = urllib.request.Request( | |
| f"{VLLM}/v1/{route}", | |
| data=json.dumps(payload).encode(), | |
| headers={"Content-Type": "application/json"}, | |
| ) | |
| urllib.request.urlopen(req, timeout=900) | |
| def pass_at_1(cells_path): | |
| cells = json.loads(Path(cells_path).read_text())["cells"] | |
| scored = [c["correct"] for c in cells.values() if not c["failed"]] | |
| return (sum(scored) / len(scored)) if scored else None, len(scored) | |
| def main(): | |
| fs = repo_files() | |
| # 1. Base-model cells, once. Every checkpoint compares against these. | |
| if "eval/base_cells.json" not in fs: | |
| print("[watch] evaluating base model first", flush=True) | |
| run_eval(BASE_MODEL, "/tmp/base_cells.json") | |
| upload("/tmp/base_cells.json", "eval/base_cells.json") | |
| summary = {} | |
| if "eval/summary.json" in fs: | |
| summary = json.loads(Path(api.hf_hub_download(REPO, "eval/summary.json")).read_text()) | |
| loaded = [] | |
| while True: | |
| fs = repo_files() | |
| if "eval/final_results.json" in fs: | |
| print("[watch] training job's final eval landed; stopping", flush=True) | |
| return | |
| for ckpt in pending_checkpoints(fs): | |
| print(f"[watch] new checkpoint: {ckpt}", flush=True) | |
| local = Path(snapshot_download(REPO, allow_patterns=[f"{ckpt}/*"])) / ckpt | |
| lora_rpc("load_lora_adapter", ckpt, local) | |
| loaded.append(ckpt) | |
| if len(loaded) > 4: # stay under --max-loras | |
| lora_rpc("unload_lora_adapter", loaded.pop(0)) | |
| try: | |
| run_eval(ckpt, f"/tmp/{ckpt}_cells.json") | |
| except subprocess.CalledProcessError as exc: | |
| # Transient eval failure (client reconnect flake, sandbox hiccup): | |
| # skip it — the next poll recomputes pending and retries it. | |
| print(f"[watch] {ckpt} eval failed ({exc}); will retry next poll", flush=True) | |
| continue | |
| upload(f"/tmp/{ckpt}_cells.json", f"eval/{ckpt}_results.json") | |
| p1, n = pass_at_1(f"/tmp/{ckpt}_cells.json") | |
| pb, nb = pass_at_1("/tmp/base_cells.json") | |
| summary[ckpt] = {"pass_at_1": p1, "scorable": n, | |
| "base_pass_at_1": pb, | |
| "delta": (p1 - pb) if p1 is not None and pb is not None else None} | |
| Path("/tmp/summary.json").write_text(json.dumps(summary, indent=2)) | |
| upload("/tmp/summary.json", "eval/summary.json") | |
| print(f"[watch] {ckpt}: pass@1={p1} (base {pb})", flush=True) | |
| time.sleep(POLL_S) | |
| if __name__ == "__main__": | |
| main() | |