Add held-out eval (base vs adapter) + standalone eval job + final eval stage in training job

eval_adapter.py

@@ -0,0 +1,116 @@
+"""Held-out evaluation: base model vs LoRA adapter on SmolDataEnvs test tasks.
+
+Runs N test tasks through the env server under OpenCode, once per model, and
+reports pass@1 (first graded attempt correct) for each, plus the per-task
+agreement breakdown the article's comparisons used.
+
+The vLLM server must serve the base model AND the adapter as a named LoRA:
+    vllm serve LiquidAI/LFM2.5-2.6B --enable-lora --max-lora-rank 32 \
+        --lora-modules trained=<adapter-repo> ...
+
+Config via env:
+    ENV_URL        env server (default http://127.0.0.1:8000)
+    VLLM_URL       policy server (default http://127.0.0.1:8001)
+    TEST_SPLIT     served test split (default /app/prepared/datasets/test)
+    BASE_MODEL     served base model id (default LiquidAI/LFM2.5-2.6B)
+    ADAPTER_NAME   LoRA name on the server (default "trained"); "" evaluates base only
+    N_TASKS        how many test tasks (default 50)
+    CONCURRENCY    parallel rollouts (default 8; keep <= MAX_CONCURRENT_ENVS)
+    OUT            results JSON path (default /app/eval_results.json)
+"""
+
+import json
+import os
+import threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from harbor_env import HarborEnv  # re-export of openenv.harbor.client.HarborEnv
+
+ENV_URL = os.environ.get("ENV_URL", "http://127.0.0.1:8000")
+VLLM_URL = os.environ.get("VLLM_URL", "http://127.0.0.1:8001")
+TEST_SPLIT = os.environ.get("TEST_SPLIT", "/app/prepared/datasets/test")
+BASE_MODEL = os.environ.get("BASE_MODEL", "LiquidAI/LFM2.5-2.6B")
+ADAPTER_NAME = os.environ.get("ADAPTER_NAME", "trained")
+N_TASKS = int(os.environ.get("N_TASKS", "50"))
+CONCURRENCY = int(os.environ.get("CONCURRENCY", "8"))
+OUT = os.environ.get("OUT", "/app/eval_results.json")
+
+_local = threading.local()
+
+
+def run_one(task_index, model):
+    """One graded rollout. Returns (correct, failed, error_or_none)."""
+    if not hasattr(_local, "env"):
+        _local.env = HarborEnv(base_url=ENV_URL)
+    kwargs = dict(
+        split=TEST_SPLIT,
+        task_index=task_index,
+        harness="opencode",
+        sandbox=os.environ.get("SANDBOX", "hf-sandbox"),
+        reward_key="correctness,reward",
+        llm_url=VLLM_URL,
+        model=model,
+        eval_sampling={"temperature": 1.0, "top_p": 1.0},
+        agent_step_limit=17,      # production eval cap from the FineEnvs Space
+        agent_timeout_sec=600,
+    )
+    try:
+        result = _local.env.run_rollout(**kwargs)
+    except TypeError:
+        # older server without eval_sampling: retry without it
+        kwargs.pop("eval_sampling")
+        result = _local.env.run_rollout(**kwargs)
+    except Exception as exc:
+        return task_index, model, None, True, f"{type(exc).__name__}: {exc}"
+    if not result.ok or result.reward is None:
+        return task_index, model, None, True, getattr(result, "error", None) or "not ok"
+    return task_index, model, float(result.reward) == 1.0, False, None
+
+
+def main():
+    models = [BASE_MODEL] + ([ADAPTER_NAME] if ADAPTER_NAME else [])
+    work = [(i, m) for i in range(N_TASKS) for m in models]
+    cells = {}  # (task_index, model) -> {"correct": bool|None, "failed": bool, "error": str|None}
+    with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool:
+        futures = [pool.submit(run_one, i, m) for i, m in work]
+        done = 0
+        for fut in as_completed(futures):
+            i, m, correct, failed, err = fut.result()
+            cells[(i, m)] = {"correct": correct, "failed": failed, "error": err}
+            done += 1
+            if done % 20 == 0 or done == len(work):
+                print(f"[eval] {done}/{len(work)} cells done", flush=True)
+
+    summary = {"n_tasks": N_TASKS, "models": {}}
+    for m in models:
+        scorable = [c["correct"] for (i, mm), c in cells.items() if mm == m and not c["failed"]]
+        failed = sum(1 for (i, mm), c in cells.items() if mm == m and c["failed"])
+        summary["models"][m] = {
+            "pass_at_1": sum(scorable) / len(scorable) if scorable else None,
+            "solved": sum(scorable),
+            "scorable": len(scorable),
+            "failed_rollouts": failed,
+        }
+    if len(models) == 2:
+        b, a = models
+        both = sum(1 for i in range(N_TASKS)
+                   if cells.get((i, b), {}).get("correct") and cells.get((i, a), {}).get("correct"))
+        only_a = sum(1 for i in range(N_TASKS)
+                     if not cells.get((i, b), {}).get("correct") and cells.get((i, a), {}).get("correct"))
+        only_b = sum(1 for i in range(N_TASKS)
+                     if cells.get((i, b), {}).get("correct") and not cells.get((i, a), {}).get("correct"))
+        summary["agreement"] = {"both_solved": both, "only_adapter": only_a, "only_base": only_b}
+        pa = summary["models"][a]["pass_at_1"]
+        pb = summary["models"][b]["pass_at_1"]
+        if pa is not None and pb is not None:
+            summary["delta"] = pa - pb
+
+    print(json.dumps(summary, indent=2))
+    with open(OUT, "w") as f:
+        json.dump({"summary": summary,
+                   "cells": {f"{i}|{m}": c for (i, m), c in cells.items()}}, f, indent=2)
+    print(f"[eval] wrote {OUT}")
+
+
+if __name__ == "__main__":
+    main()

prepare_tasks.py

@@ -46,12 +46,12 @@ def task_rows(split):
     return json.loads((ROOT / "data" / f"{split}.json").read_text())
 
 
-def prepare(destination):
+def prepare(destination, splits=("train", "test")):
     import tomli_w
     from huggingface_hub import snapshot_download
 
     destination = Path(destination).resolve()
-    for split in ("train", "test"):
+    for split in splits:
         rows = task_rows(split)
         repo, revision = DATASETS[split]
         source = Path(
@@ -94,18 +94,20 @@ def prepare(destination):
             (target / "task.toml").write_text(tomli_w.dumps(spec))
         print(f"{split}: {len(rows)} tasks -> {destination / 'datasets' / split}")
     # Overlap guard, as in the original.
-    def names(split):
-        return {r["name"] for r in task_rows(split)}
-    notebooks = [
-        {r.get("notebook") for r in task_rows(s)} - {None} for s in ("train", "test")
-    ]
-    if names("train") & names("test") or notebooks[0] & notebooks[1]:
-        raise ValueError("Train/test overlap")
-    print("no train/test overlap")
+    if set(splits) == {"train", "test"}:
+        def names(split):
+            return {r["name"] for r in task_rows(split)}
+        notebooks = [
+            {r.get("notebook") for r in task_rows(s)} - {None} for s in ("train", "test")
+        ]
+        if names("train") & names("test") or notebooks[0] & notebooks[1]:
+            raise ValueError("Train/test overlap")
+        print("no train/test overlap")
 
 
 if __name__ == "__main__":
     p = argparse.ArgumentParser(description=__doc__)
     p.add_argument("--output", type=Path, default=ROOT / "prepared")
+    p.add_argument("--splits", nargs="+", default=["train", "test"], choices=["train", "test"])
     a = p.parse_args()
-    prepare(a.output)
+    prepare(a.output, tuple(a.splits))

run_eval.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+# Standalone eval job (1 GPU): base model vs a LoRA adapter on held-out test tasks.
+#
+# Expected env:
+#   HF_TOKEN       (secret)  hub downloads + sandbox Jobs
+#   ADAPTER_REPO   (env)     Hub repo with adapter_config.json at root (or empty for base-only)
+#   N_TASKS        (env)     test tasks per model (default 50)
+#   OUT_REPO       (env)     where to upload eval_results.json (default: ADAPTER_REPO)
+set -euo pipefail
+cd "$(dirname "$0")"
+
+echo "=== [setup] ==="
+apt-get update -qq && apt-get install -y -qq --no-install-recommends git ca-certificates
+python3 -m pip install --quiet \
+    'openenv[harbor]' \
+    'harbor[hf-sandbox]>=0.24.0' \
+    math-verify tomli-w \
+    aiohttp
+git clone --depth 1 https://github.com/huggingface/OpenEnv.git /opt/OpenEnv
+SITE=$(python3 -c "import site; print(site.getsitepackages()[0])")
+echo "/opt/OpenEnv/envs" > "$SITE/openenv_examples.pth"
+
+echo "=== [setup] pinned test tasks ==="
+python3 prepare_tasks.py --output /app/prepared --splits test
+
+echo "=== [start] vLLM (base + adapter as LoRA 'trained') ==="
+LORA_ARGS=()
+if [ -n "${ADAPTER_REPO:-}" ]; then
+    LORA_ARGS=(--enable-lora --max-lora-rank 32 --lora-modules "trained=${ADAPTER_REPO}")
+fi
+CUDA_VISIBLE_DEVICES=0 \
+vllm serve "${BASE_MODEL:-LiquidAI/LFM2.5-2.6B}" \
+    --port 8001 \
+    --max-model-len 32768 \
+    --return-tokens-as-token-ids \
+    --logprobs-mode processed_logprobs \
+    --enable-auto-tool-choice \
+    --tool-call-parser lfm2 \
+    "${LORA_ARGS[@]}" &
+VLLM_PID=$!
+
+echo "=== [start] env server (test split only) ==="
+PREPARED=/app/prepared bash serve_env.sh &
+ENV_PID=$!
+
+python3 - <<'EOF'
+import time, urllib.request
+for name, url in [("vllm", "http://127.0.0.1:8001/v1/models"),
+                  ("env",  "http://127.0.0.1:8000/health")]:
+    for i in range(360):
+        try:
+            urllib.request.urlopen(url, timeout=5)
+            print(name, "ready")
+            break
+        except Exception:
+            time.sleep(5)
+    else:
+        raise SystemExit(f"{name} server never became ready")
+EOF
+
+echo "=== [eval] N_TASKS=${N_TASKS:-50} ==="
+ADAPTER_NAME="${ADAPTER_REPO:+trained}" python3 eval_adapter.py
+
+echo "=== [upload] results ==="
+python3 - <<'EOF'
+import os
+from huggingface_hub import HfApi
+repo = os.environ.get("OUT_REPO") or os.environ.get("ADAPTER_REPO")
+if repo:
+    api = HfApi()
+    api.upload_file(path_or_fileobj="/app/eval_results.json",
+                    path_in_repo=f"eval/{os.environ.get('EVAL_TAG', 'eval')}_results.json",
+                    repo_id=repo)
+    print("uploaded to", repo)
+EOF
+
+kill $VLLM_PID $ENV_PID 2>/dev/null || true
+echo "=== done ==="

run_job.sh

@@ -90,5 +90,50 @@ ENV_URL=http://127.0.0.1:8000 \
 TRAIN_SPLIT=/app/prepared/datasets/train \
     python3 train_lora.py
 
+if [ "${SKIP_EVAL:-0}" != "1" ] && [ -n "${ADAPTER_REPO:-}" ]; then
+    echo "=== [eval] restarting vLLM with the trained adapter (GPU 1) ==="
+    kill $VLLM_PID 2>/dev/null || true; sleep 10
+    # Late-bind the eval code: a fix uploaded during training still lands here.
+    mkdir -p /app-eval
+    hf download "${CODE_REPO:-SaylorTwift/lfm-opencode-lora-rl-code}" \
+        --local-dir /app-eval --include "eval_adapter.py" || cp /app/eval_adapter.py /app-eval/
+    CUDA_VISIBLE_DEVICES=1 \
+    vllm serve "${MODEL:-LiquidAI/LFM2.5-2.6B}" \
+        --port 8001 \
+        --max-model-len 32768 \
+        --return-tokens-as-token-ids \
+        --logprobs-mode processed_logprobs \
+        --enable-auto-tool-choice \
+        --tool-call-parser lfm2 \
+        --enable-lora --max-lora-rank 32 \
+        --lora-modules "trained=${ADAPTER_REPO}" &
+    VLLM_PID=$!
+    python3 - <<'EOF'
+import time, urllib.request
+for i in range(360):
+    try:
+        urllib.request.urlopen("http://127.0.0.1:8001/v1/models", timeout=5)
+        print("vllm ready")
+        break
+    except Exception:
+        time.sleep(5)
+else:
+    raise SystemExit("vllm never became ready for eval")
+EOF
+    echo "=== [eval] 250 held-out tasks x {base, trained} ==="
+    N_TASKS=250 OUT=/app/eval_results_final.json ADAPTER_NAME=trained \
+        python3 /app-eval/eval_adapter.py
+    python3 - <<'EOF'
+import os
+from huggingface_hub import HfApi
+repo = os.environ.get("ADAPTER_REPO")
+if repo:
+    HfApi().upload_file(path_or_fileobj="/app/eval_results_final.json",
+                        path_in_repo="eval/final_results.json",
+                        repo_id=repo)
+    print("uploaded to", repo)
+EOF
+fi
+
 kill $VLLM_PID $ENV_PID 2>/dev/null || true
 echo "=== done ==="