File size: 3,394 Bytes
c511323 bc532e5 c511323 bc532e5 c511323 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 | #!/usr/bin/env bash
# Standalone eval job (1 GPU): base model vs a LoRA adapter on held-out test tasks.
#
# Expected env:
# HF_TOKEN (secret) hub downloads + sandbox Jobs
# ADAPTER_REPO (env) Hub repo with adapter_config.json at root (or empty for base-only)
# N_TASKS (env) test tasks per model (default 50)
# OUT_REPO (env) where to upload eval_results.json (default: ADAPTER_REPO)
set -euo pipefail
cd "$(dirname "$0")"
echo "=== [setup] ==="
apt-get update -qq && apt-get install -y -qq --no-install-recommends git ca-certificates
python3 -m pip install --quiet \
'openenv[harbor]' \
'harbor[hf-sandbox]>=0.24.0' \
math-verify tomli-w \
aiohttp
git clone --depth 1 https://github.com/huggingface/OpenEnv.git /opt/OpenEnv
SITE=$(python3 -c "import site; print(site.getsitepackages()[0])")
echo "/opt/OpenEnv/envs" > "$SITE/openenv_examples.pth"
echo "=== [setup] pinned test tasks ==="
python3 prepare_tasks.py --output /app/prepared --splits test
# (seam patch happens below, before the env server starts)
echo "=== [start] vLLM (base + adapter as LoRA 'trained') ==="
LORA_ARGS=()
if [ -n "${ADAPTER_REPO:-}" ]; then
LORA_ARGS=(--enable-lora --max-lora-rank 32 --lora-modules "trained=${ADAPTER_REPO}")
fi
CUDA_VISIBLE_DEVICES=0 \
vllm serve "${BASE_MODEL:-LiquidAI/LFM2.5-2.6B}" \
--port 8001 \
--max-model-len 32768 \
--return-tokens-as-token-ids \
--logprobs-mode processed_logprobs \
--enable-auto-tool-choice \
--tool-call-parser lfm2 \
"${LORA_ARGS[@]}" &
VLLM_PID=$!
echo "=== [patch] opencode seam: session id as x-api-key (Jobs proxy strips Authorization) ==="
python3 - <<'EOF'
import openenv.harbor.seams as seams
path = seams.__file__
src = open(path).read()
old = '"apiKey": session,\n "timeout": 600_000,'
new = ('"apiKey": session,\n'
' "headers": {"x-api-key": session},\n'
' "timeout": 600_000,')
assert src.count(old) == 1, f"opencode seam not found in {path}"
open(path, "w").write(src.replace(old, new))
print("patched", path)
EOF
echo "=== [start] env server (test split only; tunnel-free via Jobs proxy) ==="
export SPACE_HOST="${JOB_ID}--8000.hf.jobs"
PREPARED=/app/prepared \
OPENENV_DATASETS=/app/prepared/datasets/test \
bash serve_env.sh &
ENV_PID=$!
python3 - <<'EOF'
import time, urllib.request
for name, url in [("vllm", "http://127.0.0.1:8001/v1/models"),
("env", "http://127.0.0.1:8000/health")]:
for i in range(360):
try:
urllib.request.urlopen(url, timeout=5)
print(name, "ready")
break
except Exception:
time.sleep(5)
else:
raise SystemExit(f"{name} server never became ready")
EOF
echo "=== [eval] N_TASKS=${N_TASKS:-50} ==="
ADAPTER_NAME="${ADAPTER_REPO:+trained}" python3 eval_adapter.py
echo "=== [upload] results ==="
python3 - <<'EOF'
import os
from huggingface_hub import HfApi
repo = os.environ.get("OUT_REPO") or os.environ.get("ADAPTER_REPO")
if repo:
api = HfApi()
api.upload_file(path_or_fileobj="/app/eval_results.json",
path_in_repo=f"eval/{os.environ.get('EVAL_TAG', 'eval')}_results.json",
repo_id=repo)
print("uploaded to", repo)
EOF
kill $VLLM_PID $ENV_PID 2>/dev/null || true
echo "=== done ==="
|