Add one-script free Colab evaluation with private HF backups and resume

decision-index/SUBMISSION_DRAFT.md

@@ -0,0 +1,30 @@
+# PICK-150M V5 — submission draft
+
+Do not submit this draft until the full public evaluation has completed.
+
+- Model: [lCREEl/pick-150m](https://huggingface.co/lCREEl/pick-150m)
+- Frozen model revision: `9980b9adcc14331a9b9d44bb936aca0f7deb221d`
+- Parameters: 149,015,809
+- Engine: `pick_v5.engine:PickV5Engine`, code revision in `release.json` and run fingerprint.
+- Suite: public edition 0.3, 140,178 scoreable requests.
+- Results: `lCREEl/pick-v5-decision-index-results/runs/pick-150m-v5/scores.json`.
+  The dataset is initially private; publish compact outputs or grant maintainer access before review.
+- Hardware/software: use final `environment.json` and `hardware-history.json`.
+- Settings: fp32, native CUDA TF32, no compilation, shared-state packing,
+  state reads questions and question reads options, 7,999 packed tokens,
+  7,999 forward token budget, fixed native key/description rendering.
+- Capacity: a request is unsupported if a question/option/state cannot fit without truncation.
+  Multi-question requests use native deterministic recursive bisection as needed;
+  every question and option is retained. Full counts are in the final scores.
+- Calibration: native probabilities only; no benchmark-specific postprocessing or result edits.
+- Data disclosure: prior development used related public task families and sampled benchmark feedback.
+  Exact-state/sentence decontamination exists in the V5 data builder; this is not a proof
+  that the entire historical curriculum is semantically disjoint from every benchmark.
+- Reproduction: download the frozen inference source/wheel from the model's `decision-index/`
+  directory and use its fixed model revision. The wrapper is a snapshot of local
+  uncommitted inference fixes, not just the older private GitHub repository HEAD.
+
+Before opening a PR, confirm `complete=true`, `completed=140178`, error count zero,
+canonical suite hashes, untouched outputs, accessible inference code, and reviewer access.
+Add the completed entry to `submissions/README.md` in apolinario/decision-index.
+The maintainers compute the private Full-score components and the RTX PRO 6000 latency gate.

decision-index/cloud_worker.py

@@ -1,76 +0,0 @@
-"""Run the official suite on one device, with private resumable output checkpoints."""
-import argparse
-import gzip
-import json
-import shutil
-import time
-from pathlib import Path
-
-from huggingface_hub import HfApi, hf_hub_download, snapshot_download
-from decision_index.runner import run
-from decision_index.pipeline import score_run, upload_run
-from decision_index.suite.io import Suite, atomic_json
-
-
-def main():
-    p = argparse.ArgumentParser()
-    p.add_argument('--suite-repo', required=True)
-    p.add_argument('--suite-revision', required=True)
-    p.add_argument('--results-repo', required=True)
-    p.add_argument('--run-name', default='pick-150m-v5')
-    p.add_argument('--root', default='/tmp/pick-evaluation')
-    args = p.parse_args()
-    api = HfApi()
-    # Fail closed if a benchmark dataset is accidentally public.
-    if not api.dataset_info(args.suite_repo, revision=args.suite_revision).private:
-        raise ValueError('Benchmark input repository must remain private')
-    root = Path(args.root)
-    suite_dir = root / 'suite-0.3'
-    snapshot_download(args.suite_repo, repo_type='dataset', revision=args.suite_revision,
-                      local_dir=suite_dir)
-    suite = Suite(suite_dir, '0.3')
-    verification = suite.verify(strict=True)
-    if not verification.get('exclusions_match'):
-        raise ValueError('Canonical exclusion file does not match')
-    out = root / 'runs' / args.run_name
-    out.mkdir(parents=True, exist_ok=True)
-    atomic_json(out / 'suite-verification.json', verification)
-    api.create_repo(args.results_repo, repo_type='dataset', private=True, exist_ok=True)
-    if not api.dataset_info(args.results_repo).private:
-        raise ValueError('Work-in-progress results repository must remain private')
-    remote_path = f'runs/{args.run_name}'
-    existing = api.list_repo_files(args.results_repo, repo_type='dataset')
-    compressed = f'{remote_path}/results.jsonl.gz'
-    if not (out / 'results.jsonl').exists() and compressed in existing:
-        saved = hf_hub_download(args.results_repo, compressed, repo_type='dataset')
-        with gzip.open(saved, 'rb') as src, (out / 'results.jsonl').open('wb') as dst:
-            shutil.copyfileobj(src, dst)
-    last_upload = time.monotonic()
-
-    def checkpoint():
-        if (out / 'results.jsonl').exists():
-            score_run(suite, out / 'results.jsonl', 'pick_v5.engine:PickV5Engine', out)
-        upload_run(args.results_repo, out, path_in_repo=remote_path, private=True)
-
-    def log(line):
-        nonlocal last_upload
-        print(line, flush=True)
-        if time.monotonic() - last_upload > 300:
-            checkpoint()
-            last_upload = time.monotonic()
-
-    try:
-        run('pick_v5.engine:PickV5Engine', {'device': 'cuda'}, suite.row_paths, out,
-            compact=True, resume=True, corpus_sha256=verification['sha256'],
-            keep=suite.in_edition, log=log)
-    finally:
-        checkpoint()
-    scores = json.loads((out / 'scores.json').read_text())
-    if not scores['complete'] or scores['completed'] != 140178 or scores['counts'].get('error', 0):
-        raise RuntimeError('Full public evaluation is incomplete or has runtime errors; do not submit')
-    print(json.dumps({'complete': True, 'public_index': scores['decision_index'],
-                      'results_repo': args.results_repo}), flush=True)
-
-
-if __name__ == '__main__':
-    main()

decision-index/release.json

@@ -0,0 +1,8 @@
+{
+  "model_repo": "lCREEl/pick-150m",
+  "model_revision": "9980b9adcc14331a9b9d44bb936aca0f7deb221d",
+  "code_revision": "f1635cf8868b3359c0ea78cd7d0110e7bbe6b555",
+  "commit_url": "https://huggingface.co/lCREEl/pick-150m/commit/f1635cf8868b3359c0ea78cd7d0110e7bbe6b555",
+  "weight_sha256": "4bbf03998bf507660872fb943aa20543a18c481793c240e93a426a04b8b10033",
+  "wheel_sha256": "d595e42c206fecedbdc0442dadce5d667b763701692f3d213fcbd8d572c4df8c"
+}

decision-index/run_colab.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+# PICK V5: evaluate only. Download, install, resume, and back up; never train.
+set -euo pipefail
+
+if [[ -z "${HF_TOKEN:-}" && "${PICK_VERIFY_ONLY:-0}" != "1" ]]; then
+  echo 'HF_TOKEN이 없습니다. Colab 보안 비밀 HF_TOKEN을 환경변수로 전달하세요.' >&2
+  exit 2
+fi
+
+export HF_HUB_DISABLE_XET=1
+export PYTHONUNBUFFERED=1
+export DCT_COMPILE=0
+PICK_ROOT="${PICK_ROOT:-/content/pick-evaluation}"
+export PICK_ROOT
+mkdir -p "$PICK_ROOT/installer"
+export HF_HUB_CACHE="$PICK_ROOT/cache/hub"
+export CODE_REVISION='f1635cf8868b3359c0ea78cd7d0110e7bbe6b555'
+export SUITE_REPO='lCREEl/pick-v5-decision-index-suite-0.3'
+export SUITE_REVISION='f5a27dc47f7f233a66f06fc20eb45528839c5c9c'
+export WHEEL_SHA256='d595e42c206fecedbdc0442dadce5d667b763701692f3d213fcbd8d572c4df8c'
+export WORKER_SHA256='0bf00f9c3ace44dd91cb9d582634b36e7f8e7ff797b1fe6c2a4780519347a3e1'
+export REQUIREMENTS_SHA256='ce1394a268341811eaccf4e8c97f24108f58c3efff2b7cf85d41bb0e0cbc4d2b'
+
+echo '고정된 V5 평가 파일을 내려받아 해시를 확인합니다.'
+python3 - <<'PY'
+import hashlib, os, sys, urllib.request
+from pathlib import Path
+if not (3,11) <= sys.version_info[:2] < (3,15):
+    raise SystemExit('Python 3.11~3.14 런타임이 필요합니다.')
+base=f"https://huggingface.co/lCREEl/pick-150m/resolve/{os.environ['CODE_REVISION']}/decision-index"
+cache=Path(os.environ['PICK_ROOT'])/'installer'
+for name, expected in {
+    'pick_v5_decision_index-0.1.0-py3-none-any.whl':os.environ['WHEEL_SHA256'],
+    'colab_worker.py':os.environ['WORKER_SHA256'],
+    'requirements-colab.txt':os.environ['REQUIREMENTS_SHA256'],
+}.items():
+    path=cache/name
+    if not path.exists() or hashlib.sha256(path.read_bytes()).hexdigest()!=expected:
+        tmp=path.with_suffix(path.suffix+'.part')
+        urllib.request.urlretrieve(f'{base}/{name}',tmp)
+        if hashlib.sha256(tmp.read_bytes()).hexdigest()!=expected:
+            raise SystemExit(f'파일 해시 불일치: {name}')
+        tmp.replace(path)
+PY
+
+if [[ "${PICK_VERIFY_ONLY:-0}" == "1" ]]; then
+  echo '공개 실행 파일 다운로드와 해시 검증을 통과했습니다.'
+  exit 0
+fi
+
+PICK_PYTHON="$PICK_ROOT/.venv/bin/python"
+if [[ ! -x "$PICK_PYTHON" ]]; then
+  echo 'Colab 기본 환경과 분리된 전용 환경을 만듭니다.'
+  python3 -m venv "$PICK_ROOT/.venv"
+fi
+echo '고정된 의존성을 설치합니다. 첫 실행은 몇 분 걸릴 수 있습니다.'
+"$PICK_PYTHON" -m pip install --disable-pip-version-check -q   -r "$PICK_ROOT/installer/requirements-colab.txt"   "$PICK_ROOT/installer/pick_v5_decision_index-0.1.0-py3-none-any.whl"
+
+"$PICK_PYTHON" - <<'PY'
+import torch
+from huggingface_hub import HfApi
+from pick_v5.engine import PickV5Engine
+if not torch.cuda.is_available():
+    raise SystemExit('무료 GPU가 없습니다. 런타임 > 런타임 유형 변경 > T4 GPU를 선택하세요.')
+user=HfApi().whoami()['name']
+print('HF 계정:',user)
+print('GPU:',torch.cuda.get_device_name(0),'Torch:',torch.__version__,'CUDA:',torch.version.cuda)
+PY
+
+echo '평가를 시작/재개합니다. HF 자동 백업은 기본 120초 간격입니다.'
+exec "$PICK_PYTHON" "$PICK_ROOT/installer/colab_worker.py"   --suite-repo "$SUITE_REPO" --suite-revision "$SUITE_REVISION"   --code-revision "$CODE_REVISION"   --results-repo "${PICK_RESULTS_REPO:-lCREEl/pick-v5-decision-index-results}"   --run-name "${PICK_RUN_NAME:-pick-150m-v5}"   --backup-seconds "${PICK_BACKUP_SECONDS:-120}"   --max-minutes "${PICK_MAX_MINUTES:-330}"   --root "$PICK_ROOT"