Add downloadable shell for a separate private recovery training pilot

training/run_recovery.sh

@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+set -euo pipefail
+: "${HF_TOKEN:?Colab Secrets의 HF_TOKEN을 환경변수로 설정하세요.}"
+export HF_HUB_DISABLE_XET=1
+export HF_HUB_DISABLE_TELEMETRY=1
+export TOKENIZERS_PARALLELISM=false
+export DCT_COMPILE=0
+PICK_HOST_PYTHON="${PICK_HOST_PYTHON:-python3}"
+PICK_TRAIN_ROOT="${PICK_TRAIN_ROOT:-/content/pick-recovery}"
+PICK_TRAIN_NAME="${PICK_TRAIN_NAME:-frozen-head}"
+PICK_TRAIN_MINUTES="${PICK_TRAIN_MINUTES:-150}"
+export PICK_TRAIN_ROOT PICK_TRAIN_NAME PICK_TRAIN_MINUTES
+export PICK_BUNDLE_REPO="lCREEl/pick-v6-recovery"
+export PICK_BUNDLE_REVISION="8c2597fba997b5904eb5e6c22cb40e5e83b06507"
+export PICK_BUNDLE_SHA256="a441bd244998aeda94e0b0aaefd43064cf840f42dfa24a343ccfd9bb25e70ded"
+mkdir -p "$PICK_TRAIN_ROOT"
+if ! "$PICK_HOST_PYTHON" -c 'import huggingface_hub' >/dev/null 2>&1; then
+  "$PICK_HOST_PYTHON" -m pip -q install 'huggingface_hub==0.36.2'
+fi
+"$PICK_HOST_PYTHON" - <<'PY'
+import hashlib, os, tarfile
+from pathlib import Path
+from huggingface_hub import hf_hub_download
+p=hf_hub_download(os.environ['PICK_BUNDLE_REPO'],'recovery-bundle.tar.gz',revision=os.environ['PICK_BUNDLE_REVISION'])
+with open(p,'rb') as f:assert hashlib.file_digest(f,'sha256').hexdigest()==os.environ['PICK_BUNDLE_SHA256'],'Bundle hash mismatch'
+with tarfile.open(p) as t:t.extractall(os.environ['PICK_TRAIN_ROOT'],filter='data')
+print('학습 코드·데이터 다운로드 및 해시 검증 완료.',flush=True)
+PY
+if [[ "${PICK_VERIFY_ONLY:-0}" == "1" ]]; then exit 0; fi
+if ! command -v nvidia-smi >/dev/null 2>&1; then
+  echo 'Colab 런타임을 무료 GPU로 선택한 뒤 다시 실행하세요.' >&2
+  exit 1
+fi
+PICK_ENV="$PICK_TRAIN_ROOT/.venv"
+PICK_PYTHON="$PICK_ENV/bin/python3"
+if [[ ! -x "$PICK_PYTHON" || ! -f "$PICK_ENV/pyvenv.cfg" ]]; then
+  "$PICK_HOST_PYTHON" -m venv --without-pip "$PICK_ENV"
+fi
+"$PICK_HOST_PYTHON" -m pip --python "$PICK_PYTHON" -q install -r "$PICK_TRAIN_ROOT/requirements.txt"
+"$PICK_PYTHON" - <<'PY'
+import hashlib,json,os,shutil
+from pathlib import Path
+import torch
+from huggingface_hub import HfApi,hf_hub_download,snapshot_download
+assert torch.cuda.is_available(),'CUDA GPU is unavailable'
+root=Path(os.environ['PICK_TRAIN_ROOT']);repo=os.environ['PICK_BUNDLE_REPO'];api=HfApi()
+info=api.repo_info(repo);assert info.private,'Recovery checkpoints must stay private'
+rev=info.sha
+base=snapshot_download('lCREEl/pick-150m',revision='9980b9adcc14331a9b9d44bb936aca0f7deb221d',
+    allow_patterns=list(json.loads((root/'base-model-manifest.json').read_text())))
+for name,expected in json.loads((root/'base-model-manifest.json').read_text()).items():
+    with (Path(base)/name).open('rb') as f:assert hashlib.file_digest(f,'sha256').hexdigest()==expected,name
+link=root/'base-model'
+if not link.exists():link.symlink_to(Path(base),target_is_directory=True)
+assert link.resolve()==Path(base).resolve(),'Unexpected existing base-model path'
+out=root/'runs'/os.environ['PICK_TRAIN_NAME'];out.mkdir(parents=True,exist_ok=True)
+prefix='runs/'+os.environ['PICK_TRAIN_NAME']+'/'
+files=set(api.list_repo_files(repo,revision=rev))
+if not (out/'latest.pt').exists() and prefix+'latest.pt' in files:
+    for name in ['run-config.json','baseline.json','latest.pt']:
+        assert prefix+name in files,'Incomplete remote checkpoint'
+        source=hf_hub_download(repo,prefix+name,revision=rev)
+        shutil.copyfile(source,out/name)
+    print('이전 HF 학습 체크포인트를 복원했습니다.',flush=True)
+print('GPU:',torch.cuda.get_device_name(0),'· V5 원본 해시 검증 완료',flush=True)
+PY
+PICK_TRAIN_SECONDS=$((PICK_TRAIN_MINUTES * 60))
+"$PICK_PYTHON" "$PICK_TRAIN_ROOT/scripts/run_pilot.py" \
+  --export "$PICK_TRAIN_ROOT/base-model" --device cuda --name "$PICK_TRAIN_NAME" \
+  --steps 600 --seconds "$PICK_TRAIN_SECONDS" --grad-accum 8 --lr 3e-6 --head-lr 0 \
+  --backup-repo "$PICK_BUNDLE_REPO" --backup-seconds 900
+PICK_NEXT=$("$PICK_PYTHON" - <<'PY'
+import json,os
+from pathlib import Path
+d=json.loads((Path(os.environ['PICK_TRAIN_ROOT'])/'runs'/os.environ['PICK_TRAIN_NAME']/'decision.json').read_text())
+print('resume' if d['step']<600 else 'confirm' if d['decision']=='candidate_only' else 'discard')
+PY
+)
+if [[ "$PICK_NEXT" == "resume" ]]; then
+  echo '이번 실행 구간을 마치고 HF에 백업했습니다. 같은 셀을 다시 실행하면 이어집니다.'
+  exit 0
+fi
+if [[ "$PICK_NEXT" == "discard" ]]; then
+  echo '600단계 완료. 개발 검증 기준을 통과하지 않아 후보를 채택하지 않았습니다. 결과는 HF에 보관했습니다.'
+  exit 0
+fi
+echo '개발 검증 통과. 별도 확인용 데이터에서 V5와 후보를 비교합니다.'
+"$PICK_PYTHON" "$PICK_TRAIN_ROOT/scripts/run_pilot.py" --export "$PICK_TRAIN_ROOT/base-model" \
+  --device cuda --name "$PICK_TRAIN_NAME-confirm-v5" --eval-only --confirm
+"$PICK_PYTHON" "$PICK_TRAIN_ROOT/scripts/run_pilot.py" --export "$PICK_TRAIN_ROOT/base-model" \
+  --checkpoint "$PICK_TRAIN_ROOT/runs/$PICK_TRAIN_NAME/latest.pt" \
+  --device cuda --name "$PICK_TRAIN_NAME-confirm-candidate" --eval-only --confirm
+"$PICK_PYTHON" - <<'PY'
+import json,os
+from pathlib import Path
+from huggingface_hub import HfApi
+r=Path(os.environ['PICK_TRAIN_ROOT']);name=os.environ['PICK_TRAIN_NAME'];out=r/'runs'/name
+base=json.loads((r/'runs'/(name+'-confirm-v5')/'confirm.json').read_text())
+cand=json.loads((r/'runs'/(name+'-confirm-candidate')/'confirm.json').read_text())
+delta=cand['proxy_score']-base['proxy_score']
+guard=min(cand['per_task'][n]['score']-base['per_task'][n]['score'] for n in ['banking','clinc','home'])
+summary=dict(status='confirmation_passed_candidate' if delta>=0.5 and guard>=-0.015 else 'discard',
+    baseline_proxy=base['proxy_score'],candidate_proxy=cand['proxy_score'],delta=delta,
+    worst_anchor_delta=100*guard,public_index=None,
+    note='Internal validation only. Submitted V5 unchanged; a new full public evaluation is required.')
+(out/'confirmation-summary.json').write_text(json.dumps(summary,ensure_ascii=False,indent=2)+'\n')
+api=HfApi()
+for folder in [out,r/'runs'/(name+'-confirm-v5'),r/'runs'/(name+'-confirm-candidate')]:
+    api.upload_folder(repo_id=os.environ['PICK_BUNDLE_REPO'],folder_path=str(folder),path_in_repo='runs/'+folder.name,
+        commit_message='Save separate confirmation results for '+name)
+print(json.dumps(summary,ensure_ascii=False),flush=True)
+PY
+echo "학습·확인 검증 완료: https://huggingface.co/lCREEl/pick-v6-recovery/tree/main/runs/$PICK_TRAIN_NAME"