docs/reports: add scripts/colab_run.sh

64fc1b6 verified 2 days ago

9.15 kB

#!/usr/bin/env bash
# One-click execution script for Google Colab (or any CUDA VM)
# V4 Auto-Calibrated Fine-Tuning (starting from v3_stage1_best.pt) -> V4 Model Soup -> 8K Export -> Public 16 Benchmarks
set -euo pipefail
cd "$(dirname "$0")/.."

echo "======================================================================"
echo "🚀 PICK-150M V4 Auto-Calibrated Fine-Tuning & Model Soup Suite (Colab)"
echo "======================================================================"

echo "=== 1. Checking GPU ==="
nvidia-smi

echo "=== 2. Installing Dependencies ==="
pip install -q "transformers==4.57.6" "datasets" "safetensors" "tqdm" "huggingface_hub>=0.25.0"
pip install -q "decision-index[rebuild] @ git+https://github.com/apolinario/decision-index@9eb2dbe2a358004c8782c66e40a83ac07b953fec"

echo "=== 3. Downloading V4 Assets & V3 Checkpoint from Hugging Face ==="
rm -rf data/finetune-v2 checkpoints/pick-150m-v4
mkdir -p data checkpoints
python3 -c "
import os, tarfile, shutil
from huggingface_hub import hf_hub_download
token = os.environ.get('HF_TOKEN')
assert token, 'HF_TOKEN environment variable is required!'
print('Downloading V4 finetune-v2.tar.gz...', flush=True)
p1 = hf_hub_download(repo_id='lCREEl/pick-150m-runpod', filename='finetune-v2.tar.gz', token=token, force_download=True)
with tarfile.open(p1) as tar:
    tar.extractall('data/')
print('Downloading v3_stage1_best.pt and soup23_best.pt...', flush=True)
hf_hub_download(repo_id='lCREEl/pick-150m-runpod', filename='v3_stage1_best.pt', token=token, local_dir='checkpoints')
hf_hub_download(repo_id='lCREEl/pick-150m-runpod', filename='soup23_best.pt', token=token, local_dir='checkpoints')
bench_p = 'data/benchmarks/crivo/public/rows.jsonl.gz'
if not os.path.exists(bench_p):
    print('Downloading public benchmarks...', flush=True)
    os.makedirs('data/benchmarks/crivo/public', exist_ok=True)
    os.makedirs('data/benchmarks/crivo/reference/crivo-150m', exist_ok=True)
    bp = hf_hub_download(repo_id='lCREEl/pick-150m-runpod', filename='benchmark_public.jsonl.gz', token=token)
    shutil.copyfile(bp, bench_p)
    hf_hub_download(repo_id='lCREEl/pick-150m-runpod', filename='data/benchmarks/crivo/public/manifest.json', token=token, local_dir='.')
    hf_hub_download(repo_id='lCREEl/pick-150m-runpod', filename='data/benchmarks/crivo/reference/crivo-150m/results.jsonl.gz', token=token, local_dir='.')
print('All V4 assets downloaded successfully!', flush=True)
"

export PYTHONPATH="$PWD/src${PYTHONPATH:+:$PYTHONPATH}"

echo "=== 4A. V4 Stage 1: Auto-Calibrated Prior & Reasoning Refinement (400 steps, 2560 len) ==="
python3 scripts/train.py \
  --model cross-encoder/ettin-reranker-150m-v1 \
  --arch pick \
  --head-type uncentered \
  --loss ce-brier \
  --init checkpoints/v3_stage1_best.pt \
  --save-dir checkpoints/pick-150m-v4/stage1 \
  --train-paths \
      data/finetune-v2/fmt-cladder-hard/train.jsonl \
      data/finetune-v2/fmt-when2call-aligned/train.jsonl \
      data/finetune-v2/fmt-hover-multihop/train.jsonl \
      data/finetune-v2/fmt-acos-calibrated/train.jsonl \
      data/finetune-v2/fmt-contractnli-dual/train.jsonl \
      data/finetune-v2/fmt-home-sim-hard/train.jsonl \
      data/finetune-v2/fmt-nli4ct-aligned/train.jsonl \
      data/finetune-v2/fmt-ragtruth-aligned/train.jsonl \
      data/finetune-v2/fmt-hellaswag/train.jsonl \
  --replay-paths \
      data/finetune-v2/anchor-replay/train.jsonl \
  --replay-share 0.25 \
  --mix-temperature 0.0 \
  --class-balance 0.0 \
  --dataset-weight \
      fmt-cladder-hard=3.5 \
      fmt-when2call-aligned=2.8 \
      fmt-hover-multihop=2.5 \
      fmt-acos-calibrated=2.2 \
      fmt-contractnli-dual=2.2 \
      fmt-home-sim-hard=2.0 \
      fmt-nli4ct-aligned=1.8 \
      fmt-ragtruth-aligned=1.6 \
      fmt-hellaswag=1.2 \
  --batch-cases 16 \
  --grad-accum 2 \
  --lr 6e-6 \
  --head-lr 2.5e-5 \
  --steps 400 \
  --warmup 30 \
  --schedule cosine \
  --eval-every 100 \
  --val-cases-per-set 50 \
  --save-every 100 \
  --max-length 2560 \
  --max-state-tokens 1536 \
  --max-question-tokens 512 \
  --device cuda \
  --precision bf16 \
  --workers 2

STAGE1_PT="checkpoints/pick-150m-v4/stage1/best.pt"
[ -f "$STAGE1_PT" ] || STAGE1_PT="checkpoints/pick-150m-v4/stage1/final.pt"

if [ -n "${HF_TOKEN:-}" ]; then
  echo "Backing up v4_stage1_best.pt to Hugging Face immediately..."
  python3 -c "
import os
from huggingface_hub import HfApi
HfApi(token=os.environ['HF_TOKEN']).upload_file(
    path_or_fileobj='$STAGE1_PT',
    path_in_repo='v4_stage1_best.pt',
    repo_id='lCREEl/pick-150m-runpod',
    commit_message='feat: backup v4_stage1_best.pt after V4 Stage 1'
)
print('✅ v4_stage1_best.pt backed up to Hugging Face!')
"
fi

echo "=== 4B. V4 Stage 2: Long-Context 6K Full-Document Polisher (200 steps, 6144 len) ==="
python3 scripts/train.py \
  --model cross-encoder/ettin-reranker-150m-v1 \
  --arch pick \
  --head-type uncentered \
  --loss ce-brier \
  --init "$STAGE1_PT" \
  --save-dir checkpoints/pick-150m-v4/stage2 \
  --train-paths \
      data/finetune-v2/fmt-contractnli-dual/train.jsonl \
      data/finetune-v2/fmt-home-sim-hard/train.jsonl \
      data/finetune-v2/fmt-ragtruth-aligned/train.jsonl \
      data/finetune-v2/fmt-hover-multihop/train.jsonl \
      data/finetune-v2/fmt-when2call-aligned/train.jsonl \
      data/finetune-v2/fmt-acos-calibrated/train.jsonl \
      data/finetune-v2/fmt-nli4ct-aligned/train.jsonl \
  --replay-paths \
      data/finetune-v2/anchor-replay/train.jsonl \
      data/finetune-v2/fmt-cladder-hard/train.jsonl \
      data/finetune-v2/fmt-hellaswag/train.jsonl \
  --replay-share 0.35 \
  --mix-temperature 0.0 \
  --class-balance 0.0 \
  --dataset-weight \
      fmt-contractnli-dual=3.2 \
      fmt-home-sim-hard=2.2 \
      fmt-ragtruth-aligned=2.0 \
      fmt-hover-multihop=2.0 \
      fmt-when2call-aligned=2.0 \
      fmt-acos-calibrated=1.8 \
      fmt-nli4ct-aligned=1.5 \
  --batch-cases 8 \
  --grad-accum 4 \
  --lr 3e-6 \
  --head-lr 1.2e-5 \
  --steps 200 \
  --warmup 20 \
  --schedule cosine \
  --eval-every 100 \
  --val-cases-per-set 50 \
  --save-every 100 \
  --max-length 6144 \
  --max-state-tokens 4608 \
  --max-question-tokens 512 \
  --device cuda \
  --precision bf16 \
  --workers 2

STAGE2_PT="checkpoints/pick-150m-v4/stage2/best.pt"
[ -f "$STAGE2_PT" ] || STAGE2_PT="checkpoints/pick-150m-v4/stage2/final.pt"

echo "=== 5. Building V4 Model Soup (0.08 Base + 0.32 V3_Stage1 + 0.60 V4_Stage2) & Exporting 8K Model ==="
SOUP_PT="checkpoints/pick-150m-v4/soup_v4.pt"
python3 scripts/average-checkpoints.py \
  checkpoints/soup23_best.pt checkpoints/v3_stage1_best.pt "$STAGE2_PT" \
  --out "$SOUP_PT" \
  --weights 0.08 0.32 0.60

python3 scripts/export-pipeline-checkpoint.py \
  --ckpt "$SOUP_PT" \
  --out checkpoints/pick-150m-v4/export \
  --model cross-encoder/ettin-reranker-150m-v1 \
  --max-length 7999 \
  --score-scaling normalized

echo "=== 6. Uploading V4 Checkpoints & Exported Model to Hugging Face ==="
if [ -n "${HF_TOKEN:-}" ]; then
  python3 -c "
import os
from huggingface_hub import HfApi
api = HfApi(token=os.environ['HF_TOKEN'])
api.upload_file(path_or_fileobj='$STAGE2_PT', path_in_repo='v4_stage2_best.pt', repo_id='lCREEl/pick-150m-runpod', commit_message='feat: upload v4_stage2_best.pt')
api.upload_file(path_or_fileobj='$SOUP_PT', path_in_repo='v4_soup.pt', repo_id='lCREEl/pick-150m-runpod', commit_message='feat: upload v4_soup.pt')
api.upload_folder(folder_path='checkpoints/pick-150m-v4/export', repo_id='lCREEl/pick-150m-runpod', commit_message='feat: export V4 Auto-Calibrated 8K model')
print('✅ Uploaded V4 checkpoints and exported model to Hugging Face!')
"
fi

echo "=== 7. Running Full 16-Benchmark Public Evaluation ==="
python3 -m decision_transformer.evaluation.suite run checkpoints/pick-150m-v4/export \
  --name "pick-150m-v4-soup" \
  --device cuda \
  --suites public

echo "=== 8. Running Post-Hoc Calibration Recipe Comparison ==="
python3 scripts/calibrate_benchmark_results.py \
  --results reports/benchmarks/pick-150m-v4-soup/public/results.jsonl \
  --out reports/benchmarks/pick-150m-v4-soup/public/results_calibrated.jsonl

if [ -n "${HF_TOKEN:-}" ] && [ -f "reports/benchmarks.jsonl" ]; then
  python3 -c "
import os
from huggingface_hub import HfApi
api = HfApi(token=os.environ['HF_TOKEN'])
api.upload_file(path_or_fileobj='reports/benchmarks.jsonl', path_in_repo='reports/benchmarks.jsonl', repo_id='lCREEl/pick-150m-runpod', commit_message='reports: add pick-150m-v4-soup to benchmark ledger')
for p in ('reports/benchmarks/pick-150m-v4-soup/public/results.jsonl', 'reports/benchmarks/pick-150m-v4-soup/public/results_calibrated.jsonl'):
    if os.path.exists(p):
        api.upload_file(path_or_fileobj=p, path_in_repo=p, repo_id='lCREEl/pick-150m-runpod', commit_message=f'reports: upload {p}')
print('✅ Uploaded V4 benchmark ledger and results to Hugging Face!')
"
fi

echo "======================================================================"
echo "🎉 V4 ALL FINISHED! Check reports/benchmarks/pick-150m-v4-soup"
echo "======================================================================"

	#!/usr/bin/env bash
	# One-click execution script for Google Colab (or any CUDA VM)
	# V4 Auto-Calibrated Fine-Tuning (starting from v3_stage1_best.pt) -> V4 Model Soup -> 8K Export -> Public 16 Benchmarks
	set -euo pipefail
	cd "$(dirname "$0")/.."

	echo "======================================================================"
	echo "🚀 PICK-150M V4 Auto-Calibrated Fine-Tuning & Model Soup Suite (Colab)"
	echo "======================================================================"

	echo "=== 1. Checking GPU ==="
	nvidia-smi

	echo "=== 2. Installing Dependencies ==="
	pip install -q "transformers==4.57.6" "datasets" "safetensors" "tqdm" "huggingface_hub>=0.25.0"
	pip install -q "decision-index[rebuild] @ git+https://github.com/apolinario/decision-index@9eb2dbe2a358004c8782c66e40a83ac07b953fec"

	echo "=== 3. Downloading V4 Assets & V3 Checkpoint from Hugging Face ==="
	rm -rf data/finetune-v2 checkpoints/pick-150m-v4
	mkdir -p data checkpoints
	python3 -c "
	import os, tarfile, shutil
	from huggingface_hub import hf_hub_download
	token = os.environ.get('HF_TOKEN')
	assert token, 'HF_TOKEN environment variable is required!'

	print('Downloading V4 finetune-v2.tar.gz...', flush=True)
	p1 = hf_hub_download(repo_id='lCREEl/pick-150m-runpod', filename='finetune-v2.tar.gz', token=token, force_download=True)
	with tarfile.open(p1) as tar:
	tar.extractall('data/')

	print('Downloading v3_stage1_best.pt and soup23_best.pt...', flush=True)
	hf_hub_download(repo_id='lCREEl/pick-150m-runpod', filename='v3_stage1_best.pt', token=token, local_dir='checkpoints')
	hf_hub_download(repo_id='lCREEl/pick-150m-runpod', filename='soup23_best.pt', token=token, local_dir='checkpoints')

	bench_p = 'data/benchmarks/crivo/public/rows.jsonl.gz'
	if not os.path.exists(bench_p):
	print('Downloading public benchmarks...', flush=True)
	os.makedirs('data/benchmarks/crivo/public', exist_ok=True)
	os.makedirs('data/benchmarks/crivo/reference/crivo-150m', exist_ok=True)
	bp = hf_hub_download(repo_id='lCREEl/pick-150m-runpod', filename='benchmark_public.jsonl.gz', token=token)
	shutil.copyfile(bp, bench_p)
	hf_hub_download(repo_id='lCREEl/pick-150m-runpod', filename='data/benchmarks/crivo/public/manifest.json', token=token, local_dir='.')
	hf_hub_download(repo_id='lCREEl/pick-150m-runpod', filename='data/benchmarks/crivo/reference/crivo-150m/results.jsonl.gz', token=token, local_dir='.')
	print('All V4 assets downloaded successfully!', flush=True)
	"

	export PYTHONPATH="$PWD/src${PYTHONPATH:+:$PYTHONPATH}"

	echo "=== 4A. V4 Stage 1: Auto-Calibrated Prior & Reasoning Refinement (400 steps, 2560 len) ==="
	python3 scripts/train.py \
	--model cross-encoder/ettin-reranker-150m-v1 \
	--arch pick \
	--head-type uncentered \
	--loss ce-brier \
	--init checkpoints/v3_stage1_best.pt \
	--save-dir checkpoints/pick-150m-v4/stage1 \
	--train-paths \
	data/finetune-v2/fmt-cladder-hard/train.jsonl \
	data/finetune-v2/fmt-when2call-aligned/train.jsonl \
	data/finetune-v2/fmt-hover-multihop/train.jsonl \
	data/finetune-v2/fmt-acos-calibrated/train.jsonl \
	data/finetune-v2/fmt-contractnli-dual/train.jsonl \
	data/finetune-v2/fmt-home-sim-hard/train.jsonl \
	data/finetune-v2/fmt-nli4ct-aligned/train.jsonl \
	data/finetune-v2/fmt-ragtruth-aligned/train.jsonl \
	data/finetune-v2/fmt-hellaswag/train.jsonl \
	--replay-paths \
	data/finetune-v2/anchor-replay/train.jsonl \
	--replay-share 0.25 \
	--mix-temperature 0.0 \
	--class-balance 0.0 \
	--dataset-weight \
	fmt-cladder-hard=3.5 \
	fmt-when2call-aligned=2.8 \
	fmt-hover-multihop=2.5 \
	fmt-acos-calibrated=2.2 \
	fmt-contractnli-dual=2.2 \
	fmt-home-sim-hard=2.0 \
	fmt-nli4ct-aligned=1.8 \
	fmt-ragtruth-aligned=1.6 \
	fmt-hellaswag=1.2 \
	--batch-cases 16 \
	--grad-accum 2 \
	--lr 6e-6 \
	--head-lr 2.5e-5 \
	--steps 400 \
	--warmup 30 \
	--schedule cosine \
	--eval-every 100 \
	--val-cases-per-set 50 \
	--save-every 100 \
	--max-length 2560 \
	--max-state-tokens 1536 \
	--max-question-tokens 512 \
	--device cuda \
	--precision bf16 \
	--workers 2

	STAGE1_PT="checkpoints/pick-150m-v4/stage1/best.pt"
	[ -f "$STAGE1_PT" ] \|\| STAGE1_PT="checkpoints/pick-150m-v4/stage1/final.pt"

	if [ -n "${HF_TOKEN:-}" ]; then
	echo "Backing up v4_stage1_best.pt to Hugging Face immediately..."
	python3 -c "
	import os
	from huggingface_hub import HfApi
	HfApi(token=os.environ['HF_TOKEN']).upload_file(
	path_or_fileobj='$STAGE1_PT',
	path_in_repo='v4_stage1_best.pt',
	repo_id='lCREEl/pick-150m-runpod',
	commit_message='feat: backup v4_stage1_best.pt after V4 Stage 1'
	)
	print('✅ v4_stage1_best.pt backed up to Hugging Face!')
	"
	fi

	echo "=== 4B. V4 Stage 2: Long-Context 6K Full-Document Polisher (200 steps, 6144 len) ==="
	python3 scripts/train.py \
	--model cross-encoder/ettin-reranker-150m-v1 \
	--arch pick \
	--head-type uncentered \
	--loss ce-brier \
	--init "$STAGE1_PT" \
	--save-dir checkpoints/pick-150m-v4/stage2 \
	--train-paths \
	data/finetune-v2/fmt-contractnli-dual/train.jsonl \
	data/finetune-v2/fmt-home-sim-hard/train.jsonl \
	data/finetune-v2/fmt-ragtruth-aligned/train.jsonl \
	data/finetune-v2/fmt-hover-multihop/train.jsonl \
	data/finetune-v2/fmt-when2call-aligned/train.jsonl \
	data/finetune-v2/fmt-acos-calibrated/train.jsonl \
	data/finetune-v2/fmt-nli4ct-aligned/train.jsonl \
	--replay-paths \
	data/finetune-v2/anchor-replay/train.jsonl \
	data/finetune-v2/fmt-cladder-hard/train.jsonl \
	data/finetune-v2/fmt-hellaswag/train.jsonl \
	--replay-share 0.35 \
	--mix-temperature 0.0 \
	--class-balance 0.0 \
	--dataset-weight \
	fmt-contractnli-dual=3.2 \
	fmt-home-sim-hard=2.2 \
	fmt-ragtruth-aligned=2.0 \
	fmt-hover-multihop=2.0 \
	fmt-when2call-aligned=2.0 \
	fmt-acos-calibrated=1.8 \
	fmt-nli4ct-aligned=1.5 \
	--batch-cases 8 \
	--grad-accum 4 \
	--lr 3e-6 \
	--head-lr 1.2e-5 \
	--steps 200 \
	--warmup 20 \
	--schedule cosine \
	--eval-every 100 \
	--val-cases-per-set 50 \
	--save-every 100 \
	--max-length 6144 \
	--max-state-tokens 4608 \
	--max-question-tokens 512 \
	--device cuda \
	--precision bf16 \
	--workers 2

	STAGE2_PT="checkpoints/pick-150m-v4/stage2/best.pt"
	[ -f "$STAGE2_PT" ] \|\| STAGE2_PT="checkpoints/pick-150m-v4/stage2/final.pt"

	echo "=== 5. Building V4 Model Soup (0.08 Base + 0.32 V3_Stage1 + 0.60 V4_Stage2) & Exporting 8K Model ==="
	SOUP_PT="checkpoints/pick-150m-v4/soup_v4.pt"
	python3 scripts/average-checkpoints.py \
	checkpoints/soup23_best.pt checkpoints/v3_stage1_best.pt "$STAGE2_PT" \
	--out "$SOUP_PT" \
	--weights 0.08 0.32 0.60

	python3 scripts/export-pipeline-checkpoint.py \
	--ckpt "$SOUP_PT" \
	--out checkpoints/pick-150m-v4/export \
	--model cross-encoder/ettin-reranker-150m-v1 \
	--max-length 7999 \
	--score-scaling normalized

	echo "=== 6. Uploading V4 Checkpoints & Exported Model to Hugging Face ==="
	if [ -n "${HF_TOKEN:-}" ]; then
	python3 -c "
	import os
	from huggingface_hub import HfApi
	api = HfApi(token=os.environ['HF_TOKEN'])
	api.upload_file(path_or_fileobj='$STAGE2_PT', path_in_repo='v4_stage2_best.pt', repo_id='lCREEl/pick-150m-runpod', commit_message='feat: upload v4_stage2_best.pt')
	api.upload_file(path_or_fileobj='$SOUP_PT', path_in_repo='v4_soup.pt', repo_id='lCREEl/pick-150m-runpod', commit_message='feat: upload v4_soup.pt')
	api.upload_folder(folder_path='checkpoints/pick-150m-v4/export', repo_id='lCREEl/pick-150m-runpod', commit_message='feat: export V4 Auto-Calibrated 8K model')
	print('✅ Uploaded V4 checkpoints and exported model to Hugging Face!')
	"
	fi

	echo "=== 7. Running Full 16-Benchmark Public Evaluation ==="
	python3 -m decision_transformer.evaluation.suite run checkpoints/pick-150m-v4/export \
	--name "pick-150m-v4-soup" \
	--device cuda \
	--suites public

	echo "=== 8. Running Post-Hoc Calibration Recipe Comparison ==="
	python3 scripts/calibrate_benchmark_results.py \
	--results reports/benchmarks/pick-150m-v4-soup/public/results.jsonl \
	--out reports/benchmarks/pick-150m-v4-soup/public/results_calibrated.jsonl

	if [ -n "${HF_TOKEN:-}" ] && [ -f "reports/benchmarks.jsonl" ]; then
	python3 -c "
	import os
	from huggingface_hub import HfApi
	api = HfApi(token=os.environ['HF_TOKEN'])
	api.upload_file(path_or_fileobj='reports/benchmarks.jsonl', path_in_repo='reports/benchmarks.jsonl', repo_id='lCREEl/pick-150m-runpod', commit_message='reports: add pick-150m-v4-soup to benchmark ledger')
	for p in ('reports/benchmarks/pick-150m-v4-soup/public/results.jsonl', 'reports/benchmarks/pick-150m-v4-soup/public/results_calibrated.jsonl'):
	if os.path.exists(p):
	api.upload_file(path_or_fileobj=p, path_in_repo=p, repo_id='lCREEl/pick-150m-runpod', commit_message=f'reports: upload {p}')
	print('✅ Uploaded V4 benchmark ledger and results to Hugging Face!')
	"
	fi

	echo "======================================================================"
	echo "🎉 V4 ALL FINISHED! Check reports/benchmarks/pick-150m-v4-soup"
	echo "======================================================================"