File size: 4,064 Bytes
8787bd3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 | """Deploy FATHOM training job to a GPU-powered HF Space.
Creates a Docker Space with L4 GPU ($0.80/hr) that:
1. Installs deps
2. Generates dataset (1000 train / 200 eval / 500 SFT)
3. Runs SFT warm-start on 0.5B model
4. Runs GRPO training (400 steps)
5. Pushes fine-tuned model to HF Hub
Usage: python scripts/deploy_training.py
Cost: ~$2-4 for 0.5B smoke, ~$8-15 for 1.5B full run
"""
from __future__ import annotations
import os
import sys
from pathlib import Path
if sys.platform == "win32":
import io
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
HF_TOKEN = os.getenv("HF_TOKEN")
SPACE_NAME = os.environ.get("FATHOM_TRAIN_SPACE", "Pratham-math/fathom-train")
REPO_ROOT = Path(__file__).parent.parent
def deploy():
if not HF_TOKEN:
print("ERROR: Set HF_TOKEN environment variable first")
sys.exit(1)
from huggingface_hub import HfApi
api = HfApi(token=HF_TOKEN)
me = api.whoami()
print(f"Logged in as: {me['name']}")
# Create GPU Space with L4 ($0.80/hr, 24GB VRAM)
print(f"Creating GPU Space: {SPACE_NAME} ...")
api.create_repo(
repo_id=SPACE_NAME,
repo_type="space",
space_sdk="docker",
private=False,
exist_ok=True,
)
# Set Space hardware to L4 GPU
try:
api.request_space_hardware(
repo_id=SPACE_NAME,
hardware="l4x1", # L4 24GB - $0.80/hr
)
print("Hardware set to L4 (24GB VRAM, $0.80/hr)")
except Exception as e:
print(f"Hardware request note: {e}")
print("You may need to set hardware manually at https://huggingface.co/spaces/{SPACE_NAME}/settings")
# Set HF_TOKEN as a Space secret (needed to push the fine-tuned model)
try:
api.add_space_secret(repo_id=SPACE_NAME, key="HF_TOKEN", value=HF_TOKEN)
print("HF_TOKEN secret set")
except Exception as e:
print(f"Secret set note: {e}")
# Collect ALL files needed for training
uploads = []
# Training Dockerfile
uploads.append((REPO_ROOT / "space" / "Dockerfile.train", "Dockerfile"))
# Space README with metadata
uploads.append((REPO_ROOT / "space" / "README_train.md", "README.md"))
# Run script
uploads.append((REPO_ROOT / "scripts" / "run_training.py", "run_training.py"))
# pyproject.toml
uploads.append((REPO_ROOT / "pyproject.toml", "pyproject.toml"))
# All Python packages
for pkg in ["train", "rewards", "data", "env", "configs"]:
pkg_dir = REPO_ROOT / pkg
if not pkg_dir.exists():
continue
for p in pkg_dir.rglob("*"):
if p.is_file() and not p.name.startswith(".") and "__pycache__" not in str(p):
rel = str(p.relative_to(REPO_ROOT)).replace("\\", "/")
uploads.append((p, rel))
# seeds.json
seeds = REPO_ROOT / "data" / "seeds.json"
if seeds.exists():
uploads.append((seeds, "data/seeds.json"))
print(f"Uploading {len(uploads)} files...")
for local, remote in uploads:
if local.exists():
api.upload_file(
path_or_fileobj=str(local),
path_in_repo=remote,
repo_id=SPACE_NAME,
repo_type="space",
commit_message=f"Train deploy: {remote}",
)
print(f" OK {remote}")
else:
print(f" SKIP {local}")
hf_url = f"https://huggingface.co/spaces/{SPACE_NAME}"
print("=" * 60)
print("Training Space deployed!")
print(f" Monitor: {hf_url}")
print(f" Logs: {hf_url}?logs=container")
print()
print("The Space will:")
print(" 1. Build Docker image (~3 min)")
print(" 2. Generate dataset")
print(" 3. Run SFT warm-start on 0.5B model (~10 min)")
print(" 4. Run GRPO 400 steps (~2-3 hrs on L4)")
print(" 5. Push fine-tuned model to Pratham-math/fathom-0.5b-grpo")
print()
print("Estimated cost: $2-4 for 0.5B run")
print("=" * 60)
if __name__ == "__main__":
deploy()
|