alter_ego / training /kernels /common /bootstrap.py
geraldaton20's picture
initial upload: source code (no build artifacts)
f4f166e verified
Raw History Blame Contribute Delete
2.42 kB
import os
import subprocess
import sys
DEPS = ["torch", "transformers", "datasets", "accelerate", "huggingface_hub"]
def install_deps():
"""Install any missing runtime deps (Kaggle T4 image usually has torch)."""
missing = []
for m in DEPS:
try:
__import__(m)
except Exception:
missing.append(m)
if missing:
subprocess.run([sys.executable, "-m", "pip", "install", "-q", *missing], check=True)
print(f"[bootstrap] deps ok ({len(DEPS)} libs)", flush=True)
def hardware_guard():
"""Fail fast unless exactly 2x Tesla T4 (the P100-fallback trap, risk #12)."""
import torch
n = torch.cuda.device_count()
names = [torch.cuda.get_device_name(i) for i in range(n)]
ok = (n == 2) and all("Tesla T4" in nm for nm in names)
if not ok:
with open("BOOT_FAILURE.txt", "w") as f:
f.write(f"HARDWARE GUARD FAIL: device_count={n} names={names}\n")
print(f"[bootstrap] HARDWARE GUARD FAIL: {n} devices, {names}", flush=True)
sys.exit(1)
print(f"[bootstrap] hardware OK: {n} x Tesla T4", flush=True)
return names
def get_hf_token_or_none():
return os.environ.get("HF_TOKEN") or os.environ.get("HF_TOKEN_ENV")
def get_hf_token():
tok = get_hf_token_or_none()
if not tok:
raise RuntimeError("HF_TOKEN not found in environment/Kaggle secret")
return tok
def load_or_init_ckpt(ckpt_dir, repo_id, token, subdir="state"):
"""Download latest checkpoint shard set from HF into ckpt_dir, else fresh init.
Returns (has_ckpt: bool, ckpt_dir)."""
import shutil
from huggingface_hub import HfApi
os.makedirs(ckpt_dir, exist_ok=True)
api = HfApi(token=token)
try:
files = list(api.list_repo_files(repo_id=repo_id, repo_type="model"))
except Exception as e:
print(f"[bootstrap] no remote ckpt ({e}); fresh init", flush=True)
return False, ckpt_dir
# Flat layout written by save_local: state_*.bin + meta.json (not a subdir).
state_files = [f for f in files if f == "meta.json" or f.startswith("state_")]
if not state_files:
return False, ckpt_dir
for f in state_files:
api.hf_hub_download(repo_id=repo_id, filename=f, repo_type="model",
local_dir=ckpt_dir, token=token)
print(f"[bootstrap] resumed {len(state_files)} ckpt files", flush=True)
return True, ckpt_dir