import os import subprocess import sys DEPS = ["torch", "transformers", "datasets", "accelerate", "huggingface_hub"] def install_deps(): """Install any missing runtime deps (Kaggle T4 image usually has torch).""" missing = [] for m in DEPS: try: __import__(m) except Exception: missing.append(m) if missing: subprocess.run([sys.executable, "-m", "pip", "install", "-q", *missing], check=True) print(f"[bootstrap] deps ok ({len(DEPS)} libs)", flush=True) def hardware_guard(): """Fail fast unless exactly 2x Tesla T4 (the P100-fallback trap, risk #12).""" import torch n = torch.cuda.device_count() names = [torch.cuda.get_device_name(i) for i in range(n)] ok = (n == 2) and all("Tesla T4" in nm for nm in names) if not ok: with open("BOOT_FAILURE.txt", "w") as f: f.write(f"HARDWARE GUARD FAIL: device_count={n} names={names}\n") print(f"[bootstrap] HARDWARE GUARD FAIL: {n} devices, {names}", flush=True) sys.exit(1) print(f"[bootstrap] hardware OK: {n} x Tesla T4", flush=True) return names def get_hf_token_or_none(): return os.environ.get("HF_TOKEN") or os.environ.get("HF_TOKEN_ENV") def get_hf_token(): tok = get_hf_token_or_none() if not tok: raise RuntimeError("HF_TOKEN not found in environment/Kaggle secret") return tok def load_or_init_ckpt(ckpt_dir, repo_id, token, subdir="state"): """Download latest checkpoint shard set from HF into ckpt_dir, else fresh init. Returns (has_ckpt: bool, ckpt_dir).""" import shutil from huggingface_hub import HfApi os.makedirs(ckpt_dir, exist_ok=True) api = HfApi(token=token) try: files = list(api.list_repo_files(repo_id=repo_id, repo_type="model")) except Exception as e: print(f"[bootstrap] no remote ckpt ({e}); fresh init", flush=True) return False, ckpt_dir # Flat layout written by save_local: state_*.bin + meta.json (not a subdir). state_files = [f for f in files if f == "meta.json" or f.startswith("state_")] if not state_files: return False, ckpt_dir for f in state_files: api.hf_hub_download(repo_id=repo_id, filename=f, repo_type="model", local_dir=ckpt_dir, token=token) print(f"[bootstrap] resumed {len(state_files)} ckpt files", flush=True) return True, ckpt_dir