Download training/kernels/common/bootstrap.py from geraldaton20/alter_ego: direct link, hf CLI and curl.
- Browser
- Download file 2.42 kB
-
https://huggingface.co/geraldaton20/alter_ego/resolve/main/training/kernels/common/bootstrap.py
- Command line
-
hf download hf://geraldaton20/alter_ego/training/kernels/common/bootstrap.py
-
curl -L -o bootstrap.py https://huggingface.co/geraldaton20/alter_ego/resolve/main/training/kernels/common/bootstrap.py
2.42 kB
| import os | |
| import subprocess | |
| import sys | |
| DEPS = ["torch", "transformers", "datasets", "accelerate", "huggingface_hub"] | |
| def install_deps(): | |
| """Install any missing runtime deps (Kaggle T4 image usually has torch).""" | |
| missing = [] | |
| for m in DEPS: | |
| try: | |
| __import__(m) | |
| except Exception: | |
| missing.append(m) | |
| if missing: | |
| subprocess.run([sys.executable, "-m", "pip", "install", "-q", *missing], check=True) | |
| print(f"[bootstrap] deps ok ({len(DEPS)} libs)", flush=True) | |
| def hardware_guard(): | |
| """Fail fast unless exactly 2x Tesla T4 (the P100-fallback trap, risk #12).""" | |
| import torch | |
| n = torch.cuda.device_count() | |
| names = [torch.cuda.get_device_name(i) for i in range(n)] | |
| ok = (n == 2) and all("Tesla T4" in nm for nm in names) | |
| if not ok: | |
| with open("BOOT_FAILURE.txt", "w") as f: | |
| f.write(f"HARDWARE GUARD FAIL: device_count={n} names={names}\n") | |
| print(f"[bootstrap] HARDWARE GUARD FAIL: {n} devices, {names}", flush=True) | |
| sys.exit(1) | |
| print(f"[bootstrap] hardware OK: {n} x Tesla T4", flush=True) | |
| return names | |
| def get_hf_token_or_none(): | |
| return os.environ.get("HF_TOKEN") or os.environ.get("HF_TOKEN_ENV") | |
| def get_hf_token(): | |
| tok = get_hf_token_or_none() | |
| if not tok: | |
| raise RuntimeError("HF_TOKEN not found in environment/Kaggle secret") | |
| return tok | |
| def load_or_init_ckpt(ckpt_dir, repo_id, token, subdir="state"): | |
| """Download latest checkpoint shard set from HF into ckpt_dir, else fresh init. | |
| Returns (has_ckpt: bool, ckpt_dir).""" | |
| import shutil | |
| from huggingface_hub import HfApi | |
| os.makedirs(ckpt_dir, exist_ok=True) | |
| api = HfApi(token=token) | |
| try: | |
| files = list(api.list_repo_files(repo_id=repo_id, repo_type="model")) | |
| except Exception as e: | |
| print(f"[bootstrap] no remote ckpt ({e}); fresh init", flush=True) | |
| return False, ckpt_dir | |
| # Flat layout written by save_local: state_*.bin + meta.json (not a subdir). | |
| state_files = [f for f in files if f == "meta.json" or f.startswith("state_")] | |
| if not state_files: | |
| return False, ckpt_dir | |
| for f in state_files: | |
| api.hf_hub_download(repo_id=repo_id, filename=f, repo_type="model", | |
| local_dir=ckpt_dir, token=token) | |
| print(f"[bootstrap] resumed {len(state_files)} ckpt files", flush=True) | |
| return True, ckpt_dir | |