File size: 2,424 Bytes
f4f166e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
import os
import subprocess
import sys

DEPS = ["torch", "transformers", "datasets", "accelerate", "huggingface_hub"]


def install_deps():
    """Install any missing runtime deps (Kaggle T4 image usually has torch)."""
    missing = []
    for m in DEPS:
        try:
            __import__(m)
        except Exception:
            missing.append(m)
    if missing:
        subprocess.run([sys.executable, "-m", "pip", "install", "-q", *missing], check=True)
    print(f"[bootstrap] deps ok ({len(DEPS)} libs)", flush=True)


def hardware_guard():
    """Fail fast unless exactly 2x Tesla T4 (the P100-fallback trap, risk #12)."""
    import torch
    n = torch.cuda.device_count()
    names = [torch.cuda.get_device_name(i) for i in range(n)]
    ok = (n == 2) and all("Tesla T4" in nm for nm in names)
    if not ok:
        with open("BOOT_FAILURE.txt", "w") as f:
            f.write(f"HARDWARE GUARD FAIL: device_count={n} names={names}\n")
        print(f"[bootstrap] HARDWARE GUARD FAIL: {n} devices, {names}", flush=True)
        sys.exit(1)
    print(f"[bootstrap] hardware OK: {n} x Tesla T4", flush=True)
    return names


def get_hf_token_or_none():
    return os.environ.get("HF_TOKEN") or os.environ.get("HF_TOKEN_ENV")


def get_hf_token():
    tok = get_hf_token_or_none()
    if not tok:
        raise RuntimeError("HF_TOKEN not found in environment/Kaggle secret")
    return tok


def load_or_init_ckpt(ckpt_dir, repo_id, token, subdir="state"):
    """Download latest checkpoint shard set from HF into ckpt_dir, else fresh init.
    Returns (has_ckpt: bool, ckpt_dir)."""
    import shutil
    from huggingface_hub import HfApi
    os.makedirs(ckpt_dir, exist_ok=True)
    api = HfApi(token=token)
    try:
        files = list(api.list_repo_files(repo_id=repo_id, repo_type="model"))
    except Exception as e:
        print(f"[bootstrap] no remote ckpt ({e}); fresh init", flush=True)
        return False, ckpt_dir
    # Flat layout written by save_local: state_*.bin + meta.json (not a subdir).
    state_files = [f for f in files if f == "meta.json" or f.startswith("state_")]
    if not state_files:
        return False, ckpt_dir
    for f in state_files:
        api.hf_hub_download(repo_id=repo_id, filename=f, repo_type="model",
                            local_dir=ckpt_dir, token=token)
    print(f"[bootstrap] resumed {len(state_files)} ckpt files", flush=True)
    return True, ckpt_dir