Download training/kernels/train.py from geraldaton20/alter_ego: direct link, hf CLI and curl.
- Browser
- Download file 30.8 kB
-
https://huggingface.co/geraldaton20/alter_ego/resolve/main/training/kernels/train.py
- Command line
-
hf download hf://geraldaton20/alter_ego/training/kernels/train.py
-
curl -L -o train.py https://huggingface.co/geraldaton20/alter_ego/resolve/main/training/kernels/train.py
30.8 kB
| """Stage 2+ training kernel: auto-checkpoint + auto-resume (self-contained). | |
| Mirrors orchestrator/hf_ckpt + kernels/common. Kaggle script push does NOT bundle | |
| subpackages, so every helper is inlined. Fail-fast on guard failures. | |
| Boot: resume latest valid ckpt from HF (or fresh init). If that ckpt is FINAL, | |
| exit cheaply after re-writing DONE (no weight re-upload). | |
| Loop: AMP-fp16 train -> NaN watchdog -> checkpoint every N min (val probe + | |
| metrics, shards-first/meta-last) -> heartbeat every 5 min -> forced exit at | |
| MAX_MINUTES or GPU-hour cap WITHOUT DONE (orchestrator relaunches) -> | |
| completion at MAX_STEPS writes DONE + final artifacts. | |
| """ | |
| import io | |
| import json | |
| import math | |
| import os | |
| import subprocess | |
| import sys | |
| import time | |
| import numpy as np | |
| # ---------------- config (mirrors config/pipeline.yaml) ---------------- | |
| CFG = { | |
| "student": {"vocab": 8192, "d_model": 512, "n_layers": 16, "ffn_ratio": 4, | |
| "n_heads": 8, "max_seq_len": 256, "tied_embeddings": True}, | |
| "training": {"lr": 3.0e-3, "warmup_frac": 0.05, "amp": "fp16"}, | |
| "hf_ckpt_repo": "geraldaton20/student-ckpts-v1", | |
| "checkpoint_interval_min": 10, | |
| "heartbeat_interval_min": 5, | |
| } | |
| REPO = os.environ.get("HF_CKPT_REPO", CFG["hf_ckpt_repo"]) | |
| CKPT_DIR = "ckpt" | |
| MAX_MINUTES = int(os.environ.get("MAX_MINUTES", "240")) | |
| MAX_STEPS = int(os.environ.get("MAX_STEPS", "21362")) or None | |
| QAT_START_FRAC = float(os.environ.get("QAT_START_FRAC", "0.75")) | |
| GPU_HOUR_CAP = float(os.environ.get("GPU_HOUR_CAP", "30")) # yaml hard cap | |
| # ---- Stage-5 real-data mode (ae-corpus-v1 + ae-labels-v1) ------------------- | |
| AE_DRY = os.environ.get("AE_DRY", "0") == "1" # force synthetic dry-run | |
| AE_ALLOW_DRY_FALLBACK = os.environ.get("AE_ALLOW_DRY", "0") == "1" | |
| DISTILL_ALPHA = float(os.environ.get("DISTILL_ALPHA", "0.75")) | |
| SOFT_TOPK = int(os.environ.get("SOFT_TOPK", "64")) | |
| CKPT_INT = CFG["checkpoint_interval_min"] * 60 | |
| HB_INT = CFG["heartbeat_interval_min"] * 60 | |
| # ---------------- bootstrap (inlined) ---------------- | |
| def install_deps(): | |
| missing = [] | |
| for m in ("torch", "transformers", "datasets", "accelerate", "huggingface_hub"): | |
| try: | |
| __import__(m) | |
| except Exception: | |
| missing.append(m) | |
| if missing: | |
| subprocess.run([sys.executable, "-m", "pip", "install", "-q", *missing], check=True) | |
| print("[bootstrap] deps ok", flush=True) | |
| def get_hf_token_or_none(): | |
| if HF_TOKEN_INJECTED and HF_TOKEN_INJECTED != "_HF_TOKEN_PLACEHOLDER_": | |
| return HF_TOKEN_INJECTED | |
| tok = os.environ.get("HF_TOKEN") or os.environ.get("HF_TOKEN_ENV") | |
| if tok: | |
| return tok | |
| try: | |
| from kaggle_secrets import UserSecretsClient | |
| return UserSecretsClient().get_secret("HF_TOKEN") | |
| except Exception: | |
| return None | |
| def _try_push_text(fname, text, token): | |
| if not token: | |
| return | |
| try: | |
| from huggingface_hub import HfApi | |
| p = os.path.abspath(fname) | |
| with open(p, "w") as f: | |
| f.write(text) | |
| HfApi(token=token).upload_file(path_or_fileobj=p, path_in_repo=fname, | |
| repo_id=REPO, repo_type="model", token=token) | |
| print(f"[bootstrap] pushed {fname}", flush=True) | |
| except Exception as e: | |
| print(f"[bootstrap] {fname} push failed (non-fatal): {e}", flush=True) | |
| def hardware_guard(): | |
| import torch | |
| n = torch.cuda.device_count() | |
| names = [torch.cuda.get_device_name(i) for i in range(n)] | |
| ok = (n == 2) and all("Tesla T4" in nm for nm in names) | |
| if not ok: | |
| msg = f"HARDWARE GUARD FAIL: device_count={n} names={names}\n" | |
| _try_push_text("BOOT_FAILURE.txt", msg, get_hf_token_or_none()) | |
| print(f"[bootstrap] {msg}", flush=True) | |
| sys.exit(1) | |
| print(f"[bootstrap] hardware OK: {n} x Tesla T4", flush=True) | |
| # Populated at push time by the orchestrator (private kernel only). The repo | |
| # copy keeps the placeholder; kaggle_client injects the real token. | |
| HF_TOKEN_INJECTED = "_HF_TOKEN_PLACEHOLDER_" | |
| def load_or_init_ckpt(ckpt_dir, repo_id, token): | |
| from huggingface_hub import HfApi | |
| os.makedirs(ckpt_dir, exist_ok=True) | |
| api = HfApi(token=token) | |
| try: | |
| files = list(api.list_repo_files(repo_id=repo_id, repo_type="model")) | |
| except Exception as e: | |
| print(f"[bootstrap] no remote ckpt ({e}); fresh init", flush=True) | |
| return False | |
| state_files = [f for f in files if f == "meta.json" or f.startswith("state_")] | |
| if not state_files: | |
| print("[bootstrap] fresh init (no ckpt on HF)", flush=True) | |
| return False | |
| for f in state_files: | |
| api.hf_hub_download(repo_id=repo_id, filename=f, repo_type="model", | |
| local_dir=ckpt_dir, token=token) | |
| print(f"[bootstrap] resumed {len(state_files)} ckpt files", flush=True) | |
| return True | |
| # ---------------- model builder (inlined) ---------------- | |
| def build_model(cfg): | |
| import torch | |
| import torch.nn as nn | |
| d, L, ffn = cfg["d_model"], cfg["n_layers"], cfg["ffn_ratio"] * cfg["d_model"] | |
| h, v, maxlen = cfg["n_heads"], cfg["vocab"], cfg["max_seq_len"] | |
| class Attn(nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| self.qkv = nn.Linear(d, 3 * d, bias=False) | |
| self.o = nn.Linear(d, d, bias=False) | |
| def forward(self, x): | |
| B, T, _ = x.shape | |
| q, k, vv = self.qkv(x).chunk(3, -1) | |
| q = q.view(B, T, h, d // h).transpose(1, 2) | |
| k = k.view(B, T, h, d // h).transpose(1, 2) | |
| vv = vv.view(B, T, h, d // h).transpose(1, 2) | |
| o = torch.nn.functional.scaled_dot_product_attention(q, k, vv) | |
| return self.o(o.transpose(1, 2).reshape(B, T, d)) | |
| class Block(nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| self.ln1 = nn.LayerNorm(d) | |
| self.attn = Attn() | |
| self.ln2 = nn.LayerNorm(d) | |
| self.mlp = nn.Sequential(nn.Linear(d, ffn), nn.GELU(), nn.Linear(ffn, d)) | |
| def forward(self, x): | |
| x = x + self.attn(self.ln1(x)) | |
| return x + self.mlp(self.ln2(x)) | |
| class SAN(nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| self.tok_emb = nn.Embedding(v, d) | |
| self.pos = nn.Parameter(torch.zeros(1, maxlen, d)) | |
| self.blocks = nn.ModuleList([Block() for _ in range(L)]) | |
| self.ln_f = nn.LayerNorm(d) | |
| self.head = nn.Linear(d, v, bias=False) | |
| if cfg.get("tied_embeddings", True): | |
| self.head.weight = self.tok_emb.weight | |
| def forward(self, idx): | |
| B, T = idx.shape | |
| x = self.tok_emb(idx) + self.pos[:, :T] | |
| for b in self.blocks: | |
| x = b(x) | |
| return self.head(self.ln_f(x)) | |
| return SAN() | |
| def spec_guard(model, cfg): | |
| """Fail fast unless the built model matches the PRD/yaml spec (~50M SAN, | |
| embedding share well under the vocab-trap zone).""" | |
| s = cfg["student"] | |
| emb = s["vocab"] * s["d_model"] + s["max_seq_len"] * s["d_model"] | |
| total = sum(p.numel() for p in model.parameters()) | |
| frac = emb / total | |
| lines = [f"params_total={total:,}", f"params_emb={emb:,}", f"emb_frac={frac:.3f}"] | |
| print("[spec] " + " | ".join(lines), flush=True) | |
| problems = [] | |
| if not (45_000_000 <= total <= 60_000_000): | |
| problems.append(f"param count {total:,} outside 45-60M spec") | |
| if frac > 0.126: | |
| problems.append(f"embedding share {frac:.1%} violates vocab-trap rule (>12.6%)") | |
| if s["d_model"] % s["n_heads"] != 0: | |
| problems.append("d_model not divisible by n_heads") | |
| if problems: | |
| _try_push_text("BOOT_FAILURE.txt", "\n".join(problems), get_hf_token_or_none()) | |
| raise SystemExit("SPEC GUARD FAIL: " + "; ".join(problems)) | |
| # ---------------- checkpoint lib (inlined: save/load/push) ---------------- | |
| SHARD_BYTES = 48 * 1024 * 1024 | |
| def _make_json_safe(obj): | |
| if isinstance(obj, bytes): | |
| return {"__bytes__": obj.decode("latin-1")} | |
| if isinstance(obj, tuple): | |
| return [_make_json_safe(x) for x in obj] | |
| if isinstance(obj, list): | |
| return [_make_json_safe(x) for x in obj] | |
| if isinstance(obj, dict): | |
| return {k: _make_json_safe(v) for k, v in obj.items()} | |
| return obj | |
| def _sha256_file(path): | |
| import hashlib | |
| h = hashlib.sha256() | |
| with open(path, "rb") as f: | |
| for chunk in iter(lambda: f.read(1 << 20), b""): | |
| h.update(chunk) | |
| return h.hexdigest() | |
| def save_local(ckpt_dir, state, tensor_meta, meta, shard_bytes=SHARD_BYTES): | |
| """Spanning sharder: any blob (incl. multi-hundred-MB optimizer bundles) is | |
| split across as many shards as needed. assignments[name] = [[shard, off, | |
| len], ...]. Removes stale state_*.bin from prior generations first.""" | |
| os.makedirs(ckpt_dir, exist_ok=True) | |
| for f in os.listdir(ckpt_dir): | |
| if f.startswith("state_") and f.endswith(".bin"): | |
| os.remove(os.path.join(ckpt_dir, f)) | |
| shards, assignments = [], {} | |
| cur, cur_off = bytearray(), 0 | |
| def flush(): | |
| nonlocal cur, cur_off | |
| if cur: | |
| shards.append(bytes(cur)) | |
| cur, cur_off = bytearray(), 0 | |
| for name in sorted(state.keys()): | |
| header = json.dumps({ | |
| "name": name, | |
| "shape": tensor_meta[name]["shape"], | |
| "dtype": tensor_meta[name]["dtype"], | |
| "nbytes": len(state[name]), | |
| }).encode() | |
| blob = header + b"\n" + state[name] | |
| segs = [] | |
| pos = 0 | |
| while pos < len(blob): | |
| if len(cur) >= shard_bytes: | |
| flush() | |
| take = min(shard_bytes - len(cur), len(blob) - pos) | |
| segs.append([len(shards), cur_off, take]) | |
| cur += blob[pos:pos + take] | |
| cur_off += take | |
| pos += take | |
| if len(cur) >= shard_bytes: | |
| flush() | |
| assignments[name] = segs | |
| flush() | |
| shard_files, shard_shas = [], [] | |
| for i, s in enumerate(shards): | |
| fn = f"state_{i:03d}.bin" | |
| with open(os.path.join(ckpt_dir, fn), "wb") as f: | |
| f.write(s) | |
| shard_files.append(fn) | |
| shard_shas.append(_sha256_file(os.path.join(ckpt_dir, fn))) | |
| out = dict(meta) | |
| out["shards"] = shard_files | |
| out["shard_shas"] = shard_shas | |
| out["assignments"] = assignments | |
| out["tensor_meta"] = tensor_meta | |
| out["created"] = time.time() | |
| tmp = os.path.join(ckpt_dir, "meta.json.tmp") | |
| with open(tmp, "w") as f: | |
| json.dump(_make_json_safe(out), f) | |
| os.replace(tmp, os.path.join(ckpt_dir, "meta.json")) # atomic final write | |
| return out | |
| def load_local(ckpt_dir, verify_sha=True): | |
| meta_path = os.path.join(ckpt_dir, "meta.json") | |
| if not os.path.exists(meta_path): | |
| raise FileNotFoundError(f"no meta.json in {ckpt_dir}") | |
| with open(meta_path) as f: | |
| meta = json.load(f) | |
| if verify_sha: | |
| for fn, exp in zip(meta["shards"], meta["shard_shas"]): | |
| actual = _sha256_file(os.path.join(ckpt_dir, fn)) | |
| if actual != exp: | |
| raise ValueError(f"shard {fn} sha mismatch (corrupt/partial)") | |
| shard_data = {fn: open(os.path.join(ckpt_dir, fn), "rb").read() | |
| for fn in meta["shards"]} | |
| state = {} | |
| for name, segs in meta["assignments"].items(): | |
| if segs and isinstance(segs[0], int): | |
| i, off = segs # legacy single-segment layout | |
| blob = shard_data[meta["shards"][i]][off:] | |
| nl = blob.index(b"\n") | |
| header = json.loads(blob[:nl].decode()) | |
| state[name] = blob[nl + 1: nl + 1 + header["nbytes"]] | |
| else: | |
| buf = b"".join(shard_data[meta["shards"][i]][o:o + n] for i, o, n in segs) | |
| nl = buf.index(b"\n") | |
| header = json.loads(buf[:nl].decode()) | |
| state[name] = buf[nl + 1: nl + 1 + header["nbytes"]] | |
| return state, meta | |
| def _upload_with_retry(api, local_path, repo_path, repo_id, token, attempts=4): | |
| import time as _t | |
| last = None | |
| for i in range(attempts): | |
| try: | |
| api.upload_file(path_or_fileobj=local_path, path_in_repo=repo_path, | |
| repo_id=repo_id, repo_type="model", token=token) | |
| return | |
| except Exception as e: | |
| last = e | |
| print(f"[push] {repo_path} attempt {i + 1}/{attempts} failed: {e}", flush=True) | |
| _t.sleep(2 * (i + 1)) | |
| raise last | |
| def push_hf(ckpt_dir, repo_id, token=None): | |
| from huggingface_hub import HfApi | |
| api = HfApi(token=token) | |
| files = sorted(os.listdir(ckpt_dir)) | |
| shards = [f for f in files if f.startswith("state_") and f.endswith(".bin")] | |
| # CRITICAL: shards + meta.json must stay mutually consistent. Upload that | |
| # pair first (meta LAST within the pair) so a flaky auxiliary upload can | |
| # never strand a stale meta.json while new shards are already live. | |
| for f in shards: | |
| _upload_with_retry(api, os.path.join(ckpt_dir, f), f, repo_id, token) | |
| if "meta.json" in files: | |
| _upload_with_retry(api, os.path.join(ckpt_dir, "meta.json"), | |
| "meta.json", repo_id, token) | |
| others = [f for f in files | |
| if f not in shards and f != "meta.json" and not f.endswith(".tmp")] | |
| for f in others: | |
| try: | |
| _upload_with_retry(api, os.path.join(ckpt_dir, f), f, repo_id, token) | |
| except Exception as e: | |
| print(f"[push] aux {f} upload failed (non-fatal): {e}", flush=True) | |
| def push_file(local_path, repo_path, token): | |
| from huggingface_hub import HfApi | |
| _upload_with_retry(HfApi(token=token), local_path, repo_path, REPO, token) | |
| # ---------------- serialize / deserialize ---------------- | |
| def sd_to_state(sd): | |
| state, tm = {}, {} | |
| for k, v in sd.items(): | |
| arr = v.detach().cpu().contiguous().numpy() | |
| state[k] = arr.tobytes() | |
| tm[k] = {"shape": list(arr.shape), "dtype": str(arr.dtype)} | |
| return state, tm | |
| def state_to_sd(state, tm): | |
| import torch | |
| sd = {} | |
| for k in state: | |
| if k.startswith("__"): | |
| continue | |
| dt = np.dtype(tm[k]["dtype"]) | |
| arr = np.frombuffer(state[k], dt).reshape(tm[k]["shape"]) | |
| sd[k] = torch.from_numpy(arr.copy()) | |
| return sd | |
| # ---------------- Stage-5 real data (corpus + teacher labels) ---------------- | |
| def _kaggle_creds(): | |
| try: | |
| from kaggle_secrets import UserSecretsClient | |
| c = UserSecretsClient() | |
| return c.get_secret("KAGGLE_USERNAME"), c.get_secret("KAGGLE_API_TOKEN") | |
| except Exception: | |
| return None, None | |
| def _fetch_dataset(suffix, dst): | |
| user, key = _kaggle_creds() | |
| if not (user and key): | |
| print(f"[data] no Kaggle creds; cannot fetch {suffix}", flush=True) | |
| return False | |
| subprocess.run([sys.executable, "-m", "pip", "install", "-q", "kaggle"], check=True) | |
| env = dict(os.environ, KAGGLE_USERNAME=user, KAGGLE_KEY=key) | |
| r = subprocess.run([sys.executable, "-m", "kaggle", "datasets", "download", | |
| f"{user}/{suffix}", "-p", dst, "--unzip"], | |
| env=env, capture_output=True, text=True) | |
| print(f"[data] {user}/{suffix} rc={r.returncode} " | |
| f"{((r.stdout or '') + (r.stderr or ''))[-160:]}", flush=True) | |
| return r.returncode == 0 and os.path.isdir(dst) | |
| class RealSampler: | |
| """256-token windows from ae-corpus-v1 (+ optional teacher top-K rows | |
| aligned to stream positions for hybrid distillation loss).""" | |
| def __init__(self, work, dev): | |
| import numpy as np | |
| self.np = np | |
| self.dev = dev | |
| cdir = os.path.join(work, "corpus") | |
| self.tokens = np.memmap(os.path.join(cdir, "tokens.i16"), dtype="<i2", mode="r") | |
| z = np.load(os.path.join(cdir, "arrays.npz")) | |
| lengths = z["lengths"].astype("i8") | |
| self.n_windows = int(self.tokens.shape[0]) // 256 | |
| ldir = os.path.join(work, "labels") | |
| li, lp = os.path.join(ldir, "labels_idx.u16"), os.path.join(ldir, "labels_prob.f16") | |
| lm = os.path.join(ldir, "label_meta.npz") | |
| self.has_labels = all(os.path.exists(p) for p in (li, lp, lm)) | |
| if self.has_labels: | |
| m = np.load(lm) | |
| doc = m["doc"].astype("i8") | |
| llen = m["len"].astype("i8") | |
| order = np.argsort(doc) | |
| doc, llen = doc[order], llen[order] | |
| starts_all = np.concatenate([[0], np.cumsum(lengths)])[:-1] | |
| self.st = torch.from_numpy(starts_all[doc].copy()).to(dev) | |
| self.en = self.st + torch.from_numpy(llen.copy()).to(dev) | |
| base = np.concatenate([[0], np.cumsum(llen)])[:-1] | |
| self.base = torch.from_numpy(base).to(dev) | |
| self.idx_mm = np.memmap(li, dtype="<u2", mode="r").reshape(-1, SOFT_TOPK) | |
| self.prob_mm = np.memmap(lp, dtype="<f2", mode="r").reshape(-1, SOFT_TOPK) | |
| print(f"[data] labels: docs={len(doc)} tok={int(llen.sum()):,} " | |
| f"topk={SOFT_TOPK}", flush=True) | |
| else: | |
| print("[data] labels absent -> pure hard CE on corpus", flush=True) | |
| def sample(self, B): | |
| np = self.np | |
| widx = np.random.randint(0, max(1, self.n_windows), size=B) | |
| offs = (widx[:, None] * 256 + np.arange(256)[None, :]).astype("i8").ravel() | |
| win = torch.from_numpy(self.tokens[offs].astype("i64")).view(B, 256).to(self.dev) | |
| inp, tgt = win[:, :-1], win[:, 1:] | |
| P = tgt.reshape(-1) | |
| if not self.has_labels: | |
| return inp, tgt, None, None | |
| j = (torch.searchsorted(self.st.contiguous(), P, right=True) - 1).clamp( | |
| min=0, max=self.en.numel() - 1) | |
| valid = (P >= self.st[j]) & (P < self.en[j]) | |
| rows = (self.base[j] + (P - self.st[j])).long() | |
| rows = torch.where(valid, rows, torch.full_like(rows, -1)) | |
| return inp, tgt, rows, valid | |
| def teacher_rows(self, urows_np): | |
| idx = self.idx_mm[urows_np].astype("i64") | |
| pr = self.prob_mm[urows_np].astype("f4").clip(min=0) | |
| pr /= np.clip(pr.sum(axis=1, keepdims=True), 1e-9, None) | |
| lp = np.log(np.clip(pr, 1e-9, None)) | |
| return idx, lp | |
| def resolve_data(work, dev): | |
| """Returns ('real', sampler) | ('dry', None) — never silent noise.""" | |
| global data_mode, sampler | |
| if AE_DRY: | |
| print("[data] AE_DRY=1 -> synthetic dry-run mode", flush=True) | |
| data_mode, sampler = "dry", None | |
| return | |
| ok_c = _fetch_dataset("ae-corpus-v1", os.path.join(work, "corpus_ds")) \ | |
| or os.path.exists(os.path.join(work, "corpus_ds", "tokens.i16")) | |
| cdir = os.path.join(work, "corpus_ds") | |
| have_corpus = os.path.exists(os.path.join(cdir, "tokens.i16")) | |
| if not have_corpus: | |
| _fetch_dataset("ae-corpus-v1", cdir) | |
| have_corpus = os.path.exists(os.path.join(cdir, "tokens.i16")) | |
| if not have_corpus: | |
| msg = ("Stage-5 needs ae-corpus-v1 but it is missing. " | |
| "Refusing to train on noise. Finish preprocess first, " | |
| "or set AE_DRY=1 explicitly.") | |
| print(f"[data] FATAL: {msg}", flush=True) | |
| _try_push_text("FAILURE.txt", msg + "\n", get_hf_token_or_none()) | |
| raise SystemExit(msg) | |
| _fetch_dataset("ae-labels-v1", os.path.join(work, "labels")) | |
| s = RealSampler(cdir, dev) | |
| data_mode, sampler = "real", s | |
| # ---------------- main ---------------- | |
| data_mode, sampler = "dry", None | |
| def main(): | |
| install_deps() | |
| hardware_guard() | |
| token = get_hf_token_or_none() | |
| if not token: | |
| print("[train] WARNING: no HF token; checkpoints local-only (no cross-session resume)", flush=True) | |
| import torch | |
| s = CFG["student"] | |
| V = s["vocab"] | |
| model = build_model(s) | |
| spec_guard(model if not hasattr(model, "module") else model.module, CFG) | |
| if torch.cuda.device_count() > 1: | |
| model = torch.nn.DataParallel(model) | |
| model.cuda() | |
| opt = torch.optim.AdamW(model.parameters(), lr=CFG["training"]["lr"]) | |
| scaler = torch.cuda.amp.GradScaler() # yaml: amp fp16 | |
| def core(mdl): | |
| return mdl.module if hasattr(mdl, "module") else mdl | |
| def load_weights(sd): | |
| core(model).load_state_dict(sd) | |
| step, tokens, qat_enabled = 0, 0, False | |
| train_seconds = 0.0 | |
| resumed = load_or_init_ckpt(CKPT_DIR, REPO, token) if token else False | |
| if resumed: | |
| try: | |
| st, meta = load_local(CKPT_DIR) | |
| load_weights(state_to_sd(st, meta["tensor_meta"])) | |
| if "__bundle__" in st: | |
| d = torch.load(io.BytesIO(st["__bundle__"]), weights_only=False) | |
| opt.load_state_dict(d["opt"]) | |
| scaler.load_state_dict(d["scaler"]) | |
| torch.set_rng_state(d["rng_cpu"].clone()) | |
| for i, cst in enumerate(d["rng_cuda"]): | |
| torch.cuda.set_rng_state(cst.cpu(), i) | |
| else: | |
| opt.load_state_dict(torch.load(io.BytesIO(st["__optim__"]), weights_only=False)) | |
| step, tokens = meta["step"], meta["tokens"] | |
| qat_enabled = meta.get("qat_enabled", False) | |
| train_seconds = float(meta.get("train_seconds_total", 0.0)) | |
| print(f"[train] RESUMED at step={step} tokens={tokens} gpu_hours={train_seconds/3600:.2f}", flush=True) | |
| if meta.get("final"): | |
| # Already complete: refresh DONE cheaply, skip all weight uploads. | |
| with open(os.path.join(CKPT_DIR, "DONE"), "w") as f: | |
| f.write("training complete\n") | |
| if token: | |
| try: | |
| push_file(os.path.join(CKPT_DIR, "DONE"), "DONE", token) | |
| except Exception as e: | |
| print(f"[train] DONE push failed: {e}", flush=True) | |
| print(f"[train] checkpoint already FINAL @ step={step}; nothing to do", flush=True) | |
| return | |
| except Exception as e: | |
| import traceback | |
| traceback.print_exc() | |
| print(f"[train] resume failed ({e}); fresh init", flush=True) | |
| resumed = False | |
| if not resumed: | |
| print("[train] FRESH INIT", flush=True) | |
| B, T = 128, s["max_seq_len"] | |
| resolve_data("/kaggle/tmp/aedata", "cuda") | |
| warmup = max(1, int((MAX_STEPS or 100000) * CFG["training"]["warmup_frac"])) | |
| def lr_now(): | |
| if step < warmup: | |
| return CFG["training"]["lr"] * (step + 1) / warmup | |
| prog = (step - warmup) / max(1, (MAX_STEPS or 100000) - warmup) | |
| return CFG["training"]["lr"] * 0.5 * (1 + math.cos(math.pi * min(1.0, prog))) | |
| if data_mode == "real": | |
| val_inp, val_tgt, _, _ = sampler.sample(32) | |
| else: | |
| val_inp = torch.randint(0, V, (B, T), device="cuda") | |
| val_tgt = torch.randint(0, V, (B, T), device="cuda") | |
| def val_probe(): | |
| model.eval() | |
| with torch.autocast(device_type="cuda", dtype=torch.float16): | |
| vl = torch.nn.functional.cross_entropy( | |
| model(val_inp).view(-1, V), val_tgt.view(-1)).item() | |
| model.train() | |
| return vl | |
| t_start = time.time() | |
| last_ckpt = t_start | |
| last_hb = t_start | |
| running_loss, n_loss = 0.0, 0 | |
| nan_streak = 0 | |
| def write_heartbeat(): | |
| with open(os.path.join(CKPT_DIR, "heartbeat.json"), "w") as f: | |
| json.dump({"step": step, "tokens": tokens, "ts": time.time(), | |
| "gpu_hours": round(train_seconds / 3600, 3)}, f) | |
| if token: | |
| try: | |
| push_file(os.path.join(CKPT_DIR, "heartbeat.json"), "heartbeat.json", token) | |
| except Exception as e: | |
| print(f"[train] heartbeat push failed: {e}", flush=True) | |
| def snapshot_state(): | |
| sd = core(model).state_dict() | |
| state, tm = sd_to_state(sd) | |
| bundle = io.BytesIO() | |
| torch.save({ | |
| "opt": opt.state_dict(), | |
| "scaler": scaler.state_dict(), | |
| "rng_cpu": torch.get_rng_state(), | |
| "rng_cuda": torch.cuda.get_rng_state_all(), | |
| }, bundle) | |
| b = bundle.getvalue() | |
| state["__bundle__"] = b | |
| tm["__bundle__"] = {"shape": [len(b)], "dtype": "bytes"} | |
| return state, tm | |
| def save_checkpoint(final=False): | |
| nonlocal step, tokens, qat_enabled, train_seconds | |
| state, tm = snapshot_state() | |
| vl = val_probe() | |
| meta = {"step": step, "tokens": tokens, "qat_enabled": qat_enabled, | |
| "final": final, "created": time.time(), "data_mode": data_mode, | |
| "distill_alpha": DISTILL_ALPHA, | |
| "train_seconds_total": train_seconds, "val_loss": vl} | |
| save_local(CKPT_DIR, state, tm, meta) | |
| with open(os.path.join(CKPT_DIR, "metrics.jsonl"), "a") as f: | |
| f.write(json.dumps({"step": step, "tokens": tokens, | |
| "loss": round(running_loss / max(1, n_loss), 4), | |
| "val_loss": round(vl, 4), | |
| "gpu_hours": round(train_seconds / 3600, 3), | |
| "ts": time.time()}) + "\n") | |
| if token: | |
| try: | |
| push_hf(CKPT_DIR, REPO, token) | |
| print(f"[train] checkpoint pushed @ step={step} tokens={tokens} " | |
| f"loss={running_loss/max(1,n_loss):.4f} val_loss={vl:.4f}", flush=True) | |
| except Exception as e: | |
| print(f"[train] push failed: {e}", flush=True) | |
| running_loss, n_loss = 0.0, 0 # per-interval window (cleaner telemetry) | |
| def rollback_last_ckpt(): | |
| try: | |
| st, meta = load_local(CKPT_DIR) | |
| load_weights(state_to_sd(st, meta["tensor_meta"])) | |
| if "__bundle__" in st: | |
| d = torch.load(io.BytesIO(st["__bundle__"]), weights_only=False) | |
| opt.load_state_dict(d["opt"]) | |
| scaler.load_state_dict(d["scaler"]) | |
| print("[watchdog] rolled back to last checkpoint", flush=True) | |
| return True | |
| except Exception as e: | |
| print(f"[watchdog] rollback failed: {e}", flush=True) | |
| return False | |
| if not resumed: | |
| save_checkpoint() # first valid ckpt lands on HF promptly | |
| write_heartbeat() | |
| while True: | |
| if MAX_STEPS and step >= QAT_START_FRAC * MAX_STEPS and not qat_enabled: | |
| qat_enabled = True | |
| print(f"[train] QAT phase flagged @ step={step} (fake-quant lands with Stage 5)", flush=True) | |
| blk = time.time() | |
| for _ in range(20): | |
| if data_mode == "real": | |
| inp, tgt, lrows, vmask = sampler.sample(B) | |
| else: | |
| inp = torch.randint(0, V, (B, T), device="cuda") | |
| tgt = torch.randint(0, V, (B, T), device="cuda") | |
| lrows = None | |
| for pg in opt.param_groups: | |
| pg["lr"] = lr_now() | |
| opt.zero_grad(set_to_none=True) | |
| with torch.autocast(device_type="cuda", dtype=torch.float16): | |
| logits = model(inp).view(-1, V) | |
| hard = torch.nn.functional.cross_entropy( | |
| logits.float(), tgt.reshape(-1), reduction="none") | |
| if lrows is not None: | |
| vsel = vmask.nonzero(as_tuple=True)[0] | |
| soft_vec = torch.zeros_like(hard) | |
| if vsel.numel() > 0: | |
| urows, _inv = lrows[vsel].unique(return_inverse=True) | |
| ur_np = urows.cpu().numpy() | |
| t_idx = torch.from_numpy( | |
| sampler.teacher_rows(ur_np)[0]).to(inp.device).long() | |
| t_lp = torch.from_numpy( | |
| sampler.teacher_rows(ur_np)[1]).to(inp.device).float() | |
| sl = logits.index_select(0, vsel).gather(1, t_idx) | |
| q = torch.log_softmax(sl.float(), dim=-1) | |
| kl = (t_lp.exp() * (t_lp - q)).sum(-1) | |
| soft_vec[vsel] = kl | |
| loss_vec = torch.where( | |
| vmask, | |
| DISTILL_ALPHA * soft_vec + (1.0 - DISTILL_ALPHA) * hard, | |
| hard, | |
| ) | |
| loss = loss_vec.mean() | |
| else: | |
| loss = hard.mean() | |
| # NaN crash breaker (pipeline.yaml: nan_crash_breaker) | |
| if not torch.isfinite(loss): | |
| nan_streak += 1 | |
| opt.zero_grad(set_to_none=True) | |
| print(f"[watchdog] non-finite loss (streak={nan_streak}) @ step={step}", flush=True) | |
| if nan_streak >= 10: | |
| if rollback_last_ckpt(): | |
| nan_streak = 0 | |
| else: | |
| _try_push_text("FAILURE.txt", "NaN streak; rollback failed\n", | |
| get_hf_token_or_none()) | |
| raise SystemExit("NaN watchdog: rollback failed") | |
| continue | |
| nan_streak = 0 | |
| scaler.scale(loss).backward() | |
| scaler.step(opt) | |
| scaler.update() | |
| step += 1 | |
| tokens += B * T | |
| running_loss += loss.item() | |
| n_loss += 1 | |
| train_seconds += time.time() - blk | |
| now = time.time() | |
| if now - last_ckpt >= CKPT_INT: | |
| save_checkpoint() | |
| last_ckpt = now | |
| if now - last_hb >= HB_INT: | |
| write_heartbeat() | |
| last_hb = now | |
| if MAX_STEPS and step >= MAX_STEPS: | |
| save_checkpoint(final=True) | |
| with open(os.path.join(CKPT_DIR, "DONE"), "w") as f: | |
| f.write("training complete\n") | |
| if token: | |
| try: | |
| push_file(os.path.join(CKPT_DIR, "DONE"), "DONE", token) | |
| except Exception as e: | |
| print(f"[train] DONE push failed: {e}", flush=True) | |
| print(f"[train] DONE @ step={step} tokens={tokens} val_loss={val_probe():.4f}", flush=True) | |
| return | |
| if MAX_MINUTES and (now - t_start) >= MAX_MINUTES * 60: | |
| save_checkpoint(final=False) | |
| print(f"[train] MAX_MINUTES reached ({MAX_MINUTES}); graceful exit WITHOUT DONE " | |
| f"(orchestrator relaunches + resumes @ step={step})", flush=True) | |
| return | |
| if train_seconds >= GPU_HOUR_CAP * 3600: | |
| save_checkpoint(final=False) | |
| print(f"[train] GPU-hour cap {GPU_HOUR_CAP}h reached " | |
| f"({train_seconds/3600:.2f}h trained); stopping WITHOUT DONE", flush=True) | |
| return | |
| if __name__ == "__main__": | |
| try: | |
| main() | |
| except SystemExit: | |
| raise | |
| except Exception as e: | |
| import traceback | |
| traceback.print_exc() | |
| _try_push_text("FAILURE.txt", f"{type(e).__name__}: {e}\n", get_hf_token_or_none()) | |
| print(f"[train] FAILURE: {e}", flush=True) | |
| raise | |