"""Stage 2+ training kernel: auto-checkpoint + auto-resume (self-contained). Mirrors orchestrator/hf_ckpt + kernels/common. Kaggle script push does NOT bundle subpackages, so every helper is inlined. Fail-fast on guard failures. Boot: resume latest valid ckpt from HF (or fresh init). If that ckpt is FINAL, exit cheaply after re-writing DONE (no weight re-upload). Loop: AMP-fp16 train -> NaN watchdog -> checkpoint every N min (val probe + metrics, shards-first/meta-last) -> heartbeat every 5 min -> forced exit at MAX_MINUTES or GPU-hour cap WITHOUT DONE (orchestrator relaunches) -> completion at MAX_STEPS writes DONE + final artifacts. """ import io import json import math import os import subprocess import sys import time import numpy as np # ---------------- config (mirrors config/pipeline.yaml) ---------------- CFG = { "student": {"vocab": 8192, "d_model": 512, "n_layers": 16, "ffn_ratio": 4, "n_heads": 8, "max_seq_len": 256, "tied_embeddings": True}, "training": {"lr": 3.0e-3, "warmup_frac": 0.05, "amp": "fp16"}, "hf_ckpt_repo": "geraldaton20/student-ckpts-v1", "checkpoint_interval_min": 10, "heartbeat_interval_min": 5, } REPO = os.environ.get("HF_CKPT_REPO", CFG["hf_ckpt_repo"]) CKPT_DIR = "ckpt" MAX_MINUTES = int(os.environ.get("MAX_MINUTES", "240")) MAX_STEPS = int(os.environ.get("MAX_STEPS", "21362")) or None QAT_START_FRAC = float(os.environ.get("QAT_START_FRAC", "0.75")) GPU_HOUR_CAP = float(os.environ.get("GPU_HOUR_CAP", "30")) # yaml hard cap # ---- Stage-5 real-data mode (ae-corpus-v1 + ae-labels-v1) ------------------- AE_DRY = os.environ.get("AE_DRY", "0") == "1" # force synthetic dry-run AE_ALLOW_DRY_FALLBACK = os.environ.get("AE_ALLOW_DRY", "0") == "1" DISTILL_ALPHA = float(os.environ.get("DISTILL_ALPHA", "0.75")) SOFT_TOPK = int(os.environ.get("SOFT_TOPK", "64")) CKPT_INT = CFG["checkpoint_interval_min"] * 60 HB_INT = CFG["heartbeat_interval_min"] * 60 # ---------------- bootstrap (inlined) ---------------- def install_deps(): missing = [] for m in ("torch", "transformers", "datasets", "accelerate", "huggingface_hub"): try: __import__(m) except Exception: missing.append(m) if missing: subprocess.run([sys.executable, "-m", "pip", "install", "-q", *missing], check=True) print("[bootstrap] deps ok", flush=True) def get_hf_token_or_none(): if HF_TOKEN_INJECTED and HF_TOKEN_INJECTED != "_HF_TOKEN_PLACEHOLDER_": return HF_TOKEN_INJECTED tok = os.environ.get("HF_TOKEN") or os.environ.get("HF_TOKEN_ENV") if tok: return tok try: from kaggle_secrets import UserSecretsClient return UserSecretsClient().get_secret("HF_TOKEN") except Exception: return None def _try_push_text(fname, text, token): if not token: return try: from huggingface_hub import HfApi p = os.path.abspath(fname) with open(p, "w") as f: f.write(text) HfApi(token=token).upload_file(path_or_fileobj=p, path_in_repo=fname, repo_id=REPO, repo_type="model", token=token) print(f"[bootstrap] pushed {fname}", flush=True) except Exception as e: print(f"[bootstrap] {fname} push failed (non-fatal): {e}", flush=True) def hardware_guard(): import torch n = torch.cuda.device_count() names = [torch.cuda.get_device_name(i) for i in range(n)] ok = (n == 2) and all("Tesla T4" in nm for nm in names) if not ok: msg = f"HARDWARE GUARD FAIL: device_count={n} names={names}\n" _try_push_text("BOOT_FAILURE.txt", msg, get_hf_token_or_none()) print(f"[bootstrap] {msg}", flush=True) sys.exit(1) print(f"[bootstrap] hardware OK: {n} x Tesla T4", flush=True) # Populated at push time by the orchestrator (private kernel only). The repo # copy keeps the placeholder; kaggle_client injects the real token. HF_TOKEN_INJECTED = "_HF_TOKEN_PLACEHOLDER_" def load_or_init_ckpt(ckpt_dir, repo_id, token): from huggingface_hub import HfApi os.makedirs(ckpt_dir, exist_ok=True) api = HfApi(token=token) try: files = list(api.list_repo_files(repo_id=repo_id, repo_type="model")) except Exception as e: print(f"[bootstrap] no remote ckpt ({e}); fresh init", flush=True) return False state_files = [f for f in files if f == "meta.json" or f.startswith("state_")] if not state_files: print("[bootstrap] fresh init (no ckpt on HF)", flush=True) return False for f in state_files: api.hf_hub_download(repo_id=repo_id, filename=f, repo_type="model", local_dir=ckpt_dir, token=token) print(f"[bootstrap] resumed {len(state_files)} ckpt files", flush=True) return True # ---------------- model builder (inlined) ---------------- def build_model(cfg): import torch import torch.nn as nn d, L, ffn = cfg["d_model"], cfg["n_layers"], cfg["ffn_ratio"] * cfg["d_model"] h, v, maxlen = cfg["n_heads"], cfg["vocab"], cfg["max_seq_len"] class Attn(nn.Module): def __init__(self): super().__init__() self.qkv = nn.Linear(d, 3 * d, bias=False) self.o = nn.Linear(d, d, bias=False) def forward(self, x): B, T, _ = x.shape q, k, vv = self.qkv(x).chunk(3, -1) q = q.view(B, T, h, d // h).transpose(1, 2) k = k.view(B, T, h, d // h).transpose(1, 2) vv = vv.view(B, T, h, d // h).transpose(1, 2) o = torch.nn.functional.scaled_dot_product_attention(q, k, vv) return self.o(o.transpose(1, 2).reshape(B, T, d)) class Block(nn.Module): def __init__(self): super().__init__() self.ln1 = nn.LayerNorm(d) self.attn = Attn() self.ln2 = nn.LayerNorm(d) self.mlp = nn.Sequential(nn.Linear(d, ffn), nn.GELU(), nn.Linear(ffn, d)) def forward(self, x): x = x + self.attn(self.ln1(x)) return x + self.mlp(self.ln2(x)) class SAN(nn.Module): def __init__(self): super().__init__() self.tok_emb = nn.Embedding(v, d) self.pos = nn.Parameter(torch.zeros(1, maxlen, d)) self.blocks = nn.ModuleList([Block() for _ in range(L)]) self.ln_f = nn.LayerNorm(d) self.head = nn.Linear(d, v, bias=False) if cfg.get("tied_embeddings", True): self.head.weight = self.tok_emb.weight def forward(self, idx): B, T = idx.shape x = self.tok_emb(idx) + self.pos[:, :T] for b in self.blocks: x = b(x) return self.head(self.ln_f(x)) return SAN() def spec_guard(model, cfg): """Fail fast unless the built model matches the PRD/yaml spec (~50M SAN, embedding share well under the vocab-trap zone).""" s = cfg["student"] emb = s["vocab"] * s["d_model"] + s["max_seq_len"] * s["d_model"] total = sum(p.numel() for p in model.parameters()) frac = emb / total lines = [f"params_total={total:,}", f"params_emb={emb:,}", f"emb_frac={frac:.3f}"] print("[spec] " + " | ".join(lines), flush=True) problems = [] if not (45_000_000 <= total <= 60_000_000): problems.append(f"param count {total:,} outside 45-60M spec") if frac > 0.126: problems.append(f"embedding share {frac:.1%} violates vocab-trap rule (>12.6%)") if s["d_model"] % s["n_heads"] != 0: problems.append("d_model not divisible by n_heads") if problems: _try_push_text("BOOT_FAILURE.txt", "\n".join(problems), get_hf_token_or_none()) raise SystemExit("SPEC GUARD FAIL: " + "; ".join(problems)) # ---------------- checkpoint lib (inlined: save/load/push) ---------------- SHARD_BYTES = 48 * 1024 * 1024 def _make_json_safe(obj): if isinstance(obj, bytes): return {"__bytes__": obj.decode("latin-1")} if isinstance(obj, tuple): return [_make_json_safe(x) for x in obj] if isinstance(obj, list): return [_make_json_safe(x) for x in obj] if isinstance(obj, dict): return {k: _make_json_safe(v) for k, v in obj.items()} return obj def _sha256_file(path): import hashlib h = hashlib.sha256() with open(path, "rb") as f: for chunk in iter(lambda: f.read(1 << 20), b""): h.update(chunk) return h.hexdigest() def save_local(ckpt_dir, state, tensor_meta, meta, shard_bytes=SHARD_BYTES): """Spanning sharder: any blob (incl. multi-hundred-MB optimizer bundles) is split across as many shards as needed. assignments[name] = [[shard, off, len], ...]. Removes stale state_*.bin from prior generations first.""" os.makedirs(ckpt_dir, exist_ok=True) for f in os.listdir(ckpt_dir): if f.startswith("state_") and f.endswith(".bin"): os.remove(os.path.join(ckpt_dir, f)) shards, assignments = [], {} cur, cur_off = bytearray(), 0 def flush(): nonlocal cur, cur_off if cur: shards.append(bytes(cur)) cur, cur_off = bytearray(), 0 for name in sorted(state.keys()): header = json.dumps({ "name": name, "shape": tensor_meta[name]["shape"], "dtype": tensor_meta[name]["dtype"], "nbytes": len(state[name]), }).encode() blob = header + b"\n" + state[name] segs = [] pos = 0 while pos < len(blob): if len(cur) >= shard_bytes: flush() take = min(shard_bytes - len(cur), len(blob) - pos) segs.append([len(shards), cur_off, take]) cur += blob[pos:pos + take] cur_off += take pos += take if len(cur) >= shard_bytes: flush() assignments[name] = segs flush() shard_files, shard_shas = [], [] for i, s in enumerate(shards): fn = f"state_{i:03d}.bin" with open(os.path.join(ckpt_dir, fn), "wb") as f: f.write(s) shard_files.append(fn) shard_shas.append(_sha256_file(os.path.join(ckpt_dir, fn))) out = dict(meta) out["shards"] = shard_files out["shard_shas"] = shard_shas out["assignments"] = assignments out["tensor_meta"] = tensor_meta out["created"] = time.time() tmp = os.path.join(ckpt_dir, "meta.json.tmp") with open(tmp, "w") as f: json.dump(_make_json_safe(out), f) os.replace(tmp, os.path.join(ckpt_dir, "meta.json")) # atomic final write return out def load_local(ckpt_dir, verify_sha=True): meta_path = os.path.join(ckpt_dir, "meta.json") if not os.path.exists(meta_path): raise FileNotFoundError(f"no meta.json in {ckpt_dir}") with open(meta_path) as f: meta = json.load(f) if verify_sha: for fn, exp in zip(meta["shards"], meta["shard_shas"]): actual = _sha256_file(os.path.join(ckpt_dir, fn)) if actual != exp: raise ValueError(f"shard {fn} sha mismatch (corrupt/partial)") shard_data = {fn: open(os.path.join(ckpt_dir, fn), "rb").read() for fn in meta["shards"]} state = {} for name, segs in meta["assignments"].items(): if segs and isinstance(segs[0], int): i, off = segs # legacy single-segment layout blob = shard_data[meta["shards"][i]][off:] nl = blob.index(b"\n") header = json.loads(blob[:nl].decode()) state[name] = blob[nl + 1: nl + 1 + header["nbytes"]] else: buf = b"".join(shard_data[meta["shards"][i]][o:o + n] for i, o, n in segs) nl = buf.index(b"\n") header = json.loads(buf[:nl].decode()) state[name] = buf[nl + 1: nl + 1 + header["nbytes"]] return state, meta def _upload_with_retry(api, local_path, repo_path, repo_id, token, attempts=4): import time as _t last = None for i in range(attempts): try: api.upload_file(path_or_fileobj=local_path, path_in_repo=repo_path, repo_id=repo_id, repo_type="model", token=token) return except Exception as e: last = e print(f"[push] {repo_path} attempt {i + 1}/{attempts} failed: {e}", flush=True) _t.sleep(2 * (i + 1)) raise last def push_hf(ckpt_dir, repo_id, token=None): from huggingface_hub import HfApi api = HfApi(token=token) files = sorted(os.listdir(ckpt_dir)) shards = [f for f in files if f.startswith("state_") and f.endswith(".bin")] # CRITICAL: shards + meta.json must stay mutually consistent. Upload that # pair first (meta LAST within the pair) so a flaky auxiliary upload can # never strand a stale meta.json while new shards are already live. for f in shards: _upload_with_retry(api, os.path.join(ckpt_dir, f), f, repo_id, token) if "meta.json" in files: _upload_with_retry(api, os.path.join(ckpt_dir, "meta.json"), "meta.json", repo_id, token) others = [f for f in files if f not in shards and f != "meta.json" and not f.endswith(".tmp")] for f in others: try: _upload_with_retry(api, os.path.join(ckpt_dir, f), f, repo_id, token) except Exception as e: print(f"[push] aux {f} upload failed (non-fatal): {e}", flush=True) def push_file(local_path, repo_path, token): from huggingface_hub import HfApi _upload_with_retry(HfApi(token=token), local_path, repo_path, REPO, token) # ---------------- serialize / deserialize ---------------- def sd_to_state(sd): state, tm = {}, {} for k, v in sd.items(): arr = v.detach().cpu().contiguous().numpy() state[k] = arr.tobytes() tm[k] = {"shape": list(arr.shape), "dtype": str(arr.dtype)} return state, tm def state_to_sd(state, tm): import torch sd = {} for k in state: if k.startswith("__"): continue dt = np.dtype(tm[k]["dtype"]) arr = np.frombuffer(state[k], dt).reshape(tm[k]["shape"]) sd[k] = torch.from_numpy(arr.copy()) return sd # ---------------- Stage-5 real data (corpus + teacher labels) ---------------- def _kaggle_creds(): try: from kaggle_secrets import UserSecretsClient c = UserSecretsClient() return c.get_secret("KAGGLE_USERNAME"), c.get_secret("KAGGLE_API_TOKEN") except Exception: return None, None def _fetch_dataset(suffix, dst): user, key = _kaggle_creds() if not (user and key): print(f"[data] no Kaggle creds; cannot fetch {suffix}", flush=True) return False subprocess.run([sys.executable, "-m", "pip", "install", "-q", "kaggle"], check=True) env = dict(os.environ, KAGGLE_USERNAME=user, KAGGLE_KEY=key) r = subprocess.run([sys.executable, "-m", "kaggle", "datasets", "download", f"{user}/{suffix}", "-p", dst, "--unzip"], env=env, capture_output=True, text=True) print(f"[data] {user}/{suffix} rc={r.returncode} " f"{((r.stdout or '') + (r.stderr or ''))[-160:]}", flush=True) return r.returncode == 0 and os.path.isdir(dst) class RealSampler: """256-token windows from ae-corpus-v1 (+ optional teacher top-K rows aligned to stream positions for hybrid distillation loss).""" def __init__(self, work, dev): import numpy as np self.np = np self.dev = dev cdir = os.path.join(work, "corpus") self.tokens = np.memmap(os.path.join(cdir, "tokens.i16"), dtype=" pure hard CE on corpus", flush=True) def sample(self, B): np = self.np widx = np.random.randint(0, max(1, self.n_windows), size=B) offs = (widx[:, None] * 256 + np.arange(256)[None, :]).astype("i8").ravel() win = torch.from_numpy(self.tokens[offs].astype("i64")).view(B, 256).to(self.dev) inp, tgt = win[:, :-1], win[:, 1:] P = tgt.reshape(-1) if not self.has_labels: return inp, tgt, None, None j = (torch.searchsorted(self.st.contiguous(), P, right=True) - 1).clamp( min=0, max=self.en.numel() - 1) valid = (P >= self.st[j]) & (P < self.en[j]) rows = (self.base[j] + (P - self.st[j])).long() rows = torch.where(valid, rows, torch.full_like(rows, -1)) return inp, tgt, rows, valid def teacher_rows(self, urows_np): idx = self.idx_mm[urows_np].astype("i64") pr = self.prob_mm[urows_np].astype("f4").clip(min=0) pr /= np.clip(pr.sum(axis=1, keepdims=True), 1e-9, None) lp = np.log(np.clip(pr, 1e-9, None)) return idx, lp def resolve_data(work, dev): """Returns ('real', sampler) | ('dry', None) — never silent noise.""" global data_mode, sampler if AE_DRY: print("[data] AE_DRY=1 -> synthetic dry-run mode", flush=True) data_mode, sampler = "dry", None return ok_c = _fetch_dataset("ae-corpus-v1", os.path.join(work, "corpus_ds")) \ or os.path.exists(os.path.join(work, "corpus_ds", "tokens.i16")) cdir = os.path.join(work, "corpus_ds") have_corpus = os.path.exists(os.path.join(cdir, "tokens.i16")) if not have_corpus: _fetch_dataset("ae-corpus-v1", cdir) have_corpus = os.path.exists(os.path.join(cdir, "tokens.i16")) if not have_corpus: msg = ("Stage-5 needs ae-corpus-v1 but it is missing. " "Refusing to train on noise. Finish preprocess first, " "or set AE_DRY=1 explicitly.") print(f"[data] FATAL: {msg}", flush=True) _try_push_text("FAILURE.txt", msg + "\n", get_hf_token_or_none()) raise SystemExit(msg) _fetch_dataset("ae-labels-v1", os.path.join(work, "labels")) s = RealSampler(cdir, dev) data_mode, sampler = "real", s # ---------------- main ---------------- data_mode, sampler = "dry", None def main(): install_deps() hardware_guard() token = get_hf_token_or_none() if not token: print("[train] WARNING: no HF token; checkpoints local-only (no cross-session resume)", flush=True) import torch s = CFG["student"] V = s["vocab"] model = build_model(s) spec_guard(model if not hasattr(model, "module") else model.module, CFG) if torch.cuda.device_count() > 1: model = torch.nn.DataParallel(model) model.cuda() opt = torch.optim.AdamW(model.parameters(), lr=CFG["training"]["lr"]) scaler = torch.cuda.amp.GradScaler() # yaml: amp fp16 def core(mdl): return mdl.module if hasattr(mdl, "module") else mdl def load_weights(sd): core(model).load_state_dict(sd) step, tokens, qat_enabled = 0, 0, False train_seconds = 0.0 resumed = load_or_init_ckpt(CKPT_DIR, REPO, token) if token else False if resumed: try: st, meta = load_local(CKPT_DIR) load_weights(state_to_sd(st, meta["tensor_meta"])) if "__bundle__" in st: d = torch.load(io.BytesIO(st["__bundle__"]), weights_only=False) opt.load_state_dict(d["opt"]) scaler.load_state_dict(d["scaler"]) torch.set_rng_state(d["rng_cpu"].clone()) for i, cst in enumerate(d["rng_cuda"]): torch.cuda.set_rng_state(cst.cpu(), i) else: opt.load_state_dict(torch.load(io.BytesIO(st["__optim__"]), weights_only=False)) step, tokens = meta["step"], meta["tokens"] qat_enabled = meta.get("qat_enabled", False) train_seconds = float(meta.get("train_seconds_total", 0.0)) print(f"[train] RESUMED at step={step} tokens={tokens} gpu_hours={train_seconds/3600:.2f}", flush=True) if meta.get("final"): # Already complete: refresh DONE cheaply, skip all weight uploads. with open(os.path.join(CKPT_DIR, "DONE"), "w") as f: f.write("training complete\n") if token: try: push_file(os.path.join(CKPT_DIR, "DONE"), "DONE", token) except Exception as e: print(f"[train] DONE push failed: {e}", flush=True) print(f"[train] checkpoint already FINAL @ step={step}; nothing to do", flush=True) return except Exception as e: import traceback traceback.print_exc() print(f"[train] resume failed ({e}); fresh init", flush=True) resumed = False if not resumed: print("[train] FRESH INIT", flush=True) B, T = 128, s["max_seq_len"] resolve_data("/kaggle/tmp/aedata", "cuda") warmup = max(1, int((MAX_STEPS or 100000) * CFG["training"]["warmup_frac"])) def lr_now(): if step < warmup: return CFG["training"]["lr"] * (step + 1) / warmup prog = (step - warmup) / max(1, (MAX_STEPS or 100000) - warmup) return CFG["training"]["lr"] * 0.5 * (1 + math.cos(math.pi * min(1.0, prog))) if data_mode == "real": val_inp, val_tgt, _, _ = sampler.sample(32) else: val_inp = torch.randint(0, V, (B, T), device="cuda") val_tgt = torch.randint(0, V, (B, T), device="cuda") @torch.no_grad() def val_probe(): model.eval() with torch.autocast(device_type="cuda", dtype=torch.float16): vl = torch.nn.functional.cross_entropy( model(val_inp).view(-1, V), val_tgt.view(-1)).item() model.train() return vl t_start = time.time() last_ckpt = t_start last_hb = t_start running_loss, n_loss = 0.0, 0 nan_streak = 0 def write_heartbeat(): with open(os.path.join(CKPT_DIR, "heartbeat.json"), "w") as f: json.dump({"step": step, "tokens": tokens, "ts": time.time(), "gpu_hours": round(train_seconds / 3600, 3)}, f) if token: try: push_file(os.path.join(CKPT_DIR, "heartbeat.json"), "heartbeat.json", token) except Exception as e: print(f"[train] heartbeat push failed: {e}", flush=True) def snapshot_state(): sd = core(model).state_dict() state, tm = sd_to_state(sd) bundle = io.BytesIO() torch.save({ "opt": opt.state_dict(), "scaler": scaler.state_dict(), "rng_cpu": torch.get_rng_state(), "rng_cuda": torch.cuda.get_rng_state_all(), }, bundle) b = bundle.getvalue() state["__bundle__"] = b tm["__bundle__"] = {"shape": [len(b)], "dtype": "bytes"} return state, tm def save_checkpoint(final=False): nonlocal step, tokens, qat_enabled, train_seconds state, tm = snapshot_state() vl = val_probe() meta = {"step": step, "tokens": tokens, "qat_enabled": qat_enabled, "final": final, "created": time.time(), "data_mode": data_mode, "distill_alpha": DISTILL_ALPHA, "train_seconds_total": train_seconds, "val_loss": vl} save_local(CKPT_DIR, state, tm, meta) with open(os.path.join(CKPT_DIR, "metrics.jsonl"), "a") as f: f.write(json.dumps({"step": step, "tokens": tokens, "loss": round(running_loss / max(1, n_loss), 4), "val_loss": round(vl, 4), "gpu_hours": round(train_seconds / 3600, 3), "ts": time.time()}) + "\n") if token: try: push_hf(CKPT_DIR, REPO, token) print(f"[train] checkpoint pushed @ step={step} tokens={tokens} " f"loss={running_loss/max(1,n_loss):.4f} val_loss={vl:.4f}", flush=True) except Exception as e: print(f"[train] push failed: {e}", flush=True) running_loss, n_loss = 0.0, 0 # per-interval window (cleaner telemetry) def rollback_last_ckpt(): try: st, meta = load_local(CKPT_DIR) load_weights(state_to_sd(st, meta["tensor_meta"])) if "__bundle__" in st: d = torch.load(io.BytesIO(st["__bundle__"]), weights_only=False) opt.load_state_dict(d["opt"]) scaler.load_state_dict(d["scaler"]) print("[watchdog] rolled back to last checkpoint", flush=True) return True except Exception as e: print(f"[watchdog] rollback failed: {e}", flush=True) return False if not resumed: save_checkpoint() # first valid ckpt lands on HF promptly write_heartbeat() while True: if MAX_STEPS and step >= QAT_START_FRAC * MAX_STEPS and not qat_enabled: qat_enabled = True print(f"[train] QAT phase flagged @ step={step} (fake-quant lands with Stage 5)", flush=True) blk = time.time() for _ in range(20): if data_mode == "real": inp, tgt, lrows, vmask = sampler.sample(B) else: inp = torch.randint(0, V, (B, T), device="cuda") tgt = torch.randint(0, V, (B, T), device="cuda") lrows = None for pg in opt.param_groups: pg["lr"] = lr_now() opt.zero_grad(set_to_none=True) with torch.autocast(device_type="cuda", dtype=torch.float16): logits = model(inp).view(-1, V) hard = torch.nn.functional.cross_entropy( logits.float(), tgt.reshape(-1), reduction="none") if lrows is not None: vsel = vmask.nonzero(as_tuple=True)[0] soft_vec = torch.zeros_like(hard) if vsel.numel() > 0: urows, _inv = lrows[vsel].unique(return_inverse=True) ur_np = urows.cpu().numpy() t_idx = torch.from_numpy( sampler.teacher_rows(ur_np)[0]).to(inp.device).long() t_lp = torch.from_numpy( sampler.teacher_rows(ur_np)[1]).to(inp.device).float() sl = logits.index_select(0, vsel).gather(1, t_idx) q = torch.log_softmax(sl.float(), dim=-1) kl = (t_lp.exp() * (t_lp - q)).sum(-1) soft_vec[vsel] = kl loss_vec = torch.where( vmask, DISTILL_ALPHA * soft_vec + (1.0 - DISTILL_ALPHA) * hard, hard, ) loss = loss_vec.mean() else: loss = hard.mean() # NaN crash breaker (pipeline.yaml: nan_crash_breaker) if not torch.isfinite(loss): nan_streak += 1 opt.zero_grad(set_to_none=True) print(f"[watchdog] non-finite loss (streak={nan_streak}) @ step={step}", flush=True) if nan_streak >= 10: if rollback_last_ckpt(): nan_streak = 0 else: _try_push_text("FAILURE.txt", "NaN streak; rollback failed\n", get_hf_token_or_none()) raise SystemExit("NaN watchdog: rollback failed") continue nan_streak = 0 scaler.scale(loss).backward() scaler.step(opt) scaler.update() step += 1 tokens += B * T running_loss += loss.item() n_loss += 1 train_seconds += time.time() - blk now = time.time() if now - last_ckpt >= CKPT_INT: save_checkpoint() last_ckpt = now if now - last_hb >= HB_INT: write_heartbeat() last_hb = now if MAX_STEPS and step >= MAX_STEPS: save_checkpoint(final=True) with open(os.path.join(CKPT_DIR, "DONE"), "w") as f: f.write("training complete\n") if token: try: push_file(os.path.join(CKPT_DIR, "DONE"), "DONE", token) except Exception as e: print(f"[train] DONE push failed: {e}", flush=True) print(f"[train] DONE @ step={step} tokens={tokens} val_loss={val_probe():.4f}", flush=True) return if MAX_MINUTES and (now - t_start) >= MAX_MINUTES * 60: save_checkpoint(final=False) print(f"[train] MAX_MINUTES reached ({MAX_MINUTES}); graceful exit WITHOUT DONE " f"(orchestrator relaunches + resumes @ step={step})", flush=True) return if train_seconds >= GPU_HOUR_CAP * 3600: save_checkpoint(final=False) print(f"[train] GPU-hour cap {GPU_HOUR_CAP}h reached " f"({train_seconds/3600:.2f}h trained); stopping WITHOUT DONE", flush=True) return if __name__ == "__main__": try: main() except SystemExit: raise except Exception as e: import traceback traceback.print_exc() _try_push_text("FAILURE.txt", f"{type(e).__name__}: {e}\n", get_hf_token_or_none()) print(f"[train] FAILURE: {e}", flush=True) raise