alter_ego / training /kernels /train.py
geraldaton20's picture
initial upload: source code (no build artifacts)
f4f166e verified
Raw History Blame Contribute Delete
30.8 kB
"""Stage 2+ training kernel: auto-checkpoint + auto-resume (self-contained).
Mirrors orchestrator/hf_ckpt + kernels/common. Kaggle script push does NOT bundle
subpackages, so every helper is inlined. Fail-fast on guard failures.
Boot: resume latest valid ckpt from HF (or fresh init). If that ckpt is FINAL,
exit cheaply after re-writing DONE (no weight re-upload).
Loop: AMP-fp16 train -> NaN watchdog -> checkpoint every N min (val probe +
metrics, shards-first/meta-last) -> heartbeat every 5 min -> forced exit at
MAX_MINUTES or GPU-hour cap WITHOUT DONE (orchestrator relaunches) ->
completion at MAX_STEPS writes DONE + final artifacts.
"""
import io
import json
import math
import os
import subprocess
import sys
import time
import numpy as np
# ---------------- config (mirrors config/pipeline.yaml) ----------------
CFG = {
"student": {"vocab": 8192, "d_model": 512, "n_layers": 16, "ffn_ratio": 4,
"n_heads": 8, "max_seq_len": 256, "tied_embeddings": True},
"training": {"lr": 3.0e-3, "warmup_frac": 0.05, "amp": "fp16"},
"hf_ckpt_repo": "geraldaton20/student-ckpts-v1",
"checkpoint_interval_min": 10,
"heartbeat_interval_min": 5,
}
REPO = os.environ.get("HF_CKPT_REPO", CFG["hf_ckpt_repo"])
CKPT_DIR = "ckpt"
MAX_MINUTES = int(os.environ.get("MAX_MINUTES", "240"))
MAX_STEPS = int(os.environ.get("MAX_STEPS", "21362")) or None
QAT_START_FRAC = float(os.environ.get("QAT_START_FRAC", "0.75"))
GPU_HOUR_CAP = float(os.environ.get("GPU_HOUR_CAP", "30")) # yaml hard cap
# ---- Stage-5 real-data mode (ae-corpus-v1 + ae-labels-v1) -------------------
AE_DRY = os.environ.get("AE_DRY", "0") == "1" # force synthetic dry-run
AE_ALLOW_DRY_FALLBACK = os.environ.get("AE_ALLOW_DRY", "0") == "1"
DISTILL_ALPHA = float(os.environ.get("DISTILL_ALPHA", "0.75"))
SOFT_TOPK = int(os.environ.get("SOFT_TOPK", "64"))
CKPT_INT = CFG["checkpoint_interval_min"] * 60
HB_INT = CFG["heartbeat_interval_min"] * 60
# ---------------- bootstrap (inlined) ----------------
def install_deps():
missing = []
for m in ("torch", "transformers", "datasets", "accelerate", "huggingface_hub"):
try:
__import__(m)
except Exception:
missing.append(m)
if missing:
subprocess.run([sys.executable, "-m", "pip", "install", "-q", *missing], check=True)
print("[bootstrap] deps ok", flush=True)
def get_hf_token_or_none():
if HF_TOKEN_INJECTED and HF_TOKEN_INJECTED != "_HF_TOKEN_PLACEHOLDER_":
return HF_TOKEN_INJECTED
tok = os.environ.get("HF_TOKEN") or os.environ.get("HF_TOKEN_ENV")
if tok:
return tok
try:
from kaggle_secrets import UserSecretsClient
return UserSecretsClient().get_secret("HF_TOKEN")
except Exception:
return None
def _try_push_text(fname, text, token):
if not token:
return
try:
from huggingface_hub import HfApi
p = os.path.abspath(fname)
with open(p, "w") as f:
f.write(text)
HfApi(token=token).upload_file(path_or_fileobj=p, path_in_repo=fname,
repo_id=REPO, repo_type="model", token=token)
print(f"[bootstrap] pushed {fname}", flush=True)
except Exception as e:
print(f"[bootstrap] {fname} push failed (non-fatal): {e}", flush=True)
def hardware_guard():
import torch
n = torch.cuda.device_count()
names = [torch.cuda.get_device_name(i) for i in range(n)]
ok = (n == 2) and all("Tesla T4" in nm for nm in names)
if not ok:
msg = f"HARDWARE GUARD FAIL: device_count={n} names={names}\n"
_try_push_text("BOOT_FAILURE.txt", msg, get_hf_token_or_none())
print(f"[bootstrap] {msg}", flush=True)
sys.exit(1)
print(f"[bootstrap] hardware OK: {n} x Tesla T4", flush=True)
# Populated at push time by the orchestrator (private kernel only). The repo
# copy keeps the placeholder; kaggle_client injects the real token.
HF_TOKEN_INJECTED = "_HF_TOKEN_PLACEHOLDER_"
def load_or_init_ckpt(ckpt_dir, repo_id, token):
from huggingface_hub import HfApi
os.makedirs(ckpt_dir, exist_ok=True)
api = HfApi(token=token)
try:
files = list(api.list_repo_files(repo_id=repo_id, repo_type="model"))
except Exception as e:
print(f"[bootstrap] no remote ckpt ({e}); fresh init", flush=True)
return False
state_files = [f for f in files if f == "meta.json" or f.startswith("state_")]
if not state_files:
print("[bootstrap] fresh init (no ckpt on HF)", flush=True)
return False
for f in state_files:
api.hf_hub_download(repo_id=repo_id, filename=f, repo_type="model",
local_dir=ckpt_dir, token=token)
print(f"[bootstrap] resumed {len(state_files)} ckpt files", flush=True)
return True
# ---------------- model builder (inlined) ----------------
def build_model(cfg):
import torch
import torch.nn as nn
d, L, ffn = cfg["d_model"], cfg["n_layers"], cfg["ffn_ratio"] * cfg["d_model"]
h, v, maxlen = cfg["n_heads"], cfg["vocab"], cfg["max_seq_len"]
class Attn(nn.Module):
def __init__(self):
super().__init__()
self.qkv = nn.Linear(d, 3 * d, bias=False)
self.o = nn.Linear(d, d, bias=False)
def forward(self, x):
B, T, _ = x.shape
q, k, vv = self.qkv(x).chunk(3, -1)
q = q.view(B, T, h, d // h).transpose(1, 2)
k = k.view(B, T, h, d // h).transpose(1, 2)
vv = vv.view(B, T, h, d // h).transpose(1, 2)
o = torch.nn.functional.scaled_dot_product_attention(q, k, vv)
return self.o(o.transpose(1, 2).reshape(B, T, d))
class Block(nn.Module):
def __init__(self):
super().__init__()
self.ln1 = nn.LayerNorm(d)
self.attn = Attn()
self.ln2 = nn.LayerNorm(d)
self.mlp = nn.Sequential(nn.Linear(d, ffn), nn.GELU(), nn.Linear(ffn, d))
def forward(self, x):
x = x + self.attn(self.ln1(x))
return x + self.mlp(self.ln2(x))
class SAN(nn.Module):
def __init__(self):
super().__init__()
self.tok_emb = nn.Embedding(v, d)
self.pos = nn.Parameter(torch.zeros(1, maxlen, d))
self.blocks = nn.ModuleList([Block() for _ in range(L)])
self.ln_f = nn.LayerNorm(d)
self.head = nn.Linear(d, v, bias=False)
if cfg.get("tied_embeddings", True):
self.head.weight = self.tok_emb.weight
def forward(self, idx):
B, T = idx.shape
x = self.tok_emb(idx) + self.pos[:, :T]
for b in self.blocks:
x = b(x)
return self.head(self.ln_f(x))
return SAN()
def spec_guard(model, cfg):
"""Fail fast unless the built model matches the PRD/yaml spec (~50M SAN,
embedding share well under the vocab-trap zone)."""
s = cfg["student"]
emb = s["vocab"] * s["d_model"] + s["max_seq_len"] * s["d_model"]
total = sum(p.numel() for p in model.parameters())
frac = emb / total
lines = [f"params_total={total:,}", f"params_emb={emb:,}", f"emb_frac={frac:.3f}"]
print("[spec] " + " | ".join(lines), flush=True)
problems = []
if not (45_000_000 <= total <= 60_000_000):
problems.append(f"param count {total:,} outside 45-60M spec")
if frac > 0.126:
problems.append(f"embedding share {frac:.1%} violates vocab-trap rule (>12.6%)")
if s["d_model"] % s["n_heads"] != 0:
problems.append("d_model not divisible by n_heads")
if problems:
_try_push_text("BOOT_FAILURE.txt", "\n".join(problems), get_hf_token_or_none())
raise SystemExit("SPEC GUARD FAIL: " + "; ".join(problems))
# ---------------- checkpoint lib (inlined: save/load/push) ----------------
SHARD_BYTES = 48 * 1024 * 1024
def _make_json_safe(obj):
if isinstance(obj, bytes):
return {"__bytes__": obj.decode("latin-1")}
if isinstance(obj, tuple):
return [_make_json_safe(x) for x in obj]
if isinstance(obj, list):
return [_make_json_safe(x) for x in obj]
if isinstance(obj, dict):
return {k: _make_json_safe(v) for k, v in obj.items()}
return obj
def _sha256_file(path):
import hashlib
h = hashlib.sha256()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def save_local(ckpt_dir, state, tensor_meta, meta, shard_bytes=SHARD_BYTES):
"""Spanning sharder: any blob (incl. multi-hundred-MB optimizer bundles) is
split across as many shards as needed. assignments[name] = [[shard, off,
len], ...]. Removes stale state_*.bin from prior generations first."""
os.makedirs(ckpt_dir, exist_ok=True)
for f in os.listdir(ckpt_dir):
if f.startswith("state_") and f.endswith(".bin"):
os.remove(os.path.join(ckpt_dir, f))
shards, assignments = [], {}
cur, cur_off = bytearray(), 0
def flush():
nonlocal cur, cur_off
if cur:
shards.append(bytes(cur))
cur, cur_off = bytearray(), 0
for name in sorted(state.keys()):
header = json.dumps({
"name": name,
"shape": tensor_meta[name]["shape"],
"dtype": tensor_meta[name]["dtype"],
"nbytes": len(state[name]),
}).encode()
blob = header + b"\n" + state[name]
segs = []
pos = 0
while pos < len(blob):
if len(cur) >= shard_bytes:
flush()
take = min(shard_bytes - len(cur), len(blob) - pos)
segs.append([len(shards), cur_off, take])
cur += blob[pos:pos + take]
cur_off += take
pos += take
if len(cur) >= shard_bytes:
flush()
assignments[name] = segs
flush()
shard_files, shard_shas = [], []
for i, s in enumerate(shards):
fn = f"state_{i:03d}.bin"
with open(os.path.join(ckpt_dir, fn), "wb") as f:
f.write(s)
shard_files.append(fn)
shard_shas.append(_sha256_file(os.path.join(ckpt_dir, fn)))
out = dict(meta)
out["shards"] = shard_files
out["shard_shas"] = shard_shas
out["assignments"] = assignments
out["tensor_meta"] = tensor_meta
out["created"] = time.time()
tmp = os.path.join(ckpt_dir, "meta.json.tmp")
with open(tmp, "w") as f:
json.dump(_make_json_safe(out), f)
os.replace(tmp, os.path.join(ckpt_dir, "meta.json")) # atomic final write
return out
def load_local(ckpt_dir, verify_sha=True):
meta_path = os.path.join(ckpt_dir, "meta.json")
if not os.path.exists(meta_path):
raise FileNotFoundError(f"no meta.json in {ckpt_dir}")
with open(meta_path) as f:
meta = json.load(f)
if verify_sha:
for fn, exp in zip(meta["shards"], meta["shard_shas"]):
actual = _sha256_file(os.path.join(ckpt_dir, fn))
if actual != exp:
raise ValueError(f"shard {fn} sha mismatch (corrupt/partial)")
shard_data = {fn: open(os.path.join(ckpt_dir, fn), "rb").read()
for fn in meta["shards"]}
state = {}
for name, segs in meta["assignments"].items():
if segs and isinstance(segs[0], int):
i, off = segs # legacy single-segment layout
blob = shard_data[meta["shards"][i]][off:]
nl = blob.index(b"\n")
header = json.loads(blob[:nl].decode())
state[name] = blob[nl + 1: nl + 1 + header["nbytes"]]
else:
buf = b"".join(shard_data[meta["shards"][i]][o:o + n] for i, o, n in segs)
nl = buf.index(b"\n")
header = json.loads(buf[:nl].decode())
state[name] = buf[nl + 1: nl + 1 + header["nbytes"]]
return state, meta
def _upload_with_retry(api, local_path, repo_path, repo_id, token, attempts=4):
import time as _t
last = None
for i in range(attempts):
try:
api.upload_file(path_or_fileobj=local_path, path_in_repo=repo_path,
repo_id=repo_id, repo_type="model", token=token)
return
except Exception as e:
last = e
print(f"[push] {repo_path} attempt {i + 1}/{attempts} failed: {e}", flush=True)
_t.sleep(2 * (i + 1))
raise last
def push_hf(ckpt_dir, repo_id, token=None):
from huggingface_hub import HfApi
api = HfApi(token=token)
files = sorted(os.listdir(ckpt_dir))
shards = [f for f in files if f.startswith("state_") and f.endswith(".bin")]
# CRITICAL: shards + meta.json must stay mutually consistent. Upload that
# pair first (meta LAST within the pair) so a flaky auxiliary upload can
# never strand a stale meta.json while new shards are already live.
for f in shards:
_upload_with_retry(api, os.path.join(ckpt_dir, f), f, repo_id, token)
if "meta.json" in files:
_upload_with_retry(api, os.path.join(ckpt_dir, "meta.json"),
"meta.json", repo_id, token)
others = [f for f in files
if f not in shards and f != "meta.json" and not f.endswith(".tmp")]
for f in others:
try:
_upload_with_retry(api, os.path.join(ckpt_dir, f), f, repo_id, token)
except Exception as e:
print(f"[push] aux {f} upload failed (non-fatal): {e}", flush=True)
def push_file(local_path, repo_path, token):
from huggingface_hub import HfApi
_upload_with_retry(HfApi(token=token), local_path, repo_path, REPO, token)
# ---------------- serialize / deserialize ----------------
def sd_to_state(sd):
state, tm = {}, {}
for k, v in sd.items():
arr = v.detach().cpu().contiguous().numpy()
state[k] = arr.tobytes()
tm[k] = {"shape": list(arr.shape), "dtype": str(arr.dtype)}
return state, tm
def state_to_sd(state, tm):
import torch
sd = {}
for k in state:
if k.startswith("__"):
continue
dt = np.dtype(tm[k]["dtype"])
arr = np.frombuffer(state[k], dt).reshape(tm[k]["shape"])
sd[k] = torch.from_numpy(arr.copy())
return sd
# ---------------- Stage-5 real data (corpus + teacher labels) ----------------
def _kaggle_creds():
try:
from kaggle_secrets import UserSecretsClient
c = UserSecretsClient()
return c.get_secret("KAGGLE_USERNAME"), c.get_secret("KAGGLE_API_TOKEN")
except Exception:
return None, None
def _fetch_dataset(suffix, dst):
user, key = _kaggle_creds()
if not (user and key):
print(f"[data] no Kaggle creds; cannot fetch {suffix}", flush=True)
return False
subprocess.run([sys.executable, "-m", "pip", "install", "-q", "kaggle"], check=True)
env = dict(os.environ, KAGGLE_USERNAME=user, KAGGLE_KEY=key)
r = subprocess.run([sys.executable, "-m", "kaggle", "datasets", "download",
f"{user}/{suffix}", "-p", dst, "--unzip"],
env=env, capture_output=True, text=True)
print(f"[data] {user}/{suffix} rc={r.returncode} "
f"{((r.stdout or '') + (r.stderr or ''))[-160:]}", flush=True)
return r.returncode == 0 and os.path.isdir(dst)
class RealSampler:
"""256-token windows from ae-corpus-v1 (+ optional teacher top-K rows
aligned to stream positions for hybrid distillation loss)."""
def __init__(self, work, dev):
import numpy as np
self.np = np
self.dev = dev
cdir = os.path.join(work, "corpus")
self.tokens = np.memmap(os.path.join(cdir, "tokens.i16"), dtype="<i2", mode="r")
z = np.load(os.path.join(cdir, "arrays.npz"))
lengths = z["lengths"].astype("i8")
self.n_windows = int(self.tokens.shape[0]) // 256
ldir = os.path.join(work, "labels")
li, lp = os.path.join(ldir, "labels_idx.u16"), os.path.join(ldir, "labels_prob.f16")
lm = os.path.join(ldir, "label_meta.npz")
self.has_labels = all(os.path.exists(p) for p in (li, lp, lm))
if self.has_labels:
m = np.load(lm)
doc = m["doc"].astype("i8")
llen = m["len"].astype("i8")
order = np.argsort(doc)
doc, llen = doc[order], llen[order]
starts_all = np.concatenate([[0], np.cumsum(lengths)])[:-1]
self.st = torch.from_numpy(starts_all[doc].copy()).to(dev)
self.en = self.st + torch.from_numpy(llen.copy()).to(dev)
base = np.concatenate([[0], np.cumsum(llen)])[:-1]
self.base = torch.from_numpy(base).to(dev)
self.idx_mm = np.memmap(li, dtype="<u2", mode="r").reshape(-1, SOFT_TOPK)
self.prob_mm = np.memmap(lp, dtype="<f2", mode="r").reshape(-1, SOFT_TOPK)
print(f"[data] labels: docs={len(doc)} tok={int(llen.sum()):,} "
f"topk={SOFT_TOPK}", flush=True)
else:
print("[data] labels absent -> pure hard CE on corpus", flush=True)
def sample(self, B):
np = self.np
widx = np.random.randint(0, max(1, self.n_windows), size=B)
offs = (widx[:, None] * 256 + np.arange(256)[None, :]).astype("i8").ravel()
win = torch.from_numpy(self.tokens[offs].astype("i64")).view(B, 256).to(self.dev)
inp, tgt = win[:, :-1], win[:, 1:]
P = tgt.reshape(-1)
if not self.has_labels:
return inp, tgt, None, None
j = (torch.searchsorted(self.st.contiguous(), P, right=True) - 1).clamp(
min=0, max=self.en.numel() - 1)
valid = (P >= self.st[j]) & (P < self.en[j])
rows = (self.base[j] + (P - self.st[j])).long()
rows = torch.where(valid, rows, torch.full_like(rows, -1))
return inp, tgt, rows, valid
def teacher_rows(self, urows_np):
idx = self.idx_mm[urows_np].astype("i64")
pr = self.prob_mm[urows_np].astype("f4").clip(min=0)
pr /= np.clip(pr.sum(axis=1, keepdims=True), 1e-9, None)
lp = np.log(np.clip(pr, 1e-9, None))
return idx, lp
def resolve_data(work, dev):
"""Returns ('real', sampler) | ('dry', None) — never silent noise."""
global data_mode, sampler
if AE_DRY:
print("[data] AE_DRY=1 -> synthetic dry-run mode", flush=True)
data_mode, sampler = "dry", None
return
ok_c = _fetch_dataset("ae-corpus-v1", os.path.join(work, "corpus_ds")) \
or os.path.exists(os.path.join(work, "corpus_ds", "tokens.i16"))
cdir = os.path.join(work, "corpus_ds")
have_corpus = os.path.exists(os.path.join(cdir, "tokens.i16"))
if not have_corpus:
_fetch_dataset("ae-corpus-v1", cdir)
have_corpus = os.path.exists(os.path.join(cdir, "tokens.i16"))
if not have_corpus:
msg = ("Stage-5 needs ae-corpus-v1 but it is missing. "
"Refusing to train on noise. Finish preprocess first, "
"or set AE_DRY=1 explicitly.")
print(f"[data] FATAL: {msg}", flush=True)
_try_push_text("FAILURE.txt", msg + "\n", get_hf_token_or_none())
raise SystemExit(msg)
_fetch_dataset("ae-labels-v1", os.path.join(work, "labels"))
s = RealSampler(cdir, dev)
data_mode, sampler = "real", s
# ---------------- main ----------------
data_mode, sampler = "dry", None
def main():
install_deps()
hardware_guard()
token = get_hf_token_or_none()
if not token:
print("[train] WARNING: no HF token; checkpoints local-only (no cross-session resume)", flush=True)
import torch
s = CFG["student"]
V = s["vocab"]
model = build_model(s)
spec_guard(model if not hasattr(model, "module") else model.module, CFG)
if torch.cuda.device_count() > 1:
model = torch.nn.DataParallel(model)
model.cuda()
opt = torch.optim.AdamW(model.parameters(), lr=CFG["training"]["lr"])
scaler = torch.cuda.amp.GradScaler() # yaml: amp fp16
def core(mdl):
return mdl.module if hasattr(mdl, "module") else mdl
def load_weights(sd):
core(model).load_state_dict(sd)
step, tokens, qat_enabled = 0, 0, False
train_seconds = 0.0
resumed = load_or_init_ckpt(CKPT_DIR, REPO, token) if token else False
if resumed:
try:
st, meta = load_local(CKPT_DIR)
load_weights(state_to_sd(st, meta["tensor_meta"]))
if "__bundle__" in st:
d = torch.load(io.BytesIO(st["__bundle__"]), weights_only=False)
opt.load_state_dict(d["opt"])
scaler.load_state_dict(d["scaler"])
torch.set_rng_state(d["rng_cpu"].clone())
for i, cst in enumerate(d["rng_cuda"]):
torch.cuda.set_rng_state(cst.cpu(), i)
else:
opt.load_state_dict(torch.load(io.BytesIO(st["__optim__"]), weights_only=False))
step, tokens = meta["step"], meta["tokens"]
qat_enabled = meta.get("qat_enabled", False)
train_seconds = float(meta.get("train_seconds_total", 0.0))
print(f"[train] RESUMED at step={step} tokens={tokens} gpu_hours={train_seconds/3600:.2f}", flush=True)
if meta.get("final"):
# Already complete: refresh DONE cheaply, skip all weight uploads.
with open(os.path.join(CKPT_DIR, "DONE"), "w") as f:
f.write("training complete\n")
if token:
try:
push_file(os.path.join(CKPT_DIR, "DONE"), "DONE", token)
except Exception as e:
print(f"[train] DONE push failed: {e}", flush=True)
print(f"[train] checkpoint already FINAL @ step={step}; nothing to do", flush=True)
return
except Exception as e:
import traceback
traceback.print_exc()
print(f"[train] resume failed ({e}); fresh init", flush=True)
resumed = False
if not resumed:
print("[train] FRESH INIT", flush=True)
B, T = 128, s["max_seq_len"]
resolve_data("/kaggle/tmp/aedata", "cuda")
warmup = max(1, int((MAX_STEPS or 100000) * CFG["training"]["warmup_frac"]))
def lr_now():
if step < warmup:
return CFG["training"]["lr"] * (step + 1) / warmup
prog = (step - warmup) / max(1, (MAX_STEPS or 100000) - warmup)
return CFG["training"]["lr"] * 0.5 * (1 + math.cos(math.pi * min(1.0, prog)))
if data_mode == "real":
val_inp, val_tgt, _, _ = sampler.sample(32)
else:
val_inp = torch.randint(0, V, (B, T), device="cuda")
val_tgt = torch.randint(0, V, (B, T), device="cuda")
@torch.no_grad()
def val_probe():
model.eval()
with torch.autocast(device_type="cuda", dtype=torch.float16):
vl = torch.nn.functional.cross_entropy(
model(val_inp).view(-1, V), val_tgt.view(-1)).item()
model.train()
return vl
t_start = time.time()
last_ckpt = t_start
last_hb = t_start
running_loss, n_loss = 0.0, 0
nan_streak = 0
def write_heartbeat():
with open(os.path.join(CKPT_DIR, "heartbeat.json"), "w") as f:
json.dump({"step": step, "tokens": tokens, "ts": time.time(),
"gpu_hours": round(train_seconds / 3600, 3)}, f)
if token:
try:
push_file(os.path.join(CKPT_DIR, "heartbeat.json"), "heartbeat.json", token)
except Exception as e:
print(f"[train] heartbeat push failed: {e}", flush=True)
def snapshot_state():
sd = core(model).state_dict()
state, tm = sd_to_state(sd)
bundle = io.BytesIO()
torch.save({
"opt": opt.state_dict(),
"scaler": scaler.state_dict(),
"rng_cpu": torch.get_rng_state(),
"rng_cuda": torch.cuda.get_rng_state_all(),
}, bundle)
b = bundle.getvalue()
state["__bundle__"] = b
tm["__bundle__"] = {"shape": [len(b)], "dtype": "bytes"}
return state, tm
def save_checkpoint(final=False):
nonlocal step, tokens, qat_enabled, train_seconds
state, tm = snapshot_state()
vl = val_probe()
meta = {"step": step, "tokens": tokens, "qat_enabled": qat_enabled,
"final": final, "created": time.time(), "data_mode": data_mode,
"distill_alpha": DISTILL_ALPHA,
"train_seconds_total": train_seconds, "val_loss": vl}
save_local(CKPT_DIR, state, tm, meta)
with open(os.path.join(CKPT_DIR, "metrics.jsonl"), "a") as f:
f.write(json.dumps({"step": step, "tokens": tokens,
"loss": round(running_loss / max(1, n_loss), 4),
"val_loss": round(vl, 4),
"gpu_hours": round(train_seconds / 3600, 3),
"ts": time.time()}) + "\n")
if token:
try:
push_hf(CKPT_DIR, REPO, token)
print(f"[train] checkpoint pushed @ step={step} tokens={tokens} "
f"loss={running_loss/max(1,n_loss):.4f} val_loss={vl:.4f}", flush=True)
except Exception as e:
print(f"[train] push failed: {e}", flush=True)
running_loss, n_loss = 0.0, 0 # per-interval window (cleaner telemetry)
def rollback_last_ckpt():
try:
st, meta = load_local(CKPT_DIR)
load_weights(state_to_sd(st, meta["tensor_meta"]))
if "__bundle__" in st:
d = torch.load(io.BytesIO(st["__bundle__"]), weights_only=False)
opt.load_state_dict(d["opt"])
scaler.load_state_dict(d["scaler"])
print("[watchdog] rolled back to last checkpoint", flush=True)
return True
except Exception as e:
print(f"[watchdog] rollback failed: {e}", flush=True)
return False
if not resumed:
save_checkpoint() # first valid ckpt lands on HF promptly
write_heartbeat()
while True:
if MAX_STEPS and step >= QAT_START_FRAC * MAX_STEPS and not qat_enabled:
qat_enabled = True
print(f"[train] QAT phase flagged @ step={step} (fake-quant lands with Stage 5)", flush=True)
blk = time.time()
for _ in range(20):
if data_mode == "real":
inp, tgt, lrows, vmask = sampler.sample(B)
else:
inp = torch.randint(0, V, (B, T), device="cuda")
tgt = torch.randint(0, V, (B, T), device="cuda")
lrows = None
for pg in opt.param_groups:
pg["lr"] = lr_now()
opt.zero_grad(set_to_none=True)
with torch.autocast(device_type="cuda", dtype=torch.float16):
logits = model(inp).view(-1, V)
hard = torch.nn.functional.cross_entropy(
logits.float(), tgt.reshape(-1), reduction="none")
if lrows is not None:
vsel = vmask.nonzero(as_tuple=True)[0]
soft_vec = torch.zeros_like(hard)
if vsel.numel() > 0:
urows, _inv = lrows[vsel].unique(return_inverse=True)
ur_np = urows.cpu().numpy()
t_idx = torch.from_numpy(
sampler.teacher_rows(ur_np)[0]).to(inp.device).long()
t_lp = torch.from_numpy(
sampler.teacher_rows(ur_np)[1]).to(inp.device).float()
sl = logits.index_select(0, vsel).gather(1, t_idx)
q = torch.log_softmax(sl.float(), dim=-1)
kl = (t_lp.exp() * (t_lp - q)).sum(-1)
soft_vec[vsel] = kl
loss_vec = torch.where(
vmask,
DISTILL_ALPHA * soft_vec + (1.0 - DISTILL_ALPHA) * hard,
hard,
)
loss = loss_vec.mean()
else:
loss = hard.mean()
# NaN crash breaker (pipeline.yaml: nan_crash_breaker)
if not torch.isfinite(loss):
nan_streak += 1
opt.zero_grad(set_to_none=True)
print(f"[watchdog] non-finite loss (streak={nan_streak}) @ step={step}", flush=True)
if nan_streak >= 10:
if rollback_last_ckpt():
nan_streak = 0
else:
_try_push_text("FAILURE.txt", "NaN streak; rollback failed\n",
get_hf_token_or_none())
raise SystemExit("NaN watchdog: rollback failed")
continue
nan_streak = 0
scaler.scale(loss).backward()
scaler.step(opt)
scaler.update()
step += 1
tokens += B * T
running_loss += loss.item()
n_loss += 1
train_seconds += time.time() - blk
now = time.time()
if now - last_ckpt >= CKPT_INT:
save_checkpoint()
last_ckpt = now
if now - last_hb >= HB_INT:
write_heartbeat()
last_hb = now
if MAX_STEPS and step >= MAX_STEPS:
save_checkpoint(final=True)
with open(os.path.join(CKPT_DIR, "DONE"), "w") as f:
f.write("training complete\n")
if token:
try:
push_file(os.path.join(CKPT_DIR, "DONE"), "DONE", token)
except Exception as e:
print(f"[train] DONE push failed: {e}", flush=True)
print(f"[train] DONE @ step={step} tokens={tokens} val_loss={val_probe():.4f}", flush=True)
return
if MAX_MINUTES and (now - t_start) >= MAX_MINUTES * 60:
save_checkpoint(final=False)
print(f"[train] MAX_MINUTES reached ({MAX_MINUTES}); graceful exit WITHOUT DONE "
f"(orchestrator relaunches + resumes @ step={step})", flush=True)
return
if train_seconds >= GPU_HOUR_CAP * 3600:
save_checkpoint(final=False)
print(f"[train] GPU-hour cap {GPU_HOUR_CAP}h reached "
f"({train_seconds/3600:.2f}h trained); stopping WITHOUT DONE", flush=True)
return
if __name__ == "__main__":
try:
main()
except SystemExit:
raise
except Exception as e:
import traceback
traceback.print_exc()
_try_push_text("FAILURE.txt", f"{type(e).__name__}: {e}\n", get_hf_token_or_none())
print(f"[train] FAILURE: {e}", flush=True)
raise