import os os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True" import argparse, json, math, time from datetime import timedelta import numpy as np import sentencepiece as spm import torch import torch.distributed as dist import torch.nn as nn import torch.nn.functional as F from torch.utils.checkpoint import checkpoint as grad_ckpt from torch.utils.data import Dataset, DataLoader torch.backends.cuda.matmul.allow_tf32 = True torch.backends.cudnn.allow_tf32 = True torch._dynamo.config.cache_size_limit = 64 DATA_DIR = "/workspace/mytrans_dl/tokenized_65k" CKPT_DIR = "/workspace/ckpt_300m" VOCAB_SIZE = 65536 UNK_ID, BOS_ID, EOS_ID, PAD_ID = 0, 1, 2, 3 TOK_2MY, TOK_2EN, TOK_FORMAL, TOK_SPOKEN, TOK_BT, TOK_DENOISE, TOK_MASK = 4, 5, 6, 7, 8, 9, 10 FLAG_BT, FLAG_FORMAL, FLAG_SPOKEN = 1, 2, 4 BULK_SUBSETS = { "MyanmarTranslator_works/myanmar_spoken_corpus", "MyanmarTranslator_works/myanmar-cc100-dataset", "MyanmarTranslator_works/myanmar-written-corpus", } def load_and_verify_contracts(data_dir: str): with open(os.path.join(data_dir, "meta.json"), "r") as f: meta = json.load(f) sp = spm.SentencePieceProcessor() sp.load(os.path.join(data_dir, "tokenizer.model")) assert sp.get_piece_size() == VOCAB_SIZE assert sp.unk_id() == UNK_ID and sp.bos_id() == BOS_ID and sp.eos_id() == EOS_ID and sp.pad_id() == PAD_ID expected = {4: "<2my>", 5: "<2en>", 6: "", 7: "", 8: "", 9: "", 10: ""} for tid, piece in expected.items(): assert sp.id_to_piece(tid) == piece, f"Token {tid} mismatch" if "special_tokens" in meta: st = meta["special_tokens"] assert st["2my"] == TOK_2MY and st["2en"] == TOK_2EN assert st["formal"] == TOK_FORMAL and st["spoken"] == TOK_SPOKEN assert st["bt"] == TOK_BT and st["denoise"] == TOK_DENOISE and st["mask"] == TOK_MASK return meta, sp # ============================================================================== # 1. MyTranslation-300M Architecture # ============================================================================== class RMSNorm(nn.Module): def __init__(self, dim: int, eps: float = 1e-6): super().__init__() self.dim = (dim,) self.eps = eps self.weight = nn.Parameter(torch.ones(dim)) def forward(self, x: torch.Tensor) -> torch.Tensor: return F.rms_norm(x, self.dim, self.weight, self.eps) class RotaryEmbedding(nn.Module): def __init__(self, dim: int, max_seq_len: int = 2048, theta: float = 50000.0): super().__init__() inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) t = torch.arange(max_seq_len, dtype=torch.float32) freqs = torch.outer(t, inv_freq) self.register_buffer("cos_cached", freqs.cos()[None, None, :, :], persistent=False) self.register_buffer("sin_cached", freqs.sin()[None, None, :, :], persistent=False) def forward(self, x: torch.Tensor) -> torch.Tensor: L = x.size(2) cos = self.cos_cached[:, :, :L, :].to(dtype=x.dtype) sin = self.sin_cached[:, :, :L, :].to(dtype=x.dtype) x1, x2 = x[..., ::2], x[..., 1::2] return torch.cat((x1 * cos - x2 * sin, x1 * sin + x2 * cos), dim=-1) class SwiGLU(nn.Module): def __init__(self, d_model: int, ffn_dim: int): super().__init__() self.gate_up = nn.Linear(d_model, 2 * ffn_dim, bias=False) self.down = nn.Linear(ffn_dim, d_model, bias=False) def forward(self, x: torch.Tensor) -> torch.Tensor: gate, up = self.gate_up(x).chunk(2, dim=-1) return self.down(F.silu(gate) * up) class GatedGQASelfAttention(nn.Module): def __init__(self, d_model: int, n_heads: int, n_kv_heads: int, is_causal: bool = False, has_v_res: bool = False): super().__init__() self.n_heads = n_heads self.n_kv_heads = n_kv_heads self.head_dim = d_model // n_heads self.is_causal = is_causal self.q_proj = nn.Linear(d_model, n_heads * self.head_dim, bias=False) self.kv_proj = nn.Linear(d_model, 2 * n_kv_heads * self.head_dim, bias=False) self.gate_proj = nn.Linear(d_model, n_heads, bias=False) self.out = nn.Linear(d_model, d_model, bias=False) self.q_norm = RMSNorm(self.head_dim) self.k_norm = RMSNorm(self.head_dim) self.v_res_logit = nn.Parameter(torch.tensor(-2.0)) if has_v_res else None def forward(self, x: torch.Tensor, rope: RotaryEmbedding, attn_mask: torch.Tensor = None, v0: torch.Tensor = None): B, L, D = x.shape q = self.q_norm(self.q_proj(x).view(B, L, self.n_heads, self.head_dim).transpose(1, 2)) k, v = self.kv_proj(x).chunk(2, dim=-1) k = self.k_norm(k.view(B, L, self.n_kv_heads, self.head_dim).transpose(1, 2)) v = v.view(B, L, self.n_kv_heads, self.head_dim).transpose(1, 2) v_raw = v if self.v_res_logit is not None and v0 is not None: alpha = torch.sigmoid(self.v_res_logit).to(dtype=v.dtype) v = (1.0 - alpha) * v + alpha * v0 q, k = rope(q), rope(k) y = F.scaled_dot_product_attention( q, k, v, attn_mask=None if self.is_causal else attn_mask, is_causal=self.is_causal, enable_gqa=True, ) gate = torch.sigmoid(self.gate_proj(x)).transpose(1, 2).unsqueeze(-1) y = (y * gate).transpose(1, 2).contiguous().view(B, L, D) return self.out(y), v_raw def step_kv(self, x_step, cos_p, sin_p, pos, k_cache, v_cache, v0=None): B, _, D = x_step.shape q = self.q_norm(self.q_proj(x_step).view(B, 1, self.n_heads, self.head_dim).transpose(1, 2)) k, v = self.kv_proj(x_step).chunk(2, dim=-1) k = self.k_norm(k.view(B, 1, self.n_kv_heads, self.head_dim).transpose(1, 2)) v = v.view(B, 1, self.n_kv_heads, self.head_dim).transpose(1, 2) v_raw = v if self.v_res_logit is not None and v0 is not None: alpha = torch.sigmoid(self.v_res_logit).to(dtype=v.dtype) v = (1.0 - alpha) * v + alpha * v0 q1, q2 = q[..., ::2], q[..., 1::2] k1, k2 = k[..., ::2], k[..., 1::2] q = torch.cat((q1 * cos_p - q2 * sin_p, q1 * sin_p + q2 * cos_p), dim=-1) k = torch.cat((k1 * cos_p - k2 * sin_p, k1 * sin_p + k2 * cos_p), dim=-1) k_cache[:, :, pos : pos + 1, :] = k v_cache[:, :, pos : pos + 1, :] = v y = F.scaled_dot_product_attention( q, k_cache[:, :, : pos + 1, :], v_cache[:, :, : pos + 1, :], is_causal=False, enable_gqa=True, ) gate = torch.sigmoid(self.gate_proj(x_step)).transpose(1, 2).unsqueeze(-1) y = (y * gate).transpose(1, 2).contiguous().view(B, 1, D) return self.out(y), v_raw class SharedCrossKVProj(nn.Module): def __init__(self, d_model: int, n_kv_heads: int, head_dim: int): super().__init__() self.n_kv_heads = n_kv_heads self.head_dim = head_dim self.norm = RMSNorm(d_model) self.kv_proj = nn.Linear(d_model, 2 * n_kv_heads * head_dim, bias=False) self.k_norm = RMSNorm(head_dim) def forward(self, enc_mix: torch.Tensor): B, S, _ = enc_mix.shape k, v = self.kv_proj(self.norm(enc_mix)).chunk(2, dim=-1) k = self.k_norm(k.view(B, S, self.n_kv_heads, self.head_dim).transpose(1, 2)) v = v.view(B, S, self.n_kv_heads, self.head_dim).transpose(1, 2) return k, v class GatedGQACrossAttention(nn.Module): def __init__(self, d_model: int, n_heads: int): super().__init__() self.n_heads = n_heads self.head_dim = d_model // n_heads self.q_proj = nn.Linear(d_model, n_heads * self.head_dim, bias=False) self.gate_proj = nn.Linear(d_model, n_heads, bias=False) self.out = nn.Linear(d_model, d_model, bias=False) self.q_norm = RMSNorm(self.head_dim) def forward(self, x: torch.Tensor, k_cross: torch.Tensor, v_cross: torch.Tensor, enc_mask: torch.Tensor): B, T, D = x.shape q = self.q_norm(self.q_proj(x).view(B, T, self.n_heads, self.head_dim).transpose(1, 2)) y = F.scaled_dot_product_attention(q, k_cross, v_cross, attn_mask=enc_mask, is_causal=False, enable_gqa=True) gate = torch.sigmoid(self.gate_proj(x)).transpose(1, 2).unsqueeze(-1) y = (y * gate).transpose(1, 2).contiguous().view(B, T, D) return self.out(y) class EncoderLayer(nn.Module): def __init__(self, d_model: int, n_heads: int, n_kv_heads: int, ffn_dim: int, has_v_res: bool): super().__init__() self.norm1 = RMSNorm(d_model) self.self_attn = GatedGQASelfAttention(d_model, n_heads, n_kv_heads, is_causal=False, has_v_res=has_v_res) self.norm2 = RMSNorm(d_model) self.mlp = SwiGLU(d_model, ffn_dim) def forward(self, x, rope, enc_mask, v0=None): attn_out, v_raw = self.self_attn(self.norm1(x), rope, enc_mask, v0=v0) x = x + attn_out x = x + self.mlp(self.norm2(x)) return x, v_raw class DecoderLayer(nn.Module): def __init__(self, d_model: int, n_heads: int, n_kv_heads: int, ffn_dim: int, has_v_res: bool): super().__init__() self.norm1 = RMSNorm(d_model) self.self_attn = GatedGQASelfAttention(d_model, n_heads, n_kv_heads, is_causal=True, has_v_res=has_v_res) self.norm2 = RMSNorm(d_model) self.cross_attn = GatedGQACrossAttention(d_model, n_heads) self.norm3 = RMSNorm(d_model) self.mlp = SwiGLU(d_model, ffn_dim) def forward(self, x, k_cross, v_cross, rope, enc_mask, v0=None): attn_out, v_raw = self.self_attn(self.norm1(x), rope, None, v0=v0) x = x + attn_out x = x + self.cross_attn(self.norm2(x), k_cross, v_cross, enc_mask) x = x + self.mlp(self.norm3(x)) return x, v_raw def step_kv(self, x_step, cos_p, sin_p, pos, k_self, v_self, k_cross, v_cross, enc_mask, v0=None): attn_out, v_raw = self.self_attn.step_kv(self.norm1(x_step), cos_p, sin_p, pos, k_self, v_self, v0=v0) x_step = x_step + attn_out x_step = x_step + self.cross_attn(self.norm2(x_step), k_cross, v_cross, enc_mask) x_step = x_step + self.mlp(self.norm3(x_step)) return x_step, v_raw class MTPHeadBlock(nn.Module): def __init__(self, d_model: int, n_heads: int, n_kv_heads: int, ffn_dim: int): super().__init__() self.norm_h = RMSNorm(d_model) self.norm_e = RMSNorm(d_model) self.proj = nn.Linear(2 * d_model, d_model, bias=False) self.norm1 = RMSNorm(d_model) self.self_attn = GatedGQASelfAttention(d_model, n_heads, n_kv_heads, is_causal=True, has_v_res=False) self.norm2 = RMSNorm(d_model) self.mlp = SwiGLU(d_model, ffn_dim) self.out_norm = RMSNorm(d_model) def forward(self, h_main: torch.Tensor, next_tok_emb: torch.Tensor, rope: RotaryEmbedding): x = self.proj(torch.cat([self.norm_h(h_main), self.norm_e(next_tok_emb)], dim=-1)) attn_out, _ = self.self_attn(self.norm1(x), rope, None) x = x + attn_out x = x + self.mlp(self.norm2(x)) return self.out_norm(x) class MyTranslation300M(nn.Module): def __init__( self, vocab_size=65536, d_model=1024, n_heads=16, n_kv_heads=4, enc_layers=12, dec_layers=6, ffn_dim=3072, max_seq_len=2048, ): super().__init__() assert d_model % n_heads == 0 and n_heads % n_kv_heads == 0 and dec_layers % 2 == 0 self.config = dict( vocab_size=vocab_size, d_model=d_model, n_heads=n_heads, n_kv_heads=n_kv_heads, enc_layers=enc_layers, dec_layers=dec_layers, ffn_dim=ffn_dim, max_seq_len=max_seq_len, ) self.d_model = d_model self.n_heads = n_heads self.n_kv_heads = n_kv_heads self.head_dim = d_model // n_heads self.enc_layers = enc_layers self.dec_layers = dec_layers self.n_cross_pairs = dec_layers // 2 self.embed_scale = math.sqrt(d_model) self.embed_tokens = nn.Embedding(vocab_size, d_model, padding_idx=PAD_ID) self.rope = RotaryEmbedding(self.head_dim, max_seq_len=max_seq_len, theta=50000.0) self.encoder = nn.ModuleList([ EncoderLayer(d_model, n_heads, n_kv_heads, ffn_dim, has_v_res=(i > 0)) for i in range(enc_layers) ]) self.enc_dwa_logits = nn.ParameterList([ nn.Parameter(torch.tensor([-4.0] * i + [0.0], dtype=torch.float32)) for i in range(1, enc_layers) ]) self.enc_router_logits = nn.Parameter(torch.zeros(self.n_cross_pairs, enc_layers)) with torch.no_grad(): for p in range(self.n_cross_pairs): center = (p + 0.5) * (enc_layers / self.n_cross_pairs) - 0.5 for k in range(enc_layers): self.enc_router_logits[p, k] = -0.35 * ((k - center) ** 2) self.cross_kv_projs = nn.ModuleList([ SharedCrossKVProj(d_model, n_kv_heads, self.head_dim) for _ in range(self.n_cross_pairs) ]) self.decoder = nn.ModuleList([ DecoderLayer(d_model, n_heads, n_kv_heads, ffn_dim, has_v_res=(i > 0)) for i in range(dec_layers) ]) self.dec_norm = RMSNorm(d_model) self.lm_head = nn.Linear(d_model, vocab_size, bias=False) self.lm_head.weight = self.embed_tokens.weight self.mtp_head = MTPHeadBlock(d_model, n_heads, n_kv_heads, ffn_dim) self._init_weights(enc_layers + dec_layers) def _init_weights(self, total_layers: int): res_std = 0.02 / math.sqrt(2 * total_layers) for name, m in self.named_modules(): if isinstance(m, nn.Linear): if name.endswith(".out") or name.endswith(".down"): nn.init.normal_(m.weight, mean=0.0, std=res_std) elif name.endswith(".gate_proj"): nn.init.normal_(m.weight, mean=0.0, std=0.01) else: nn.init.normal_(m.weight, mean=0.0, std=0.02) elif isinstance(m, nn.Embedding): nn.init.normal_(m.weight, mean=0.0, std=0.02) if m.padding_idx is not None: m.weight.data[m.padding_idx].zero_() def compile_blocks(self): for i in range(len(self.encoder)): self.encoder[i] = torch.compile(self.encoder[i], dynamic=True) for i in range(len(self.cross_kv_projs)): self.cross_kv_projs[i] = torch.compile(self.cross_kv_projs[i], dynamic=True) for i in range(len(self.decoder)): self.decoder[i] = torch.compile(self.decoder[i], dynamic=True) self.mtp_head = torch.compile(self.mtp_head, dynamic=True) def encode_and_route(self, src_ids: torch.Tensor, ckpt_mode: str = "none"): enc_mask = (src_ids != PAD_ID).unsqueeze(1).unsqueeze(2) x = self.embed_tokens(src_ids) * self.embed_scale raw_states = [x] enc_outputs = [] v0 = None for i, layer in enumerate(self.encoder): if i > 0: w_dwa = F.softmax(self.enc_dwa_logits[i - 1], dim=0).to(dtype=x.dtype) if self.training and ckpt_mode != "none": x_in = grad_ckpt( lambda w, *st: torch.einsum("j,jbsd->bsd", w, torch.stack(st, dim=0)), w_dwa, *raw_states, use_reentrant=False ) else: x_in = torch.einsum("j,jbsd->bsd", w_dwa, torch.stack(raw_states, dim=0)) else: x_in = x do_ckpt = self.training and ( (ckpt_mode == "heavy" and (i % 3 != 0)) or (ckpt_mode == "light" and (i in (3, 7, 11))) ) if do_ckpt: x_out, v_raw = grad_ckpt(layer, x_in, self.rope, enc_mask, v0, use_reentrant=False) else: x_out, v_raw = layer(x_in, self.rope, enc_mask, v0=v0) if i == 0: v0 = v_raw raw_states.append(x_out) enc_outputs.append(x_out) stacked_enc = torch.stack(enc_outputs, dim=0) router_w = F.softmax(self.enc_router_logits, dim=-1).to(dtype=stacked_enc.dtype) cross_kv = [] for p in range(self.n_cross_pairs): enc_mix = torch.einsum("k,kbsd->bsd", router_w[p], stacked_enc) cross_kv.append(self.cross_kv_projs[p](enc_mix)) return cross_kv, enc_mask def decode_seq(self, tgt_in: torch.Tensor, cross_kv: list, enc_mask: torch.Tensor, ckpt_mode: str = "none"): x = self.embed_tokens(tgt_in) * self.embed_scale v0 = None for i, layer in enumerate(self.decoder): kc, vc = cross_kv[i // 2] do_ckpt = self.training and ( (ckpt_mode == "heavy" and (i % 2 == 1)) or (ckpt_mode == "light" and (i == 3)) ) if do_ckpt: x, v_raw = grad_ckpt(layer, x, kc, vc, self.rope, enc_mask, v0, use_reentrant=False) else: x, v_raw = layer(x, kc, vc, self.rope, enc_mask, v0=v0) if i == 0: v0 = v_raw return self.dec_norm(x) def forward_and_backward( self, src_ids: torch.Tensor, tgt_in: torch.Tensor, labels: torch.Tensor, main_denom: float, mtp_denom: float, label_smoothing: float = 0.1, z_loss_weight: float = 1e-4, mtp_weight: float = 0.3, ): tot_tok = src_ids.numel() + tgt_in.numel() max_len = max(src_ids.size(1), tgt_in.size(1)) if tot_tok <= 14336: ckpt_mode = "none" elif max_len <= 256: ckpt_mode = "light" else: ckpt_mode = "heavy" with torch.amp.autocast("cuda", dtype=torch.bfloat16): cross_kv, enc_mask = self.encode_and_route(src_ids, ckpt_mode=ckpt_mode) h_main = self.decode_seq(tgt_in, cross_kv, enc_mask, ckpt_mode=ckpt_mode) valid_mask = labels.ne(-100) valid_y = labels[valid_mask] n_valid = valid_y.numel() valid_h_leaf = h_main[valid_mask].detach().requires_grad_(True) tot_main_ce_t = torch.zeros((), dtype=torch.float64, device=h_main.device) tot_z_loss_t = torch.zeros((), dtype=torch.float64, device=h_main.device) tot_mtp_ce_t = torch.zeros((), dtype=torch.float64, device=h_main.device) chunk_sz = 2048 for i in range(0, n_valid, chunk_sz): hc = valid_h_leaf[i : i + chunk_sz] yc = valid_y[i : i + chunk_sz] with torch.amp.autocast("cuda", dtype=torch.bfloat16): logits_c = self.lm_head(hc) ce_c = F.cross_entropy(logits_c, yc, label_smoothing=label_smoothing, reduction="sum") lse = torch.logsumexp(logits_c.float(), dim=-1) z_c = (lse * lse).sum() loss_c = (ce_c + z_loss_weight * z_c) / main_denom loss_c.backward() tot_main_ce_t.add_(ce_c.detach().to(torch.float64)) tot_z_loss_t.add_(z_c.detach().to(torch.float64)) del logits_c, ce_c, z_c, loss_c grad_h_main = torch.zeros_like(h_main) if valid_h_leaf.grad is not None: grad_h_main[valid_mask] = valid_h_leaf.grad n_mtp = 0 if mtp_weight > 0.0 and tgt_in.size(1) > 1: mtp_mask = labels[:, 1:].ne(-100) & labels[:, :-1].ne(-100) valid_ym = labels[:, 1:][mtp_mask] n_mtp = valid_ym.numel() if n_mtp > 0: h_main_leaf = h_main.detach().requires_grad_(True) with torch.amp.autocast("cuda", dtype=torch.bfloat16): next_emb = self.embed_tokens(tgt_in[:, 1:]) * self.embed_scale h_mtp = self.mtp_head(h_main_leaf[:, :-1, :], next_emb, self.rope) valid_hm_leaf = h_mtp[mtp_mask].detach().requires_grad_(True) for i in range(0, n_mtp, chunk_sz): hmc = valid_hm_leaf[i : i + chunk_sz] ymc = valid_ym[i : i + chunk_sz] with torch.amp.autocast("cuda", dtype=torch.bfloat16): logits_m = self.lm_head(hmc) ce_m = F.cross_entropy(logits_m, ymc, label_smoothing=label_smoothing, reduction="sum") loss_m = (mtp_weight * ce_m) / mtp_denom loss_m.backward() tot_mtp_ce_t.add_(ce_m.detach().to(torch.float64)) del logits_m, ce_m, loss_m grad_h_mtp = torch.zeros_like(h_mtp) if valid_hm_leaf.grad is not None: grad_h_mtp[mtp_mask] = valid_hm_leaf.grad h_mtp.backward(grad_h_mtp) if h_main_leaf.grad is not None: grad_h_main.add_(h_main_leaf.grad) h_main.backward(grad_h_main) ce_v, mtp_v, z_v = torch.stack([tot_main_ce_t, tot_mtp_ce_t, tot_z_loss_t]).tolist() return ce_v, mtp_v, z_v, n_valid, n_mtp @torch.no_grad() def eval_nll(self, src_ids: torch.Tensor, tgt_in: torch.Tensor, labels: torch.Tensor): with torch.amp.autocast("cuda", dtype=torch.bfloat16): cross_kv, enc_mask = self.encode_and_route(src_ids, ckpt_mode="none") h_main = self.decode_seq(tgt_in, cross_kv, enc_mask, ckpt_mode="none") valid_mask = labels.ne(-100) valid_h = h_main[valid_mask] valid_y = labels[valid_mask] n_valid = valid_y.numel() tot_nll = 0.0 for i in range(0, n_valid, 4096): logits_c = self.lm_head(valid_h[i : i + 4096]) tot_nll += float(F.cross_entropy(logits_c, valid_y[i : i + 4096], reduction="sum").item()) return tot_nll, n_valid @torch.inference_mode() def translate_kv(self, prompt_ids_list: list[list[int]], sp_proc, max_new_tokens: int = 160, rep_penalty: float = 1.12) -> list[str]: assert len(prompt_ids_list) > 0 and all(len(s) > 0 for s in prompt_ids_list), "Empty prompt batch!" max_rope = self.rope.cos_cached.size(2) assert 0 < max_new_tokens <= max_rope, f"max_new_tokens={max_new_tokens} exceeds RoPE={max_rope}" self.eval() dev = self.embed_tokens.weight.device B = len(prompt_ids_list) trimmed = [s[:max_rope] for s in prompt_ids_list] max_s = min(max_rope, ((max(len(s) for s in trimmed) + 7) // 8) * 8) src_ids = torch.full((B, max_s), PAD_ID, dtype=torch.long, device=dev) for i, ids in enumerate(trimmed): src_ids[i, : len(ids)] = torch.tensor(ids, dtype=torch.long, device=dev) with torch.amp.autocast("cuda", dtype=torch.bfloat16): cross_kv, enc_mask = self.encode_and_route(src_ids, ckpt_mode="none") k_cache = torch.empty((self.dec_layers, B, self.n_kv_heads, max_new_tokens, self.head_dim), dtype=torch.bfloat16, device=dev) v_cache = torch.empty((self.dec_layers, B, self.n_kv_heads, max_new_tokens, self.head_dim), dtype=torch.bfloat16, device=dev) cur_tok = torch.full((B, 1), BOS_ID, dtype=torch.long, device=dev) gen_ids = torch.full((B, max_new_tokens), EOS_ID, dtype=torch.long, device=dev) done = torch.zeros(B, dtype=torch.bool, device=dev) cos_all, sin_all = self.rope.cos_cached.to(torch.bfloat16), self.rope.sin_cached.to(torch.bfloat16) for pos in range(max_new_tokens): cos_p, sin_p = cos_all[:, :, pos : pos + 1, :], sin_all[:, :, pos : pos + 1, :] x_dec = self.embed_tokens(cur_tok) * self.embed_scale v0 = None for l_idx, layer in enumerate(self.decoder): kc, vc = cross_kv[l_idx // 2] x_dec, v_raw = layer.step_kv(x_dec, cos_p, sin_p, pos, k_cache[l_idx], v_cache[l_idx], kc, vc, enc_mask, v0=v0) if l_idx == 0: v0 = v_raw logits = self.lm_head(self.dec_norm(x_dec)[:, 0, :]).float() logits[:, [0, 1, 3, 4, 5, 6, 7, 8, 9, 10]] = -1e9 if rep_penalty != 1.0 and pos > 0: prev = gen_ids[:, :pos] s_l = torch.gather(logits, 1, prev) s_l = torch.where(s_l > 0, s_l / rep_penalty, s_l * rep_penalty) logits.scatter_(1, prev, s_l) nxt = torch.where(done, torch.tensor(EOS_ID, device=dev), logits.argmax(dim=-1)) gen_ids[:, pos] = nxt done = done | (nxt == EOS_ID) cur_tok = nxt.unsqueeze(1) if bool(done.all()): break out_texts = [] for row in gen_ids.cpu().tolist(): if EOS_ID in row: row = row[: row.index(EOS_ID)] out_texts.append(sp_proc.decode_ids([x for x in row if x >= 11]).strip()) return out_texts # ============================================================================== # 2. 100%-Coverage Plan + Tight-Padding Dynamic Packing + Distributed Sharding # ============================================================================== class CanonicalMemmapStore: def __init__(self, data_dir: str, split: str): self.my_path = os.path.join(data_dir, f"{split}_my.bin") self.en_path = os.path.join(data_dir, f"{split}_en.bin") self.index = np.load(os.path.join(data_dir, f"{split}_index.npy"), mmap_mode="r") self.my_mm = None self.en_mm = None def ensure_open(self): if self.my_mm is None: self.my_mm = np.memmap(self.my_path, dtype=np.uint16, mode="r") self.en_mm = np.memmap(self.en_path, dtype=np.uint16, mode="r") def get_pair(self, idx: int): self.ensure_open() rec = self.index[idx] m = self.my_mm[int(rec["my_off"]) : int(rec["my_off"]) + int(rec["my_len"])].astype(np.int64) e = self.en_mm[int(rec["en_off"]) : int(rec["en_off"]) + int(rec["en_len"])].astype(np.int64) return m, e, int(rec["flags"]) def build_or_load_epoch_plan(train_index: np.ndarray, meta: dict, stage: str, plan_path: str, is_main: bool, world_size: int, seed: int = 2026) -> np.ndarray: if is_main and not os.path.exists(plan_path): rng = np.random.RandomState(seed) n_subsets = len(meta["subset_to_id"]) sids = np.asarray(train_index["subset_id"]) assert int(sids.max()) < n_subsets and int(sids.min()) >= 0 if stage == "stable": base_rows = rng.permutation(len(train_index)).astype(np.uint32) replay_n = 12_000_000 eligible_mask = np.ones(len(train_index), dtype=bool) for k, name in meta["id_to_subset"].items(): if name in BULK_SUBSETS: eligible_mask[sids == int(k)] = False else: eligible_mask = (np.asarray(train_index["flags"]) & FLAG_BT) == 0 base_rows = np.flatnonzero(eligible_mask).astype(np.uint32) rng.shuffle(base_rows) replay_n = 1_500_000 elig_pos = np.flatnonzero(eligible_mask).astype(np.uint32) elig_sids = sids[elig_pos] order = elig_pos[np.argsort(elig_sids, kind="mergesort")] sorted_sids = sids[order] counts = np.bincount(sorted_sids, minlength=n_subsets) starts = np.zeros(n_subsets, dtype=np.int64) starts[1:] = np.cumsum(counts[:-1]) weights = np.zeros(n_subsets, dtype=np.float64) for k, name in meta["id_to_subset"].items(): sid = int(k) if counts[sid] == 0: continue if stage == "stable" and name in BULK_SUBSETS: continue if stage == "cooldown" and name == "non_myanmartranslator/finetranslations": w = float(counts[sid]) ** 0.35 * 0.25 else: w = float(counts[sid]) ** 0.50 if stage == "stable" and name.startswith("non_myanmartranslator/"): w *= 4.0 weights[sid] = w assert weights.sum() > 0, "Empty replay weights!" probs = weights / weights.sum() chosen_sids = rng.choice(n_subsets, size=replay_n, p=probs) replay_rows = np.empty(replay_n, dtype=np.uint32) for sid in range(n_subsets): m = (chosen_sids == sid) npick = int(m.sum()) if npick > 0: replay_rows[m] = order[starts[sid] + rng.randint(0, counts[sid], size=npick)] full_plan = np.concatenate([base_rows, replay_rows]) rng.shuffle(full_plan) tmp_plan = plan_path + ".tmp.npy" np.save(tmp_plan, full_plan) os.replace(tmp_plan, plan_path) print( f"📋 Built {stage} 100%-coverage plan: {len(base_rows):,} unique base rows (100% coverage) " f"+ {len(replay_rows):,} gold replay = {len(full_plan):,} total rows" ) if world_size > 1: dist.barrier() plan = np.load(plan_path, mmap_mode="r") if is_main: print(f"📋 Ready {stage} plan: {len(plan):,} rows") return plan def pack_chunk_batches( row_slice: np.ndarray, train_index: np.ndarray, max_tokens_combined: int, max_seq_len: int, step_group: int, seed: int, ): rng = np.random.RandomState(seed) my_lens = np.minimum(max_seq_len, train_index["my_len"][row_slice].astype(np.int32)) en_lens = np.minimum(max_seq_len, train_index["en_len"][row_slice].astype(np.int32)) max_side = np.maximum(my_lens, en_lens) comb_lens = my_lens + en_lens mega = 65536 batches_cost = [] for i in range(0, len(row_slice), mega): sub_r = row_slice[i : i + mega] sub_ms = max_side[i : i + mega] sub_cl = comb_lens[i : i + mega] sub_r = sub_r[np.lexsort((sub_cl, sub_ms))] cur_b, cur_s_max, cur_t_max = [], 0, 0 for r in sub_r: ri = int(r) ml = min(max_seq_len, int(train_index["my_len"][ri])) el = min(max_seq_len, int(train_index["en_len"][ri])) s_est = max(el, ml) + 4 t_est = max(ml, el) + 2 new_s = max(cur_s_max, s_est) new_t = max(cur_t_max, t_est) if cur_b and (len(cur_b) >= 256 or (len(cur_b) + 1) * (new_s + new_t) > max_tokens_combined): batches_cost.append(((max(cur_s_max, cur_t_max), len(cur_b) * (cur_s_max + cur_t_max)), cur_b)) cur_b, cur_s_max, cur_t_max = [ri], s_est, t_est else: cur_b.append(ri) cur_s_max, cur_t_max = new_s, new_t if cur_b: batches_cost.append(((max(cur_s_max, cur_t_max), len(cur_b) * (cur_s_max + cur_t_max)), cur_b)) # Load-balanced step grouping by (max_seq, tok_cost) + safe small-chunk pad batches_cost.sort(key=lambda x: x[0]) batches = [b for _, b in batches_cost] rem = len(batches) % step_group if rem != 0 and len(batches) > 0: need = step_group - rem batches.extend((batches * ((need // len(batches)) + 1))[:need]) groups = [batches[j : j + step_group] for j in range(0, len(batches), step_group)] rng.shuffle(groups) return [b for g in groups for b in g] class DynamicStreamDataset(Dataset): def __init__(self, store: CanonicalMemmapStore, batches: list[list[int]], stage: str, max_seq_len: int, seed: int): self.store = store self.batches = batches self.stage = stage self.max_seq_len = max_seq_len self.base_seed = seed self.rng = np.random.RandomState(seed) def __len__(self): return len(self.batches) def _corrupt_span(self, toks: np.ndarray): n = len(toks) if n < 4: return toks out = [] i = 0 masked_any = False while i < n: if self.rng.rand() < 0.15: span_len = min(n - i, max(1, int(self.rng.poisson(3.0)))) out.append(TOK_MASK) i += span_len masked_any = True else: out.append(int(toks[i])) i += 1 if not masked_any: idx = int(self.rng.randint(0, n)) out = list(toks[:idx]) + [TOK_MASK] + list(toks[idx + min(2, n - idx):]) return np.asarray(out, dtype=np.int64) def __getitem__(self, b_idx: int): row_ids = self.batches[b_idx] srcs, tgts = [], [] for r_id in row_ids: m, e, flg = self.store.get_pair(r_id) m = m[: self.max_seq_len - 4] e = e[: self.max_seq_len - 4] is_bt = bool(flg & FLAG_BT) is_formal = bool(flg & FLAG_FORMAL) is_spoken = bool(flg & FLAG_SPOKEN) if self.stage == "cooldown": mode = "en2my" if self.rng.rand() < 0.50 else "my2en" else: p = self.rng.rand() if is_bt: mode = "en2my" if p < 0.72 else ("my2en_bt" if p < 0.90 else "denoise") else: mode = "en2my" if p < 0.47 else ("my2en" if p < 0.94 else "denoise") style_tok = None if is_formal and self.rng.rand() < 0.70: style_tok = TOK_FORMAL elif is_spoken and self.rng.rand() < 0.70: style_tok = TOK_SPOKEN if mode == "en2my": pref = [TOK_2MY] + ([style_tok] if style_tok is not None else []) src_arr = np.concatenate([np.asarray(pref, dtype=np.int64), e, [EOS_ID]]) tgt_arr = np.concatenate([[BOS_ID], m, [EOS_ID]]) elif mode in ("my2en", "my2en_bt"): pref = [TOK_2EN] + ([TOK_BT] if mode == "my2en_bt" else []) + ([style_tok] if style_tok is not None else []) src_arr = np.concatenate([np.asarray(pref, dtype=np.int64), m, [EOS_ID]]) tgt_arr = np.concatenate([[BOS_ID], e, [EOS_ID]]) else: base = m if (is_bt or self.rng.rand() < 0.5) else e noisy = self._corrupt_span(base) src_arr = np.concatenate([[TOK_DENOISE], noisy, [EOS_ID]]) tgt_arr = np.concatenate([[BOS_ID], base, [EOS_ID]]) srcs.append(src_arr) tgts.append(tgt_arr) return srcs, tgts def worker_init_fn(worker_id: int): info = torch.utils.data.get_worker_info() if info is not None and hasattr(info.dataset, "rng"): info.dataset.rng = np.random.RandomState((info.dataset.base_seed + worker_id * 9973) % (2**31 - 1)) def collate_prebatched(batch_list): srcs, tgts = batch_list[0] B = len(srcs) max_s = min(2048, ((max(len(x) for x in srcs) + 31) // 32) * 32) max_t = min(2048, ((max(len(x) - 1 for x in tgts) + 31) // 32) * 32) src_ids = torch.full((B, max_s), PAD_ID, dtype=torch.long) tgt_in = torch.full((B, max_t), PAD_ID, dtype=torch.long) labels = torch.full((B, max_t), -100, dtype=torch.long) for i, (s, t) in enumerate(zip(srcs, tgts)): sl = min(len(s), max_s) tl = min(len(t) - 1, max_t) src_ids[i, :sl] = torch.from_numpy(s[:sl]) tgt_in[i, :tl] = torch.from_numpy(t[:tl]) labels[i, :tl] = torch.from_numpy(t[1 : tl + 1]) return src_ids, tgt_in, labels def clean_state_dict(model: nn.Module): return {k.replace("._orig_mod.", "."): v.detach().cpu() for k, v in model.state_dict().items()} def broadcast_model(model: nn.Module, world_size: int): if world_size == 1: return for p in model.parameters(): dist.broadcast(p.data, src=0) for b in model.buffers(): dist.broadcast(b.data, src=0) def allreduce_grads(model: nn.Module, world_size: int, grad_scale: float = 1024.0): if world_size == 1: return grads = [] for p in model.parameters(): if p.requires_grad: if p.grad is None: p.grad = torch.zeros_like(p.data) grads.append(p.grad) b_grads = [g.bfloat16() for g in grads] flat = torch._utils._flatten_dense_tensors(b_grads) dist.all_reduce(flat, op=dist.ReduceOp.SUM) inv_scale = 1.0 / grad_scale for orig, synced in zip(grads, torch._utils._unflatten_dense_tensors(flat, b_grads)): orig.copy_(synced.float() * inv_scale) # ============================================================================== # 3. Main Multi-GPU Training Loop # ============================================================================== def main(): parser = argparse.ArgumentParser() parser.add_argument("--stage", choices=["stable", "cooldown"], default="stable") parser.add_argument("--verify-steps", type=int, default=0) parser.add_argument("--grad-accum", type=int, default=1) parser.add_argument("--max-tok-micro", type=int, default=32768) parser.add_argument("--max-seq-len", type=int, default=1024) parser.add_argument("--chunk-rows", type=int, default=4000000) parser.add_argument("--peak-lr", type=float, default=5.0e-4) parser.add_argument("--warmup-steps", type=int, default=900) parser.add_argument("--eval-every", type=int, default=500) parser.add_argument("--save-every", type=int, default=500) parser.add_argument("--milestone-every", type=int, default=5000) parser.add_argument("--compile", action="store_true") parser.add_argument("--fresh", action="store_true") parser.add_argument("--init-weights", type=str, default="") args = parser.parse_args() world_size = int(os.environ.get("WORLD_SIZE", 1)) rank = int(os.environ.get("RANK", 0)) local_rank = int(os.environ.get("LOCAL_RANK", 0)) is_main = (rank == 0) if world_size > 1: torch.cuda.set_device(local_rank) dist.init_process_group(backend="nccl", timeout=timedelta(minutes=30), device_id=torch.device(f"cuda:{local_rank}")) device = torch.device("cuda", local_rank) else: device = torch.device("cuda" if torch.cuda.is_available() else "cpu") if is_main: os.makedirs(CKPT_DIR, exist_ok=True) if world_size > 1: dist.barrier() meta, sp = load_and_verify_contracts(DATA_DIR) train_store = CanonicalMemmapStore(DATA_DIR, "train") val_store = CanonicalMemmapStore(DATA_DIR, "val") all_val_non_bt = np.flatnonzero((val_store.index["flags"] & FLAG_BT) == 0) assert len(all_val_non_bt) > 0, "Validation set has zero non-BT rows!" val_rows = all_val_non_bt[::max(1, len(all_val_non_bt) // 1024)][:1024] val_batches = [val_rows[i : i + 64].tolist() for i in range(0, len(val_rows), 64)] plan_path = os.path.join(CKPT_DIR, f"{args.stage}_plan.npy") latest_path_early = os.path.join(CKPT_DIR, "latest.pt") if args.fresh and is_main: if os.path.exists(plan_path): os.remove(plan_path) if os.path.exists(latest_path_early): os.replace(latest_path_early, latest_path_early + ".bak") if world_size > 1: dist.barrier() full_plan = build_or_load_epoch_plan(train_store.index, meta, args.stage, plan_path, is_main, world_size, seed=2026) num_chunks = (len(full_plan) + args.chunk_rows - 1) // args.chunk_rows model = MyTranslation300M().to(device) decay_p = [p for n, p in model.named_parameters() if p.requires_grad and p.ndim >= 2] nodecay_p = [p for n, p in model.named_parameters() if p.requires_grad and p.ndim < 2] opt = torch.optim.AdamW( [{"params": decay_p, "weight_decay": 0.01}, {"params": nodecay_p, "weight_decay": 0.0}], lr=args.peak_lr, betas=(0.9, 0.95), eps=1e-8, fused=True, ) step = 0 chunk_idx = 0 batch_cursor = 0 rows_seen = 0 best_val_nll = float("inf") latest_path = os.path.join(CKPT_DIR, "latest.pt") if args.init_weights: if is_main: print(f"📥 Loading weights from {args.init_weights}") model.load_state_dict(torch.load(args.init_weights, map_location="cpu"), strict=True) elif (not args.fresh) and os.path.exists(latest_path): if is_main: print(f"🔄 Resuming from {latest_path}") ckpt = torch.load(latest_path, map_location="cpu") model.load_state_dict(ckpt["model"], strict=True) if args.stage == ckpt.get("stage", "stable"): opt.load_state_dict(ckpt["opt"]) step = int(ckpt["step"]) chunk_idx = int(ckpt.get("chunk_idx", 0)) batch_cursor = int(ckpt.get("batch_cursor", 0)) rows_seen = int(ckpt.get("rows_seen", 0)) best_val_nll = float(ckpt.get("best_val_nll", float("inf"))) elif is_main: print(f"➡️ Stage transition {ckpt.get('stage')} -> {args.stage}") broadcast_model(model, world_size) if args.compile: if is_main: print("⚡ Compiling encoder/cross_kv/decoder/MTP blocks with torch.compile(dynamic=True)...") model.compile_blocks() if is_main: tot_params = sum(p.numel() for p in model.parameters()) inf_params = tot_params - sum(p.numel() for p in model.mtp_head.parameters()) print(f"🏛️ MyTranslation-300M | GPUs: {world_size}x | Stage: {args.stage} | Inf: {inf_params/1e6:.2f}M | Train: {tot_params/1e6:.2f}M") label_smoothing = 0.1 if args.stage == "stable" else 0.0 mtp_weight = 0.3 if args.stage == "stable" else 0.15 STYLE_EVAL_PROMPTS = [ ("En->My Formal", [TOK_2MY, TOK_FORMAL], "The central bank raised interest rates to control inflation and stabilize the economy."), ("En->My Spoken", [TOK_2MY, TOK_SPOKEN], "Honestly, I didn't expect things to turn out like this, but don't worry—we'll figure it out together."), ("My->En Formal", [TOK_2EN, TOK_FORMAL], "ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးတည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။"), ("My->En Spoken", [TOK_2EN, TOK_SPOKEN], "ဒီနေ့ ရန်ကုန်မှာ မိုးတွေ အရမ်းရွာနေလို့ အိမ်ပြန်ရတာ နည်းနည်းနောက်ကျမယ်နော်။"), ] @torch.no_grad() def eval_val_nll(): model.eval() en2my_nll, en2my_tok = 0.0, 0 my2en_nll, my2en_tok = 0.0, 0 for b in val_batches: for direction in ("en2my", "my2en"): srcs, tgts = [], [] for r_id in b: m, e, _ = val_store.get_pair(int(r_id)) m, e = m[: args.max_seq_len - 4], e[: args.max_seq_len - 4] if direction == "en2my": srcs.append(np.concatenate([[TOK_2MY], e, [EOS_ID]])) tgts.append(np.concatenate([[BOS_ID], m, [EOS_ID]])) else: srcs.append(np.concatenate([[TOK_2EN], m, [EOS_ID]])) tgts.append(np.concatenate([[BOS_ID], e, [EOS_ID]])) src_ids, tgt_in, labels = collate_prebatched([(srcs, tgts)]) nll, nv = model.eval_nll(src_ids.to(device), tgt_in.to(device), labels.to(device)) if direction == "en2my": en2my_nll += nll en2my_tok += nv else: my2en_nll += nll my2en_tok += nv e2m = en2my_nll / max(1, en2my_tok) m2e = my2en_nll / max(1, my2en_tok) comb = (en2my_nll + my2en_nll) / max(1, en2my_tok + my2en_tok) return comb, e2m, m2e def run_style_generations(step_num: int): prompts = [pref + sp.encode_as_ids(txt)[:500] + [EOS_ID] for _, pref, txt in STYLE_EVAL_PROMPTS] preds = model.translate_kv(prompts, sp, max_new_tokens=96) print(f" 🗣️ Style Generations @ Step {step_num}:") for (tag, _, _), out in zip(STYLE_EVAL_PROMPTS, preds): print(f" [{tag}]: {out[:180]}") def make_chunk_loader(c_idx: int, skip_steps: int = 0): s_off = c_idx * args.chunk_rows e_off = min(len(full_plan), s_off + args.chunk_rows) row_slice = np.asarray(full_plan[s_off:e_off]) step_group = world_size * args.grad_accum all_batches = pack_chunk_batches( row_slice, train_store.index, max_tokens_combined=args.max_tok_micro, max_seq_len=args.max_seq_len, step_group=step_group, seed=2026 + c_idx * 17, ) rank_batches = all_batches[rank::world_size] if skip_steps > 0: rank_batches = rank_batches[skip_steps * args.grad_accum :] n_windows = len(rank_batches) // args.grad_accum rank_batches = rank_batches[: n_windows * args.grad_accum] ds = DynamicStreamDataset( train_store, rank_batches, args.stage, args.max_seq_len, seed=10000 + c_idx * 997 + rank * 131 + skip_steps, ) loader = DataLoader( ds, batch_size=1, shuffle=False, collate_fn=collate_prebatched, num_workers=4, pin_memory=True, worker_init_fn=worker_init_fn, persistent_workers=False, ) return loader, n_windows if is_main: comb, e2m, m2e = eval_val_nll() print(f"🔎 Step {step:6d} | Val Comb: {comb:.4f} | En->My: {e2m:.4f} | My->En: {m2e:.4f}") if world_size > 1: dist.barrier() model.train() opt.zero_grad(set_to_none=True) acc_ce = acc_mtp = acc_z = 0.0 acc_valid_tok = acc_mtp_tok = acc_tot_tok = 0 t0 = time.time() while chunk_idx < num_chunks: train_loader, n_windows = make_chunk_loader(chunk_idx, skip_steps=batch_cursor) train_iter = iter(train_loader) done_flag = False for _ in range(n_windows): window = [next(train_iter) for _ in range(args.grad_accum)] batch_cursor += 1 loc_valid = sum(int((lb != -100).sum().item()) for _, _, lb in window) loc_mtp = sum( int(((lb[:, 1:] != -100) & (lb[:, :-1] != -100)).sum().item()) if lb.size(1) > 1 else 0 for _, _, lb in window ) loc_rows = sum(s.size(0) for s, _, _ in window) if world_size > 1: counts_t = torch.tensor([loc_valid, loc_mtp, loc_rows], dtype=torch.long, device=device) dist.all_reduce(counts_t, op=dist.ReduceOp.SUM) glob_valid, glob_mtp, glob_rows = int(counts_t[0].item()), int(counts_t[1].item()), int(counts_t[2].item()) else: glob_valid, glob_mtp, glob_rows = loc_valid, loc_mtp, loc_rows g_scale = 1024.0 if world_size > 1 else 1.0 main_denom = float(max(1, glob_valid)) / g_scale mtp_denom = float(max(1, glob_mtp)) / g_scale loc_ce = loc_mtp_ce = loc_z = 0.0 loc_tok = sum(int((s != PAD_ID).sum().item()) for s, _, _ in window) for src_ids, tgt_in, labels in window: src_ids = src_ids.to(device, non_blocking=True) tgt_in = tgt_in.to(device, non_blocking=True) labels = labels.to(device, non_blocking=True) ce_s, mtp_s, z_s, nv, nm = model.forward_and_backward( src_ids, tgt_in, labels, main_denom=main_denom, mtp_denom=mtp_denom, label_smoothing=label_smoothing, mtp_weight=mtp_weight, ) loc_ce += ce_s loc_mtp_ce += mtp_s loc_z += z_s loc_tok += nv allreduce_grads(model, world_size) rows_seen += glob_rows if args.stage == "stable": lr = args.peak_lr * min(1.0, (step + 1) / max(1, args.warmup_steps)) else: if step < 100: lr = args.peak_lr * (step + 1) / 100.0 else: progress = min(1.0, rows_seen / max(1, len(full_plan))) lr = 1.0e-5 + 0.5 * (args.peak_lr - 1.0e-5) * (1.0 + math.cos(math.pi * progress)) for pg in opt.param_groups: pg["lr"] = lr grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) opt.step() opt.zero_grad(set_to_none=True) step += 1 if world_size > 1: stats_t = torch.tensor([loc_ce, loc_mtp_ce, loc_z, float(loc_tok)], dtype=torch.float64, device=device) dist.all_reduce(stats_t, op=dist.ReduceOp.SUM) acc_ce += float(stats_t[0].item()) acc_mtp += float(stats_t[1].item()) acc_z += float(stats_t[2].item()) acc_tot_tok += int(stats_t[3].item()) else: acc_ce += loc_ce acc_mtp += loc_mtp_ce acc_z += loc_z acc_tot_tok += loc_tok acc_valid_tok += glob_valid acc_mtp_tok += glob_mtp log_int = 10 if args.verify_steps > 0 else 50 if step % log_int == 0: if is_main: dt = time.time() - t0 tok_s = acc_tot_tok / max(1e-3, dt) vram_g = torch.cuda.max_memory_allocated(device) / 1e9 pct = 100.0 * min(rows_seen, len(full_plan)) / len(full_plan) print( f"Step {step:6d} ({pct:5.1f}% | Chunk {chunk_idx+1}/{num_chunks}) | " f"Main: {acc_ce/max(1,acc_valid_tok):.4f} | MTP2: {acc_mtp/max(1,acc_mtp_tok):.4f} | " f"Z: {acc_z/max(1,acc_valid_tok):.2f} | LR: {lr:.2e} | Grad: {grad_norm:.2f} | " f"{tok_s/1e3:.1f}k tok/s | VRAM: {vram_g:.1f}G" ) acc_ce = acc_mtp = acc_z = 0.0 acc_valid_tok = acc_mtp_tok = acc_tot_tok = 0 t0 = time.time() if args.verify_steps > 0 and step >= args.verify_steps: done_flag = True break if step % args.eval_every == 0: if is_main: comb, e2m, m2e = eval_val_nll() print("-" * 96) print(f"🔎 EVAL @ Step {step:6d} | Comb NLL: {comb:.4f} (PPL {math.exp(min(comb, 30.0)):.2f}) | En->My: {e2m:.4f} | My->En: {m2e:.4f}") run_style_generations(step) if comb < best_val_nll: best_val_nll = comb best_sd = {k: v.to(torch.bfloat16) for k, v in clean_state_dict(model).items()} torch.save(best_sd, os.path.join(CKPT_DIR, "best_model.pt")) print(f" ⭐ New Best Model saved! Comb NLL: {best_val_nll:.4f}") print("-" * 96) model.train() if world_size > 1: dist.barrier() t0 = time.time() if step % args.save_every == 0: if is_main: tmp_p = latest_path + ".tmp" torch.save({ "model": clean_state_dict(model), "opt": opt.state_dict(), "step": step, "stage": args.stage, "chunk_idx": chunk_idx, "batch_cursor": batch_cursor, "rows_seen": rows_seen, "best_val_nll": best_val_nll, }, tmp_p) os.replace(tmp_p, latest_path) if world_size > 1: dist.barrier() if step % args.milestone_every == 0: if is_main: ms_sd = {k: v.to(torch.bfloat16) for k, v in clean_state_dict(model).items()} torch.save(ms_sd, os.path.join(CKPT_DIR, f"{args.stage}_step_{step:06d}.pt")) if world_size > 1: dist.barrier() if done_flag: break chunk_idx += 1 batch_cursor = 0 if args.verify_steps > 0: if is_main: print(f"✅ {world_size}-GPU VERIFY PASSED ({step} steps)! Ready for launch.") if world_size > 1: dist.barrier() dist.destroy_process_group() return if is_main: comb, e2m, m2e = eval_val_nll() print("-" * 96) print(f"🏁 FINAL EVAL @ Step {step:6d} | Comb NLL: {comb:.4f} (PPL {math.exp(min(comb, 30.0)):.2f}) | En->My: {e2m:.4f} | My->En: {m2e:.4f}") run_style_generations(step) if comb < best_val_nll: best_val_nll = comb best_sd = {k: v.to(torch.bfloat16) for k, v in clean_state_dict(model).items()} torch.save(best_sd, os.path.join(CKPT_DIR, "best_model.pt")) tmp_p = latest_path + ".tmp" torch.save({ "model": clean_state_dict(model), "opt": opt.state_dict(), "step": step, "stage": args.stage, "chunk_idx": chunk_idx, "batch_cursor": batch_cursor, "rows_seen": rows_seen, "best_val_nll": best_val_nll, }, tmp_p) os.replace(tmp_p, latest_path) with torch.no_grad(): rw = F.softmax(model.enc_router_logits, dim=-1).cpu().numpy() for p in range(model.n_cross_pairs): top_layers = np.argsort(rw[p])[::-1][:4].tolist() print(f" • Cross-KV Pair {p} top encoder mixture weights: {top_layers} ({[round(float(rw[p][k]), 3) for k in top_layers]})") print(f"🎉 Stage '{args.stage}' 100% COMPLETE ({rows_seen:,} rows trained across {step:,} steps)!") if world_size > 1: dist.barrier() dist.destroy_process_group() if __name__ == "__main__": main()