Translation
Transformers
Safetensors
Burmese
English
mytranslation
text2text-generation
Translation
Safetensors
Burmese
English
myanmartranslator
MyanmarLLM
custom_code
myanmar
Instructions to use freococo/MyTranslation_300M_Base with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use freococo/MyTranslation_300M_Base with Transformers:
# Use a pipeline as a high-level helper # Warning: Pipeline type "translation" is no longer supported in transformers v5. # You must load the model directly (see below) or downgrade to v4.x with: # pip install "transformers<5.0.0" from transformers import pipeline pipe = pipeline("translation", model="freococo/MyTranslation_300M_Base", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForSeq2SeqLM model = AutoModelForSeq2SeqLM.from_pretrained("freococo/MyTranslation_300M_Base", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download training_codes.txt from freococo/MyTranslation_300M_Base: direct link, hf CLI and curl.
- Browser
- Download file 54.4 kB
-
https://huggingface.co/freococo/MyTranslation_300M_Base/resolve/main/training_codes.txt
- Command line
-
hf download hf://freococo/MyTranslation_300M_Base/training_codes.txt
-
curl -L -o training_codes.txt https://huggingface.co/freococo/MyTranslation_300M_Base/resolve/main/training_codes.txt
54.4 kB
import os
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
import argparse, json, math, time
from datetime import timedelta
import numpy as np
import sentencepiece as spm
import torch
import torch.distributed as dist
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.checkpoint import checkpoint as grad_ckpt
from torch.utils.data import Dataset, DataLoader
torch.backends.cuda.matmul.allow_tf32 = True
torch.backends.cudnn.allow_tf32 = True
torch._dynamo.config.cache_size_limit = 64
DATA_DIR = "/workspace/mytrans_dl/tokenized_65k"
CKPT_DIR = "/workspace/ckpt_300m"
VOCAB_SIZE = 65536
UNK_ID, BOS_ID, EOS_ID, PAD_ID = 0, 1, 2, 3
TOK_2MY, TOK_2EN, TOK_FORMAL, TOK_SPOKEN, TOK_BT, TOK_DENOISE, TOK_MASK = 4, 5, 6, 7, 8, 9, 10
FLAG_BT, FLAG_FORMAL, FLAG_SPOKEN = 1, 2, 4
BULK_SUBSETS = {
"MyanmarTranslator_works/myanmar_spoken_corpus",
"MyanmarTranslator_works/myanmar-cc100-dataset",
"MyanmarTranslator_works/myanmar-written-corpus",
}
def load_and_verify_contracts(data_dir: str):
with open(os.path.join(data_dir, "meta.json"), "r") as f:
meta = json.load(f)
sp = spm.SentencePieceProcessor()
sp.load(os.path.join(data_dir, "tokenizer.model"))
assert sp.get_piece_size() == VOCAB_SIZE
assert sp.unk_id() == UNK_ID and sp.bos_id() == BOS_ID and sp.eos_id() == EOS_ID and sp.pad_id() == PAD_ID
expected = {4: "<2my>", 5: "<2en>", 6: "<formal>", 7: "<spoken>", 8: "<bt>", 9: "<denoise>", 10: "<mask>"}
for tid, piece in expected.items():
assert sp.id_to_piece(tid) == piece, f"Token {tid} mismatch"
if "special_tokens" in meta:
st = meta["special_tokens"]
assert st["2my"] == TOK_2MY and st["2en"] == TOK_2EN
assert st["formal"] == TOK_FORMAL and st["spoken"] == TOK_SPOKEN
assert st["bt"] == TOK_BT and st["denoise"] == TOK_DENOISE and st["mask"] == TOK_MASK
return meta, sp
# ==============================================================================
# 1. MyTranslation-300M Architecture
# ==============================================================================
class RMSNorm(nn.Module):
def __init__(self, dim: int, eps: float = 1e-6):
super().__init__()
self.dim = (dim,)
self.eps = eps
self.weight = nn.Parameter(torch.ones(dim))
def forward(self, x: torch.Tensor) -> torch.Tensor:
return F.rms_norm(x, self.dim, self.weight, self.eps)
class RotaryEmbedding(nn.Module):
def __init__(self, dim: int, max_seq_len: int = 2048, theta: float = 50000.0):
super().__init__()
inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
t = torch.arange(max_seq_len, dtype=torch.float32)
freqs = torch.outer(t, inv_freq)
self.register_buffer("cos_cached", freqs.cos()[None, None, :, :], persistent=False)
self.register_buffer("sin_cached", freqs.sin()[None, None, :, :], persistent=False)
def forward(self, x: torch.Tensor) -> torch.Tensor:
L = x.size(2)
cos = self.cos_cached[:, :, :L, :].to(dtype=x.dtype)
sin = self.sin_cached[:, :, :L, :].to(dtype=x.dtype)
x1, x2 = x[..., ::2], x[..., 1::2]
return torch.cat((x1 * cos - x2 * sin, x1 * sin + x2 * cos), dim=-1)
class SwiGLU(nn.Module):
def __init__(self, d_model: int, ffn_dim: int):
super().__init__()
self.gate_up = nn.Linear(d_model, 2 * ffn_dim, bias=False)
self.down = nn.Linear(ffn_dim, d_model, bias=False)
def forward(self, x: torch.Tensor) -> torch.Tensor:
gate, up = self.gate_up(x).chunk(2, dim=-1)
return self.down(F.silu(gate) * up)
class GatedGQASelfAttention(nn.Module):
def __init__(self, d_model: int, n_heads: int, n_kv_heads: int, is_causal: bool = False, has_v_res: bool = False):
super().__init__()
self.n_heads = n_heads
self.n_kv_heads = n_kv_heads
self.head_dim = d_model // n_heads
self.is_causal = is_causal
self.q_proj = nn.Linear(d_model, n_heads * self.head_dim, bias=False)
self.kv_proj = nn.Linear(d_model, 2 * n_kv_heads * self.head_dim, bias=False)
self.gate_proj = nn.Linear(d_model, n_heads, bias=False)
self.out = nn.Linear(d_model, d_model, bias=False)
self.q_norm = RMSNorm(self.head_dim)
self.k_norm = RMSNorm(self.head_dim)
self.v_res_logit = nn.Parameter(torch.tensor(-2.0)) if has_v_res else None
def forward(self, x: torch.Tensor, rope: RotaryEmbedding, attn_mask: torch.Tensor = None, v0: torch.Tensor = None):
B, L, D = x.shape
q = self.q_norm(self.q_proj(x).view(B, L, self.n_heads, self.head_dim).transpose(1, 2))
k, v = self.kv_proj(x).chunk(2, dim=-1)
k = self.k_norm(k.view(B, L, self.n_kv_heads, self.head_dim).transpose(1, 2))
v = v.view(B, L, self.n_kv_heads, self.head_dim).transpose(1, 2)
v_raw = v
if self.v_res_logit is not None and v0 is not None:
alpha = torch.sigmoid(self.v_res_logit).to(dtype=v.dtype)
v = (1.0 - alpha) * v + alpha * v0
q, k = rope(q), rope(k)
y = F.scaled_dot_product_attention(
q, k, v,
attn_mask=None if self.is_causal else attn_mask,
is_causal=self.is_causal,
enable_gqa=True,
)
gate = torch.sigmoid(self.gate_proj(x)).transpose(1, 2).unsqueeze(-1)
y = (y * gate).transpose(1, 2).contiguous().view(B, L, D)
return self.out(y), v_raw
def step_kv(self, x_step, cos_p, sin_p, pos, k_cache, v_cache, v0=None):
B, _, D = x_step.shape
q = self.q_norm(self.q_proj(x_step).view(B, 1, self.n_heads, self.head_dim).transpose(1, 2))
k, v = self.kv_proj(x_step).chunk(2, dim=-1)
k = self.k_norm(k.view(B, 1, self.n_kv_heads, self.head_dim).transpose(1, 2))
v = v.view(B, 1, self.n_kv_heads, self.head_dim).transpose(1, 2)
v_raw = v
if self.v_res_logit is not None and v0 is not None:
alpha = torch.sigmoid(self.v_res_logit).to(dtype=v.dtype)
v = (1.0 - alpha) * v + alpha * v0
q1, q2 = q[..., ::2], q[..., 1::2]
k1, k2 = k[..., ::2], k[..., 1::2]
q = torch.cat((q1 * cos_p - q2 * sin_p, q1 * sin_p + q2 * cos_p), dim=-1)
k = torch.cat((k1 * cos_p - k2 * sin_p, k1 * sin_p + k2 * cos_p), dim=-1)
k_cache[:, :, pos : pos + 1, :] = k
v_cache[:, :, pos : pos + 1, :] = v
y = F.scaled_dot_product_attention(
q, k_cache[:, :, : pos + 1, :], v_cache[:, :, : pos + 1, :],
is_causal=False, enable_gqa=True,
)
gate = torch.sigmoid(self.gate_proj(x_step)).transpose(1, 2).unsqueeze(-1)
y = (y * gate).transpose(1, 2).contiguous().view(B, 1, D)
return self.out(y), v_raw
class SharedCrossKVProj(nn.Module):
def __init__(self, d_model: int, n_kv_heads: int, head_dim: int):
super().__init__()
self.n_kv_heads = n_kv_heads
self.head_dim = head_dim
self.norm = RMSNorm(d_model)
self.kv_proj = nn.Linear(d_model, 2 * n_kv_heads * head_dim, bias=False)
self.k_norm = RMSNorm(head_dim)
def forward(self, enc_mix: torch.Tensor):
B, S, _ = enc_mix.shape
k, v = self.kv_proj(self.norm(enc_mix)).chunk(2, dim=-1)
k = self.k_norm(k.view(B, S, self.n_kv_heads, self.head_dim).transpose(1, 2))
v = v.view(B, S, self.n_kv_heads, self.head_dim).transpose(1, 2)
return k, v
class GatedGQACrossAttention(nn.Module):
def __init__(self, d_model: int, n_heads: int):
super().__init__()
self.n_heads = n_heads
self.head_dim = d_model // n_heads
self.q_proj = nn.Linear(d_model, n_heads * self.head_dim, bias=False)
self.gate_proj = nn.Linear(d_model, n_heads, bias=False)
self.out = nn.Linear(d_model, d_model, bias=False)
self.q_norm = RMSNorm(self.head_dim)
def forward(self, x: torch.Tensor, k_cross: torch.Tensor, v_cross: torch.Tensor, enc_mask: torch.Tensor):
B, T, D = x.shape
q = self.q_norm(self.q_proj(x).view(B, T, self.n_heads, self.head_dim).transpose(1, 2))
y = F.scaled_dot_product_attention(q, k_cross, v_cross, attn_mask=enc_mask, is_causal=False, enable_gqa=True)
gate = torch.sigmoid(self.gate_proj(x)).transpose(1, 2).unsqueeze(-1)
y = (y * gate).transpose(1, 2).contiguous().view(B, T, D)
return self.out(y)
class EncoderLayer(nn.Module):
def __init__(self, d_model: int, n_heads: int, n_kv_heads: int, ffn_dim: int, has_v_res: bool):
super().__init__()
self.norm1 = RMSNorm(d_model)
self.self_attn = GatedGQASelfAttention(d_model, n_heads, n_kv_heads, is_causal=False, has_v_res=has_v_res)
self.norm2 = RMSNorm(d_model)
self.mlp = SwiGLU(d_model, ffn_dim)
def forward(self, x, rope, enc_mask, v0=None):
attn_out, v_raw = self.self_attn(self.norm1(x), rope, enc_mask, v0=v0)
x = x + attn_out
x = x + self.mlp(self.norm2(x))
return x, v_raw
class DecoderLayer(nn.Module):
def __init__(self, d_model: int, n_heads: int, n_kv_heads: int, ffn_dim: int, has_v_res: bool):
super().__init__()
self.norm1 = RMSNorm(d_model)
self.self_attn = GatedGQASelfAttention(d_model, n_heads, n_kv_heads, is_causal=True, has_v_res=has_v_res)
self.norm2 = RMSNorm(d_model)
self.cross_attn = GatedGQACrossAttention(d_model, n_heads)
self.norm3 = RMSNorm(d_model)
self.mlp = SwiGLU(d_model, ffn_dim)
def forward(self, x, k_cross, v_cross, rope, enc_mask, v0=None):
attn_out, v_raw = self.self_attn(self.norm1(x), rope, None, v0=v0)
x = x + attn_out
x = x + self.cross_attn(self.norm2(x), k_cross, v_cross, enc_mask)
x = x + self.mlp(self.norm3(x))
return x, v_raw
def step_kv(self, x_step, cos_p, sin_p, pos, k_self, v_self, k_cross, v_cross, enc_mask, v0=None):
attn_out, v_raw = self.self_attn.step_kv(self.norm1(x_step), cos_p, sin_p, pos, k_self, v_self, v0=v0)
x_step = x_step + attn_out
x_step = x_step + self.cross_attn(self.norm2(x_step), k_cross, v_cross, enc_mask)
x_step = x_step + self.mlp(self.norm3(x_step))
return x_step, v_raw
class MTPHeadBlock(nn.Module):
def __init__(self, d_model: int, n_heads: int, n_kv_heads: int, ffn_dim: int):
super().__init__()
self.norm_h = RMSNorm(d_model)
self.norm_e = RMSNorm(d_model)
self.proj = nn.Linear(2 * d_model, d_model, bias=False)
self.norm1 = RMSNorm(d_model)
self.self_attn = GatedGQASelfAttention(d_model, n_heads, n_kv_heads, is_causal=True, has_v_res=False)
self.norm2 = RMSNorm(d_model)
self.mlp = SwiGLU(d_model, ffn_dim)
self.out_norm = RMSNorm(d_model)
def forward(self, h_main: torch.Tensor, next_tok_emb: torch.Tensor, rope: RotaryEmbedding):
x = self.proj(torch.cat([self.norm_h(h_main), self.norm_e(next_tok_emb)], dim=-1))
attn_out, _ = self.self_attn(self.norm1(x), rope, None)
x = x + attn_out
x = x + self.mlp(self.norm2(x))
return self.out_norm(x)
class MyTranslation300M(nn.Module):
def __init__(
self,
vocab_size=65536,
d_model=1024,
n_heads=16,
n_kv_heads=4,
enc_layers=12,
dec_layers=6,
ffn_dim=3072,
max_seq_len=2048,
):
super().__init__()
assert d_model % n_heads == 0 and n_heads % n_kv_heads == 0 and dec_layers % 2 == 0
self.config = dict(
vocab_size=vocab_size, d_model=d_model, n_heads=n_heads, n_kv_heads=n_kv_heads,
enc_layers=enc_layers, dec_layers=dec_layers, ffn_dim=ffn_dim, max_seq_len=max_seq_len,
)
self.d_model = d_model
self.n_heads = n_heads
self.n_kv_heads = n_kv_heads
self.head_dim = d_model // n_heads
self.enc_layers = enc_layers
self.dec_layers = dec_layers
self.n_cross_pairs = dec_layers // 2
self.embed_scale = math.sqrt(d_model)
self.embed_tokens = nn.Embedding(vocab_size, d_model, padding_idx=PAD_ID)
self.rope = RotaryEmbedding(self.head_dim, max_seq_len=max_seq_len, theta=50000.0)
self.encoder = nn.ModuleList([
EncoderLayer(d_model, n_heads, n_kv_heads, ffn_dim, has_v_res=(i > 0))
for i in range(enc_layers)
])
self.enc_dwa_logits = nn.ParameterList([
nn.Parameter(torch.tensor([-4.0] * i + [0.0], dtype=torch.float32))
for i in range(1, enc_layers)
])
self.enc_router_logits = nn.Parameter(torch.zeros(self.n_cross_pairs, enc_layers))
with torch.no_grad():
for p in range(self.n_cross_pairs):
center = (p + 0.5) * (enc_layers / self.n_cross_pairs) - 0.5
for k in range(enc_layers):
self.enc_router_logits[p, k] = -0.35 * ((k - center) ** 2)
self.cross_kv_projs = nn.ModuleList([
SharedCrossKVProj(d_model, n_kv_heads, self.head_dim)
for _ in range(self.n_cross_pairs)
])
self.decoder = nn.ModuleList([
DecoderLayer(d_model, n_heads, n_kv_heads, ffn_dim, has_v_res=(i > 0))
for i in range(dec_layers)
])
self.dec_norm = RMSNorm(d_model)
self.lm_head = nn.Linear(d_model, vocab_size, bias=False)
self.lm_head.weight = self.embed_tokens.weight
self.mtp_head = MTPHeadBlock(d_model, n_heads, n_kv_heads, ffn_dim)
self._init_weights(enc_layers + dec_layers)
def _init_weights(self, total_layers: int):
res_std = 0.02 / math.sqrt(2 * total_layers)
for name, m in self.named_modules():
if isinstance(m, nn.Linear):
if name.endswith(".out") or name.endswith(".down"):
nn.init.normal_(m.weight, mean=0.0, std=res_std)
elif name.endswith(".gate_proj"):
nn.init.normal_(m.weight, mean=0.0, std=0.01)
else:
nn.init.normal_(m.weight, mean=0.0, std=0.02)
elif isinstance(m, nn.Embedding):
nn.init.normal_(m.weight, mean=0.0, std=0.02)
if m.padding_idx is not None:
m.weight.data[m.padding_idx].zero_()
def compile_blocks(self):
for i in range(len(self.encoder)):
self.encoder[i] = torch.compile(self.encoder[i], dynamic=True)
for i in range(len(self.cross_kv_projs)):
self.cross_kv_projs[i] = torch.compile(self.cross_kv_projs[i], dynamic=True)
for i in range(len(self.decoder)):
self.decoder[i] = torch.compile(self.decoder[i], dynamic=True)
self.mtp_head = torch.compile(self.mtp_head, dynamic=True)
def encode_and_route(self, src_ids: torch.Tensor, ckpt_mode: str = "none"):
enc_mask = (src_ids != PAD_ID).unsqueeze(1).unsqueeze(2)
x = self.embed_tokens(src_ids) * self.embed_scale
raw_states = [x]
enc_outputs = []
v0 = None
for i, layer in enumerate(self.encoder):
if i > 0:
w_dwa = F.softmax(self.enc_dwa_logits[i - 1], dim=0).to(dtype=x.dtype)
if self.training and ckpt_mode != "none":
x_in = grad_ckpt(
lambda w, *st: torch.einsum("j,jbsd->bsd", w, torch.stack(st, dim=0)),
w_dwa, *raw_states, use_reentrant=False
)
else:
x_in = torch.einsum("j,jbsd->bsd", w_dwa, torch.stack(raw_states, dim=0))
else:
x_in = x
do_ckpt = self.training and (
(ckpt_mode == "heavy" and (i % 3 != 0)) or
(ckpt_mode == "light" and (i in (3, 7, 11)))
)
if do_ckpt:
x_out, v_raw = grad_ckpt(layer, x_in, self.rope, enc_mask, v0, use_reentrant=False)
else:
x_out, v_raw = layer(x_in, self.rope, enc_mask, v0=v0)
if i == 0:
v0 = v_raw
raw_states.append(x_out)
enc_outputs.append(x_out)
stacked_enc = torch.stack(enc_outputs, dim=0)
router_w = F.softmax(self.enc_router_logits, dim=-1).to(dtype=stacked_enc.dtype)
cross_kv = []
for p in range(self.n_cross_pairs):
enc_mix = torch.einsum("k,kbsd->bsd", router_w[p], stacked_enc)
cross_kv.append(self.cross_kv_projs[p](enc_mix))
return cross_kv, enc_mask
def decode_seq(self, tgt_in: torch.Tensor, cross_kv: list, enc_mask: torch.Tensor, ckpt_mode: str = "none"):
x = self.embed_tokens(tgt_in) * self.embed_scale
v0 = None
for i, layer in enumerate(self.decoder):
kc, vc = cross_kv[i // 2]
do_ckpt = self.training and (
(ckpt_mode == "heavy" and (i % 2 == 1)) or
(ckpt_mode == "light" and (i == 3))
)
if do_ckpt:
x, v_raw = grad_ckpt(layer, x, kc, vc, self.rope, enc_mask, v0, use_reentrant=False)
else:
x, v_raw = layer(x, kc, vc, self.rope, enc_mask, v0=v0)
if i == 0:
v0 = v_raw
return self.dec_norm(x)
def forward_and_backward(
self,
src_ids: torch.Tensor,
tgt_in: torch.Tensor,
labels: torch.Tensor,
main_denom: float,
mtp_denom: float,
label_smoothing: float = 0.1,
z_loss_weight: float = 1e-4,
mtp_weight: float = 0.3,
):
tot_tok = src_ids.numel() + tgt_in.numel()
max_len = max(src_ids.size(1), tgt_in.size(1))
if tot_tok <= 14336:
ckpt_mode = "none"
elif max_len <= 256:
ckpt_mode = "light"
else:
ckpt_mode = "heavy"
with torch.amp.autocast("cuda", dtype=torch.bfloat16):
cross_kv, enc_mask = self.encode_and_route(src_ids, ckpt_mode=ckpt_mode)
h_main = self.decode_seq(tgt_in, cross_kv, enc_mask, ckpt_mode=ckpt_mode)
valid_mask = labels.ne(-100)
valid_y = labels[valid_mask]
n_valid = valid_y.numel()
valid_h_leaf = h_main[valid_mask].detach().requires_grad_(True)
tot_main_ce_t = torch.zeros((), dtype=torch.float64, device=h_main.device)
tot_z_loss_t = torch.zeros((), dtype=torch.float64, device=h_main.device)
tot_mtp_ce_t = torch.zeros((), dtype=torch.float64, device=h_main.device)
chunk_sz = 2048
for i in range(0, n_valid, chunk_sz):
hc = valid_h_leaf[i : i + chunk_sz]
yc = valid_y[i : i + chunk_sz]
with torch.amp.autocast("cuda", dtype=torch.bfloat16):
logits_c = self.lm_head(hc)
ce_c = F.cross_entropy(logits_c, yc, label_smoothing=label_smoothing, reduction="sum")
lse = torch.logsumexp(logits_c.float(), dim=-1)
z_c = (lse * lse).sum()
loss_c = (ce_c + z_loss_weight * z_c) / main_denom
loss_c.backward()
tot_main_ce_t.add_(ce_c.detach().to(torch.float64))
tot_z_loss_t.add_(z_c.detach().to(torch.float64))
del logits_c, ce_c, z_c, loss_c
grad_h_main = torch.zeros_like(h_main)
if valid_h_leaf.grad is not None:
grad_h_main[valid_mask] = valid_h_leaf.grad
n_mtp = 0
if mtp_weight > 0.0 and tgt_in.size(1) > 1:
mtp_mask = labels[:, 1:].ne(-100) & labels[:, :-1].ne(-100)
valid_ym = labels[:, 1:][mtp_mask]
n_mtp = valid_ym.numel()
if n_mtp > 0:
h_main_leaf = h_main.detach().requires_grad_(True)
with torch.amp.autocast("cuda", dtype=torch.bfloat16):
next_emb = self.embed_tokens(tgt_in[:, 1:]) * self.embed_scale
h_mtp = self.mtp_head(h_main_leaf[:, :-1, :], next_emb, self.rope)
valid_hm_leaf = h_mtp[mtp_mask].detach().requires_grad_(True)
for i in range(0, n_mtp, chunk_sz):
hmc = valid_hm_leaf[i : i + chunk_sz]
ymc = valid_ym[i : i + chunk_sz]
with torch.amp.autocast("cuda", dtype=torch.bfloat16):
logits_m = self.lm_head(hmc)
ce_m = F.cross_entropy(logits_m, ymc, label_smoothing=label_smoothing, reduction="sum")
loss_m = (mtp_weight * ce_m) / mtp_denom
loss_m.backward()
tot_mtp_ce_t.add_(ce_m.detach().to(torch.float64))
del logits_m, ce_m, loss_m
grad_h_mtp = torch.zeros_like(h_mtp)
if valid_hm_leaf.grad is not None:
grad_h_mtp[mtp_mask] = valid_hm_leaf.grad
h_mtp.backward(grad_h_mtp)
if h_main_leaf.grad is not None:
grad_h_main.add_(h_main_leaf.grad)
h_main.backward(grad_h_main)
ce_v, mtp_v, z_v = torch.stack([tot_main_ce_t, tot_mtp_ce_t, tot_z_loss_t]).tolist()
return ce_v, mtp_v, z_v, n_valid, n_mtp
@torch.no_grad()
def eval_nll(self, src_ids: torch.Tensor, tgt_in: torch.Tensor, labels: torch.Tensor):
with torch.amp.autocast("cuda", dtype=torch.bfloat16):
cross_kv, enc_mask = self.encode_and_route(src_ids, ckpt_mode="none")
h_main = self.decode_seq(tgt_in, cross_kv, enc_mask, ckpt_mode="none")
valid_mask = labels.ne(-100)
valid_h = h_main[valid_mask]
valid_y = labels[valid_mask]
n_valid = valid_y.numel()
tot_nll = 0.0
for i in range(0, n_valid, 4096):
logits_c = self.lm_head(valid_h[i : i + 4096])
tot_nll += float(F.cross_entropy(logits_c, valid_y[i : i + 4096], reduction="sum").item())
return tot_nll, n_valid
@torch.inference_mode()
def translate_kv(self, prompt_ids_list: list[list[int]], sp_proc, max_new_tokens: int = 160, rep_penalty: float = 1.12) -> list[str]:
assert len(prompt_ids_list) > 0 and all(len(s) > 0 for s in prompt_ids_list), "Empty prompt batch!"
max_rope = self.rope.cos_cached.size(2)
assert 0 < max_new_tokens <= max_rope, f"max_new_tokens={max_new_tokens} exceeds RoPE={max_rope}"
self.eval()
dev = self.embed_tokens.weight.device
B = len(prompt_ids_list)
trimmed = [s[:max_rope] for s in prompt_ids_list]
max_s = min(max_rope, ((max(len(s) for s in trimmed) + 7) // 8) * 8)
src_ids = torch.full((B, max_s), PAD_ID, dtype=torch.long, device=dev)
for i, ids in enumerate(trimmed):
src_ids[i, : len(ids)] = torch.tensor(ids, dtype=torch.long, device=dev)
with torch.amp.autocast("cuda", dtype=torch.bfloat16):
cross_kv, enc_mask = self.encode_and_route(src_ids, ckpt_mode="none")
k_cache = torch.empty((self.dec_layers, B, self.n_kv_heads, max_new_tokens, self.head_dim), dtype=torch.bfloat16, device=dev)
v_cache = torch.empty((self.dec_layers, B, self.n_kv_heads, max_new_tokens, self.head_dim), dtype=torch.bfloat16, device=dev)
cur_tok = torch.full((B, 1), BOS_ID, dtype=torch.long, device=dev)
gen_ids = torch.full((B, max_new_tokens), EOS_ID, dtype=torch.long, device=dev)
done = torch.zeros(B, dtype=torch.bool, device=dev)
cos_all, sin_all = self.rope.cos_cached.to(torch.bfloat16), self.rope.sin_cached.to(torch.bfloat16)
for pos in range(max_new_tokens):
cos_p, sin_p = cos_all[:, :, pos : pos + 1, :], sin_all[:, :, pos : pos + 1, :]
x_dec = self.embed_tokens(cur_tok) * self.embed_scale
v0 = None
for l_idx, layer in enumerate(self.decoder):
kc, vc = cross_kv[l_idx // 2]
x_dec, v_raw = layer.step_kv(x_dec, cos_p, sin_p, pos, k_cache[l_idx], v_cache[l_idx], kc, vc, enc_mask, v0=v0)
if l_idx == 0:
v0 = v_raw
logits = self.lm_head(self.dec_norm(x_dec)[:, 0, :]).float()
logits[:, [0, 1, 3, 4, 5, 6, 7, 8, 9, 10]] = -1e9
if rep_penalty != 1.0 and pos > 0:
prev = gen_ids[:, :pos]
s_l = torch.gather(logits, 1, prev)
s_l = torch.where(s_l > 0, s_l / rep_penalty, s_l * rep_penalty)
logits.scatter_(1, prev, s_l)
nxt = torch.where(done, torch.tensor(EOS_ID, device=dev), logits.argmax(dim=-1))
gen_ids[:, pos] = nxt
done = done | (nxt == EOS_ID)
cur_tok = nxt.unsqueeze(1)
if bool(done.all()):
break
out_texts = []
for row in gen_ids.cpu().tolist():
if EOS_ID in row:
row = row[: row.index(EOS_ID)]
out_texts.append(sp_proc.decode_ids([x for x in row if x >= 11]).strip())
return out_texts
# ==============================================================================
# 2. 100%-Coverage Plan + Tight-Padding Dynamic Packing + Distributed Sharding
# ==============================================================================
class CanonicalMemmapStore:
def __init__(self, data_dir: str, split: str):
self.my_path = os.path.join(data_dir, f"{split}_my.bin")
self.en_path = os.path.join(data_dir, f"{split}_en.bin")
self.index = np.load(os.path.join(data_dir, f"{split}_index.npy"), mmap_mode="r")
self.my_mm = None
self.en_mm = None
def ensure_open(self):
if self.my_mm is None:
self.my_mm = np.memmap(self.my_path, dtype=np.uint16, mode="r")
self.en_mm = np.memmap(self.en_path, dtype=np.uint16, mode="r")
def get_pair(self, idx: int):
self.ensure_open()
rec = self.index[idx]
m = self.my_mm[int(rec["my_off"]) : int(rec["my_off"]) + int(rec["my_len"])].astype(np.int64)
e = self.en_mm[int(rec["en_off"]) : int(rec["en_off"]) + int(rec["en_len"])].astype(np.int64)
return m, e, int(rec["flags"])
def build_or_load_epoch_plan(train_index: np.ndarray, meta: dict, stage: str, plan_path: str, is_main: bool, world_size: int, seed: int = 2026) -> np.ndarray:
if is_main and not os.path.exists(plan_path):
rng = np.random.RandomState(seed)
n_subsets = len(meta["subset_to_id"])
sids = np.asarray(train_index["subset_id"])
assert int(sids.max()) < n_subsets and int(sids.min()) >= 0
if stage == "stable":
base_rows = rng.permutation(len(train_index)).astype(np.uint32)
replay_n = 12_000_000
eligible_mask = np.ones(len(train_index), dtype=bool)
for k, name in meta["id_to_subset"].items():
if name in BULK_SUBSETS:
eligible_mask[sids == int(k)] = False
else:
eligible_mask = (np.asarray(train_index["flags"]) & FLAG_BT) == 0
base_rows = np.flatnonzero(eligible_mask).astype(np.uint32)
rng.shuffle(base_rows)
replay_n = 1_500_000
elig_pos = np.flatnonzero(eligible_mask).astype(np.uint32)
elig_sids = sids[elig_pos]
order = elig_pos[np.argsort(elig_sids, kind="mergesort")]
sorted_sids = sids[order]
counts = np.bincount(sorted_sids, minlength=n_subsets)
starts = np.zeros(n_subsets, dtype=np.int64)
starts[1:] = np.cumsum(counts[:-1])
weights = np.zeros(n_subsets, dtype=np.float64)
for k, name in meta["id_to_subset"].items():
sid = int(k)
if counts[sid] == 0:
continue
if stage == "stable" and name in BULK_SUBSETS:
continue
if stage == "cooldown" and name == "non_myanmartranslator/finetranslations":
w = float(counts[sid]) ** 0.35 * 0.25
else:
w = float(counts[sid]) ** 0.50
if stage == "stable" and name.startswith("non_myanmartranslator/"):
w *= 4.0
weights[sid] = w
assert weights.sum() > 0, "Empty replay weights!"
probs = weights / weights.sum()
chosen_sids = rng.choice(n_subsets, size=replay_n, p=probs)
replay_rows = np.empty(replay_n, dtype=np.uint32)
for sid in range(n_subsets):
m = (chosen_sids == sid)
npick = int(m.sum())
if npick > 0:
replay_rows[m] = order[starts[sid] + rng.randint(0, counts[sid], size=npick)]
full_plan = np.concatenate([base_rows, replay_rows])
rng.shuffle(full_plan)
tmp_plan = plan_path + ".tmp.npy"
np.save(tmp_plan, full_plan)
os.replace(tmp_plan, plan_path)
print(
f"📋 Built {stage} 100%-coverage plan: {len(base_rows):,} unique base rows (100% coverage) "
f"+ {len(replay_rows):,} gold replay = {len(full_plan):,} total rows"
)
if world_size > 1:
dist.barrier()
plan = np.load(plan_path, mmap_mode="r")
if is_main:
print(f"📋 Ready {stage} plan: {len(plan):,} rows")
return plan
def pack_chunk_batches(
row_slice: np.ndarray,
train_index: np.ndarray,
max_tokens_combined: int,
max_seq_len: int,
step_group: int,
seed: int,
):
rng = np.random.RandomState(seed)
my_lens = np.minimum(max_seq_len, train_index["my_len"][row_slice].astype(np.int32))
en_lens = np.minimum(max_seq_len, train_index["en_len"][row_slice].astype(np.int32))
max_side = np.maximum(my_lens, en_lens)
comb_lens = my_lens + en_lens
mega = 65536
batches_cost = []
for i in range(0, len(row_slice), mega):
sub_r = row_slice[i : i + mega]
sub_ms = max_side[i : i + mega]
sub_cl = comb_lens[i : i + mega]
sub_r = sub_r[np.lexsort((sub_cl, sub_ms))]
cur_b, cur_s_max, cur_t_max = [], 0, 0
for r in sub_r:
ri = int(r)
ml = min(max_seq_len, int(train_index["my_len"][ri]))
el = min(max_seq_len, int(train_index["en_len"][ri]))
s_est = max(el, ml) + 4
t_est = max(ml, el) + 2
new_s = max(cur_s_max, s_est)
new_t = max(cur_t_max, t_est)
if cur_b and (len(cur_b) >= 256 or (len(cur_b) + 1) * (new_s + new_t) > max_tokens_combined):
batches_cost.append(((max(cur_s_max, cur_t_max), len(cur_b) * (cur_s_max + cur_t_max)), cur_b))
cur_b, cur_s_max, cur_t_max = [ri], s_est, t_est
else:
cur_b.append(ri)
cur_s_max, cur_t_max = new_s, new_t
if cur_b:
batches_cost.append(((max(cur_s_max, cur_t_max), len(cur_b) * (cur_s_max + cur_t_max)), cur_b))
# Load-balanced step grouping by (max_seq, tok_cost) + safe small-chunk pad
batches_cost.sort(key=lambda x: x[0])
batches = [b for _, b in batches_cost]
rem = len(batches) % step_group
if rem != 0 and len(batches) > 0:
need = step_group - rem
batches.extend((batches * ((need // len(batches)) + 1))[:need])
groups = [batches[j : j + step_group] for j in range(0, len(batches), step_group)]
rng.shuffle(groups)
return [b for g in groups for b in g]
class DynamicStreamDataset(Dataset):
def __init__(self, store: CanonicalMemmapStore, batches: list[list[int]], stage: str, max_seq_len: int, seed: int):
self.store = store
self.batches = batches
self.stage = stage
self.max_seq_len = max_seq_len
self.base_seed = seed
self.rng = np.random.RandomState(seed)
def __len__(self):
return len(self.batches)
def _corrupt_span(self, toks: np.ndarray):
n = len(toks)
if n < 4:
return toks
out = []
i = 0
masked_any = False
while i < n:
if self.rng.rand() < 0.15:
span_len = min(n - i, max(1, int(self.rng.poisson(3.0))))
out.append(TOK_MASK)
i += span_len
masked_any = True
else:
out.append(int(toks[i]))
i += 1
if not masked_any:
idx = int(self.rng.randint(0, n))
out = list(toks[:idx]) + [TOK_MASK] + list(toks[idx + min(2, n - idx):])
return np.asarray(out, dtype=np.int64)
def __getitem__(self, b_idx: int):
row_ids = self.batches[b_idx]
srcs, tgts = [], []
for r_id in row_ids:
m, e, flg = self.store.get_pair(r_id)
m = m[: self.max_seq_len - 4]
e = e[: self.max_seq_len - 4]
is_bt = bool(flg & FLAG_BT)
is_formal = bool(flg & FLAG_FORMAL)
is_spoken = bool(flg & FLAG_SPOKEN)
if self.stage == "cooldown":
mode = "en2my" if self.rng.rand() < 0.50 else "my2en"
else:
p = self.rng.rand()
if is_bt:
mode = "en2my" if p < 0.72 else ("my2en_bt" if p < 0.90 else "denoise")
else:
mode = "en2my" if p < 0.47 else ("my2en" if p < 0.94 else "denoise")
style_tok = None
if is_formal and self.rng.rand() < 0.70:
style_tok = TOK_FORMAL
elif is_spoken and self.rng.rand() < 0.70:
style_tok = TOK_SPOKEN
if mode == "en2my":
pref = [TOK_2MY] + ([style_tok] if style_tok is not None else [])
src_arr = np.concatenate([np.asarray(pref, dtype=np.int64), e, [EOS_ID]])
tgt_arr = np.concatenate([[BOS_ID], m, [EOS_ID]])
elif mode in ("my2en", "my2en_bt"):
pref = [TOK_2EN] + ([TOK_BT] if mode == "my2en_bt" else []) + ([style_tok] if style_tok is not None else [])
src_arr = np.concatenate([np.asarray(pref, dtype=np.int64), m, [EOS_ID]])
tgt_arr = np.concatenate([[BOS_ID], e, [EOS_ID]])
else:
base = m if (is_bt or self.rng.rand() < 0.5) else e
noisy = self._corrupt_span(base)
src_arr = np.concatenate([[TOK_DENOISE], noisy, [EOS_ID]])
tgt_arr = np.concatenate([[BOS_ID], base, [EOS_ID]])
srcs.append(src_arr)
tgts.append(tgt_arr)
return srcs, tgts
def worker_init_fn(worker_id: int):
info = torch.utils.data.get_worker_info()
if info is not None and hasattr(info.dataset, "rng"):
info.dataset.rng = np.random.RandomState((info.dataset.base_seed + worker_id * 9973) % (2**31 - 1))
def collate_prebatched(batch_list):
srcs, tgts = batch_list[0]
B = len(srcs)
max_s = min(2048, ((max(len(x) for x in srcs) + 31) // 32) * 32)
max_t = min(2048, ((max(len(x) - 1 for x in tgts) + 31) // 32) * 32)
src_ids = torch.full((B, max_s), PAD_ID, dtype=torch.long)
tgt_in = torch.full((B, max_t), PAD_ID, dtype=torch.long)
labels = torch.full((B, max_t), -100, dtype=torch.long)
for i, (s, t) in enumerate(zip(srcs, tgts)):
sl = min(len(s), max_s)
tl = min(len(t) - 1, max_t)
src_ids[i, :sl] = torch.from_numpy(s[:sl])
tgt_in[i, :tl] = torch.from_numpy(t[:tl])
labels[i, :tl] = torch.from_numpy(t[1 : tl + 1])
return src_ids, tgt_in, labels
def clean_state_dict(model: nn.Module):
return {k.replace("._orig_mod.", "."): v.detach().cpu() for k, v in model.state_dict().items()}
def broadcast_model(model: nn.Module, world_size: int):
if world_size == 1:
return
for p in model.parameters():
dist.broadcast(p.data, src=0)
for b in model.buffers():
dist.broadcast(b.data, src=0)
def allreduce_grads(model: nn.Module, world_size: int, grad_scale: float = 1024.0):
if world_size == 1:
return
grads = []
for p in model.parameters():
if p.requires_grad:
if p.grad is None:
p.grad = torch.zeros_like(p.data)
grads.append(p.grad)
b_grads = [g.bfloat16() for g in grads]
flat = torch._utils._flatten_dense_tensors(b_grads)
dist.all_reduce(flat, op=dist.ReduceOp.SUM)
inv_scale = 1.0 / grad_scale
for orig, synced in zip(grads, torch._utils._unflatten_dense_tensors(flat, b_grads)):
orig.copy_(synced.float() * inv_scale)
# ==============================================================================
# 3. Main Multi-GPU Training Loop
# ==============================================================================
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--stage", choices=["stable", "cooldown"], default="stable")
parser.add_argument("--verify-steps", type=int, default=0)
parser.add_argument("--grad-accum", type=int, default=1)
parser.add_argument("--max-tok-micro", type=int, default=32768)
parser.add_argument("--max-seq-len", type=int, default=1024)
parser.add_argument("--chunk-rows", type=int, default=4000000)
parser.add_argument("--peak-lr", type=float, default=5.0e-4)
parser.add_argument("--warmup-steps", type=int, default=900)
parser.add_argument("--eval-every", type=int, default=500)
parser.add_argument("--save-every", type=int, default=500)
parser.add_argument("--milestone-every", type=int, default=5000)
parser.add_argument("--compile", action="store_true")
parser.add_argument("--fresh", action="store_true")
parser.add_argument("--init-weights", type=str, default="")
args = parser.parse_args()
world_size = int(os.environ.get("WORLD_SIZE", 1))
rank = int(os.environ.get("RANK", 0))
local_rank = int(os.environ.get("LOCAL_RANK", 0))
is_main = (rank == 0)
if world_size > 1:
torch.cuda.set_device(local_rank)
dist.init_process_group(backend="nccl", timeout=timedelta(minutes=30), device_id=torch.device(f"cuda:{local_rank}"))
device = torch.device("cuda", local_rank)
else:
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if is_main:
os.makedirs(CKPT_DIR, exist_ok=True)
if world_size > 1:
dist.barrier()
meta, sp = load_and_verify_contracts(DATA_DIR)
train_store = CanonicalMemmapStore(DATA_DIR, "train")
val_store = CanonicalMemmapStore(DATA_DIR, "val")
all_val_non_bt = np.flatnonzero((val_store.index["flags"] & FLAG_BT) == 0)
assert len(all_val_non_bt) > 0, "Validation set has zero non-BT rows!"
val_rows = all_val_non_bt[::max(1, len(all_val_non_bt) // 1024)][:1024]
val_batches = [val_rows[i : i + 64].tolist() for i in range(0, len(val_rows), 64)]
plan_path = os.path.join(CKPT_DIR, f"{args.stage}_plan.npy")
latest_path_early = os.path.join(CKPT_DIR, "latest.pt")
if args.fresh and is_main:
if os.path.exists(plan_path):
os.remove(plan_path)
if os.path.exists(latest_path_early):
os.replace(latest_path_early, latest_path_early + ".bak")
if world_size > 1:
dist.barrier()
full_plan = build_or_load_epoch_plan(train_store.index, meta, args.stage, plan_path, is_main, world_size, seed=2026)
num_chunks = (len(full_plan) + args.chunk_rows - 1) // args.chunk_rows
model = MyTranslation300M().to(device)
decay_p = [p for n, p in model.named_parameters() if p.requires_grad and p.ndim >= 2]
nodecay_p = [p for n, p in model.named_parameters() if p.requires_grad and p.ndim < 2]
opt = torch.optim.AdamW(
[{"params": decay_p, "weight_decay": 0.01}, {"params": nodecay_p, "weight_decay": 0.0}],
lr=args.peak_lr, betas=(0.9, 0.95), eps=1e-8, fused=True,
)
step = 0
chunk_idx = 0
batch_cursor = 0
rows_seen = 0
best_val_nll = float("inf")
latest_path = os.path.join(CKPT_DIR, "latest.pt")
if args.init_weights:
if is_main:
print(f"📥 Loading weights from {args.init_weights}")
model.load_state_dict(torch.load(args.init_weights, map_location="cpu"), strict=True)
elif (not args.fresh) and os.path.exists(latest_path):
if is_main:
print(f"🔄 Resuming from {latest_path}")
ckpt = torch.load(latest_path, map_location="cpu")
model.load_state_dict(ckpt["model"], strict=True)
if args.stage == ckpt.get("stage", "stable"):
opt.load_state_dict(ckpt["opt"])
step = int(ckpt["step"])
chunk_idx = int(ckpt.get("chunk_idx", 0))
batch_cursor = int(ckpt.get("batch_cursor", 0))
rows_seen = int(ckpt.get("rows_seen", 0))
best_val_nll = float(ckpt.get("best_val_nll", float("inf")))
elif is_main:
print(f"➡️ Stage transition {ckpt.get('stage')} -> {args.stage}")
broadcast_model(model, world_size)
if args.compile:
if is_main:
print("⚡ Compiling encoder/cross_kv/decoder/MTP blocks with torch.compile(dynamic=True)...")
model.compile_blocks()
if is_main:
tot_params = sum(p.numel() for p in model.parameters())
inf_params = tot_params - sum(p.numel() for p in model.mtp_head.parameters())
print(f"🏛️ MyTranslation-300M | GPUs: {world_size}x | Stage: {args.stage} | Inf: {inf_params/1e6:.2f}M | Train: {tot_params/1e6:.2f}M")
label_smoothing = 0.1 if args.stage == "stable" else 0.0
mtp_weight = 0.3 if args.stage == "stable" else 0.15
STYLE_EVAL_PROMPTS = [
("En->My Formal", [TOK_2MY, TOK_FORMAL], "The central bank raised interest rates to control inflation and stabilize the economy."),
("En->My Spoken", [TOK_2MY, TOK_SPOKEN], "Honestly, I didn't expect things to turn out like this, but don't worry—we'll figure it out together."),
("My->En Formal", [TOK_2EN, TOK_FORMAL], "ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးတည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။"),
("My->En Spoken", [TOK_2EN, TOK_SPOKEN], "ဒီနေ့ ရန်ကုန်မှာ မိုးတွေ အရမ်းရွာနေလို့ အိမ်ပြန်ရတာ နည်းနည်းနောက်ကျမယ်နော်။"),
]
@torch.no_grad()
def eval_val_nll():
model.eval()
en2my_nll, en2my_tok = 0.0, 0
my2en_nll, my2en_tok = 0.0, 0
for b in val_batches:
for direction in ("en2my", "my2en"):
srcs, tgts = [], []
for r_id in b:
m, e, _ = val_store.get_pair(int(r_id))
m, e = m[: args.max_seq_len - 4], e[: args.max_seq_len - 4]
if direction == "en2my":
srcs.append(np.concatenate([[TOK_2MY], e, [EOS_ID]]))
tgts.append(np.concatenate([[BOS_ID], m, [EOS_ID]]))
else:
srcs.append(np.concatenate([[TOK_2EN], m, [EOS_ID]]))
tgts.append(np.concatenate([[BOS_ID], e, [EOS_ID]]))
src_ids, tgt_in, labels = collate_prebatched([(srcs, tgts)])
nll, nv = model.eval_nll(src_ids.to(device), tgt_in.to(device), labels.to(device))
if direction == "en2my":
en2my_nll += nll
en2my_tok += nv
else:
my2en_nll += nll
my2en_tok += nv
e2m = en2my_nll / max(1, en2my_tok)
m2e = my2en_nll / max(1, my2en_tok)
comb = (en2my_nll + my2en_nll) / max(1, en2my_tok + my2en_tok)
return comb, e2m, m2e
def run_style_generations(step_num: int):
prompts = [pref + sp.encode_as_ids(txt)[:500] + [EOS_ID] for _, pref, txt in STYLE_EVAL_PROMPTS]
preds = model.translate_kv(prompts, sp, max_new_tokens=96)
print(f" 🗣️ Style Generations @ Step {step_num}:")
for (tag, _, _), out in zip(STYLE_EVAL_PROMPTS, preds):
print(f" [{tag}]: {out[:180]}")
def make_chunk_loader(c_idx: int, skip_steps: int = 0):
s_off = c_idx * args.chunk_rows
e_off = min(len(full_plan), s_off + args.chunk_rows)
row_slice = np.asarray(full_plan[s_off:e_off])
step_group = world_size * args.grad_accum
all_batches = pack_chunk_batches(
row_slice, train_store.index,
max_tokens_combined=args.max_tok_micro,
max_seq_len=args.max_seq_len,
step_group=step_group,
seed=2026 + c_idx * 17,
)
rank_batches = all_batches[rank::world_size]
if skip_steps > 0:
rank_batches = rank_batches[skip_steps * args.grad_accum :]
n_windows = len(rank_batches) // args.grad_accum
rank_batches = rank_batches[: n_windows * args.grad_accum]
ds = DynamicStreamDataset(
train_store, rank_batches, args.stage, args.max_seq_len,
seed=10000 + c_idx * 997 + rank * 131 + skip_steps,
)
loader = DataLoader(
ds, batch_size=1, shuffle=False, collate_fn=collate_prebatched,
num_workers=4, pin_memory=True, worker_init_fn=worker_init_fn, persistent_workers=False,
)
return loader, n_windows
if is_main:
comb, e2m, m2e = eval_val_nll()
print(f"🔎 Step {step:6d} | Val Comb: {comb:.4f} | En->My: {e2m:.4f} | My->En: {m2e:.4f}")
if world_size > 1:
dist.barrier()
model.train()
opt.zero_grad(set_to_none=True)
acc_ce = acc_mtp = acc_z = 0.0
acc_valid_tok = acc_mtp_tok = acc_tot_tok = 0
t0 = time.time()
while chunk_idx < num_chunks:
train_loader, n_windows = make_chunk_loader(chunk_idx, skip_steps=batch_cursor)
train_iter = iter(train_loader)
done_flag = False
for _ in range(n_windows):
window = [next(train_iter) for _ in range(args.grad_accum)]
batch_cursor += 1
loc_valid = sum(int((lb != -100).sum().item()) for _, _, lb in window)
loc_mtp = sum(
int(((lb[:, 1:] != -100) & (lb[:, :-1] != -100)).sum().item()) if lb.size(1) > 1 else 0
for _, _, lb in window
)
loc_rows = sum(s.size(0) for s, _, _ in window)
if world_size > 1:
counts_t = torch.tensor([loc_valid, loc_mtp, loc_rows], dtype=torch.long, device=device)
dist.all_reduce(counts_t, op=dist.ReduceOp.SUM)
glob_valid, glob_mtp, glob_rows = int(counts_t[0].item()), int(counts_t[1].item()), int(counts_t[2].item())
else:
glob_valid, glob_mtp, glob_rows = loc_valid, loc_mtp, loc_rows
g_scale = 1024.0 if world_size > 1 else 1.0
main_denom = float(max(1, glob_valid)) / g_scale
mtp_denom = float(max(1, glob_mtp)) / g_scale
loc_ce = loc_mtp_ce = loc_z = 0.0
loc_tok = sum(int((s != PAD_ID).sum().item()) for s, _, _ in window)
for src_ids, tgt_in, labels in window:
src_ids = src_ids.to(device, non_blocking=True)
tgt_in = tgt_in.to(device, non_blocking=True)
labels = labels.to(device, non_blocking=True)
ce_s, mtp_s, z_s, nv, nm = model.forward_and_backward(
src_ids, tgt_in, labels,
main_denom=main_denom, mtp_denom=mtp_denom,
label_smoothing=label_smoothing, mtp_weight=mtp_weight,
)
loc_ce += ce_s
loc_mtp_ce += mtp_s
loc_z += z_s
loc_tok += nv
allreduce_grads(model, world_size)
rows_seen += glob_rows
if args.stage == "stable":
lr = args.peak_lr * min(1.0, (step + 1) / max(1, args.warmup_steps))
else:
if step < 100:
lr = args.peak_lr * (step + 1) / 100.0
else:
progress = min(1.0, rows_seen / max(1, len(full_plan)))
lr = 1.0e-5 + 0.5 * (args.peak_lr - 1.0e-5) * (1.0 + math.cos(math.pi * progress))
for pg in opt.param_groups:
pg["lr"] = lr
grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
opt.step()
opt.zero_grad(set_to_none=True)
step += 1
if world_size > 1:
stats_t = torch.tensor([loc_ce, loc_mtp_ce, loc_z, float(loc_tok)], dtype=torch.float64, device=device)
dist.all_reduce(stats_t, op=dist.ReduceOp.SUM)
acc_ce += float(stats_t[0].item())
acc_mtp += float(stats_t[1].item())
acc_z += float(stats_t[2].item())
acc_tot_tok += int(stats_t[3].item())
else:
acc_ce += loc_ce
acc_mtp += loc_mtp_ce
acc_z += loc_z
acc_tot_tok += loc_tok
acc_valid_tok += glob_valid
acc_mtp_tok += glob_mtp
log_int = 10 if args.verify_steps > 0 else 50
if step % log_int == 0:
if is_main:
dt = time.time() - t0
tok_s = acc_tot_tok / max(1e-3, dt)
vram_g = torch.cuda.max_memory_allocated(device) / 1e9
pct = 100.0 * min(rows_seen, len(full_plan)) / len(full_plan)
print(
f"Step {step:6d} ({pct:5.1f}% | Chunk {chunk_idx+1}/{num_chunks}) | "
f"Main: {acc_ce/max(1,acc_valid_tok):.4f} | MTP2: {acc_mtp/max(1,acc_mtp_tok):.4f} | "
f"Z: {acc_z/max(1,acc_valid_tok):.2f} | LR: {lr:.2e} | Grad: {grad_norm:.2f} | "
f"{tok_s/1e3:.1f}k tok/s | VRAM: {vram_g:.1f}G"
)
acc_ce = acc_mtp = acc_z = 0.0
acc_valid_tok = acc_mtp_tok = acc_tot_tok = 0
t0 = time.time()
if args.verify_steps > 0 and step >= args.verify_steps:
done_flag = True
break
if step % args.eval_every == 0:
if is_main:
comb, e2m, m2e = eval_val_nll()
print("-" * 96)
print(f"🔎 EVAL @ Step {step:6d} | Comb NLL: {comb:.4f} (PPL {math.exp(min(comb, 30.0)):.2f}) | En->My: {e2m:.4f} | My->En: {m2e:.4f}")
run_style_generations(step)
if comb < best_val_nll:
best_val_nll = comb
best_sd = {k: v.to(torch.bfloat16) for k, v in clean_state_dict(model).items()}
torch.save(best_sd, os.path.join(CKPT_DIR, "best_model.pt"))
print(f" ⭐ New Best Model saved! Comb NLL: {best_val_nll:.4f}")
print("-" * 96)
model.train()
if world_size > 1:
dist.barrier()
t0 = time.time()
if step % args.save_every == 0:
if is_main:
tmp_p = latest_path + ".tmp"
torch.save({
"model": clean_state_dict(model),
"opt": opt.state_dict(),
"step": step,
"stage": args.stage,
"chunk_idx": chunk_idx,
"batch_cursor": batch_cursor,
"rows_seen": rows_seen,
"best_val_nll": best_val_nll,
}, tmp_p)
os.replace(tmp_p, latest_path)
if world_size > 1:
dist.barrier()
if step % args.milestone_every == 0:
if is_main:
ms_sd = {k: v.to(torch.bfloat16) for k, v in clean_state_dict(model).items()}
torch.save(ms_sd, os.path.join(CKPT_DIR, f"{args.stage}_step_{step:06d}.pt"))
if world_size > 1:
dist.barrier()
if done_flag:
break
chunk_idx += 1
batch_cursor = 0
if args.verify_steps > 0:
if is_main:
print(f"✅ {world_size}-GPU VERIFY PASSED ({step} steps)! Ready for launch.")
if world_size > 1:
dist.barrier()
dist.destroy_process_group()
return
if is_main:
comb, e2m, m2e = eval_val_nll()
print("-" * 96)
print(f"🏁 FINAL EVAL @ Step {step:6d} | Comb NLL: {comb:.4f} (PPL {math.exp(min(comb, 30.0)):.2f}) | En->My: {e2m:.4f} | My->En: {m2e:.4f}")
run_style_generations(step)
if comb < best_val_nll:
best_val_nll = comb
best_sd = {k: v.to(torch.bfloat16) for k, v in clean_state_dict(model).items()}
torch.save(best_sd, os.path.join(CKPT_DIR, "best_model.pt"))
tmp_p = latest_path + ".tmp"
torch.save({
"model": clean_state_dict(model),
"opt": opt.state_dict(),
"step": step,
"stage": args.stage,
"chunk_idx": chunk_idx,
"batch_cursor": batch_cursor,
"rows_seen": rows_seen,
"best_val_nll": best_val_nll,
}, tmp_p)
os.replace(tmp_p, latest_path)
with torch.no_grad():
rw = F.softmax(model.enc_router_logits, dim=-1).cpu().numpy()
for p in range(model.n_cross_pairs):
top_layers = np.argsort(rw[p])[::-1][:4].tolist()
print(f" • Cross-KV Pair {p} top encoder mixture weights: {top_layers} ({[round(float(rw[p][k]), 3) for k in top_layers]})")
print(f"🎉 Stage '{args.stage}' 100% COMPLETE ({rows_seen:,} rows trained across {step:,} steps)!")
if world_size > 1:
dist.barrier()
dist.destroy_process_group()
if __name__ == "__main__":
main()
import os
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
import argparse, json, math, time
from datetime import timedelta
import numpy as np
import sentencepiece as spm
import torch
import torch.distributed as dist
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.checkpoint import checkpoint as grad_ckpt
from torch.utils.data import Dataset, DataLoader
torch.backends.cuda.matmul.allow_tf32 = True
torch.backends.cudnn.allow_tf32 = True
torch._dynamo.config.cache_size_limit = 64
DATA_DIR = "/workspace/mytrans_dl/tokenized_65k"
CKPT_DIR = "/workspace/ckpt_300m"
VOCAB_SIZE = 65536
UNK_ID, BOS_ID, EOS_ID, PAD_ID = 0, 1, 2, 3
TOK_2MY, TOK_2EN, TOK_FORMAL, TOK_SPOKEN, TOK_BT, TOK_DENOISE, TOK_MASK = 4, 5, 6, 7, 8, 9, 10
FLAG_BT, FLAG_FORMAL, FLAG_SPOKEN = 1, 2, 4
BULK_SUBSETS = {
"MyanmarTranslator_works/myanmar_spoken_corpus",
"MyanmarTranslator_works/myanmar-cc100-dataset",
"MyanmarTranslator_works/myanmar-written-corpus",
}
def load_and_verify_contracts(data_dir: str):
with open(os.path.join(data_dir, "meta.json"), "r") as f:
meta = json.load(f)
sp = spm.SentencePieceProcessor()
sp.load(os.path.join(data_dir, "tokenizer.model"))
assert sp.get_piece_size() == VOCAB_SIZE
assert sp.unk_id() == UNK_ID and sp.bos_id() == BOS_ID and sp.eos_id() == EOS_ID and sp.pad_id() == PAD_ID
expected = {4: "<2my>", 5: "<2en>", 6: "<formal>", 7: "<spoken>", 8: "<bt>", 9: "<denoise>", 10: "<mask>"}
for tid, piece in expected.items():
assert sp.id_to_piece(tid) == piece, f"Token {tid} mismatch"
if "special_tokens" in meta:
st = meta["special_tokens"]
assert st["2my"] == TOK_2MY and st["2en"] == TOK_2EN
assert st["formal"] == TOK_FORMAL and st["spoken"] == TOK_SPOKEN
assert st["bt"] == TOK_BT and st["denoise"] == TOK_DENOISE and st["mask"] == TOK_MASK
return meta, sp
# ==============================================================================
# 1. MyTranslation-300M Architecture
# ==============================================================================
class RMSNorm(nn.Module):
def __init__(self, dim: int, eps: float = 1e-6):
super().__init__()
self.dim = (dim,)
self.eps = eps
self.weight = nn.Parameter(torch.ones(dim))
def forward(self, x: torch.Tensor) -> torch.Tensor:
return F.rms_norm(x, self.dim, self.weight, self.eps)
class RotaryEmbedding(nn.Module):
def __init__(self, dim: int, max_seq_len: int = 2048, theta: float = 50000.0):
super().__init__()
inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
t = torch.arange(max_seq_len, dtype=torch.float32)
freqs = torch.outer(t, inv_freq)
self.register_buffer("cos_cached", freqs.cos()[None, None, :, :], persistent=False)
self.register_buffer("sin_cached", freqs.sin()[None, None, :, :], persistent=False)
def forward(self, x: torch.Tensor) -> torch.Tensor:
L = x.size(2)
cos = self.cos_cached[:, :, :L, :].to(dtype=x.dtype)
sin = self.sin_cached[:, :, :L, :].to(dtype=x.dtype)
x1, x2 = x[..., ::2], x[..., 1::2]
return torch.cat((x1 * cos - x2 * sin, x1 * sin + x2 * cos), dim=-1)
class SwiGLU(nn.Module):
def __init__(self, d_model: int, ffn_dim: int):
super().__init__()
self.gate_up = nn.Linear(d_model, 2 * ffn_dim, bias=False)
self.down = nn.Linear(ffn_dim, d_model, bias=False)
def forward(self, x: torch.Tensor) -> torch.Tensor:
gate, up = self.gate_up(x).chunk(2, dim=-1)
return self.down(F.silu(gate) * up)
class GatedGQASelfAttention(nn.Module):
def __init__(self, d_model: int, n_heads: int, n_kv_heads: int, is_causal: bool = False, has_v_res: bool = False):
super().__init__()
self.n_heads = n_heads
self.n_kv_heads = n_kv_heads
self.head_dim = d_model // n_heads