Translation
Transformers
Safetensors
Burmese
English
myanmartranslator
feature-extraction
Translation
Safetensors
Burmese
English
MyanmarLLM
flagship
custom_code
myanmar
Instructions to use freococo/MyanmarTranslator-210M-Pro with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use freococo/MyanmarTranslator-210M-Pro with Transformers:
# Use a pipeline as a high-level helper # Warning: Pipeline type "translation" is no longer supported in transformers v5. # You must load the model directly (see below) or downgrade to v4.x with: # pip install "transformers<5.0.0" from transformers import pipeline pipe = pipeline("translation", model="freococo/MyanmarTranslator-210M-Pro", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("freococo/MyanmarTranslator-210M-Pro", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download training_codes.txt from freococo/MyanmarTranslator-210M-Pro: direct link, hf CLI and curl.
- Browser
- Download file 44.5 kB
-
https://huggingface.co/freococo/MyanmarTranslator-210M-Pro/resolve/main/training_codes.txt
- Command line
-
hf download hf://freococo/MyanmarTranslator-210M-Pro/training_codes.txt
-
curl -L -o training_codes.txt https://huggingface.co/freococo/MyanmarTranslator-210M-Pro/resolve/main/training_codes.txt
44.5 kB
| # ============================================================================== | |
| # 🏛️ CELL 11 (100% STANDALONE): Flagship MyanmarTranslator-210M-Pro | |
| # (8L Enc + 8L Dec, d=768, 210.90M Params) + Net2Wider/Deeper + Auto-Upload | |
| # ============================================================================== | |
| import os | |
| os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True" | |
| import sys, gc, math, time, json, shutil, traceback | |
| import numpy as np | |
| import sentencepiece as spm | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from torch.utils.checkpoint import checkpoint as grad_ckpt | |
| from torch.utils.data import Dataset, DataLoader, Sampler | |
| from safetensors.torch import save_file | |
| from huggingface_hub import HfApi, create_repo | |
| torch.backends.cuda.matmul.allow_tf32 = True | |
| torch.backends.cudnn.allow_tf32 = True | |
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| # ── 0. Capture Exact Cell Code & Setup Incremental Logger ── | |
| try: | |
| CELL_SOURCE_CODE = get_ipython().user_ns["In"][-1] | |
| except Exception: | |
| CELL_SOURCE_CODE = "# MyanmarTranslator-210M-Pro Standalone Training Script" | |
| PRO_REPO_ID = "freococo/MyanmarTranslator-210M-Pro" | |
| WORK_DIR = "/workspace" | |
| DATA_DIR = "/workspace/nmt_dataset_memmap" | |
| TOK_DIR = "/workspace/nmt_tokenizer" | |
| TOK_FILE = os.path.join(TOK_DIR, "tokenizer.model") | |
| TEACHER_84M = "/workspace/nmt_84m_polished/best_model.pt" | |
| PRO_CKPT = "/workspace/pro_210m_checkpoints" | |
| PRO_EXPORT = "/workspace/MyanmarTranslator-210M-Pro-export" | |
| os.makedirs(PRO_CKPT, exist_ok=True) | |
| if not os.path.exists(os.path.join(PRO_CKPT, "latest_pro.pt")): | |
| with open(os.path.join(PRO_CKPT, "trainingcelloutputwitheval.txt"), "w", encoding="utf-8") as lf: | |
| lf.write("") | |
| with open(os.path.join(PRO_CKPT, "training_codes.txt"), "w", encoding="utf-8") as f: | |
| f.write(CELL_SOURCE_CODE) | |
| LOG_LINES = [] | |
| def log(msg: str = ""): | |
| print(msg, flush=True) | |
| LOG_LINES.append(str(msg)) | |
| with open(os.path.join(PRO_CKPT, "trainingcelloutputwitheval.txt"), "a", encoding="utf-8") as lf: | |
| lf.write(str(msg) + "\n") | |
| gc.collect() | |
| if torch.cuda.is_available(): | |
| torch.cuda.empty_cache() | |
| free_vram, tot_vram = torch.cuda.mem_get_info() | |
| _, _, free_disk = shutil.disk_usage(WORK_DIR) | |
| log(f"🟢 Standalone Kernel Ready for 210M-Pro | GPU Free: {free_vram/1e9:.2f}/{tot_vram/1e9:.2f} GB | Disk Free: {free_disk/1e9:.1f} GB") | |
| sp = spm.SentencePieceProcessor() | |
| sp.load(TOK_FILE) | |
| VOCAB_SIZE, UNK_ID, BOS_ID, EOS_ID, PAD_ID = 65664, 0, 1, 2, 3 | |
| TOK_2MY, TOK_2EN, TOK_DENOISE, TOK_MASK = 65536, 65537, 65538, 65539 | |
| # ── 1. Standalone Architecture Definitions ── | |
| class RMSNorm(nn.Module): | |
| def __init__(self, dim: int, eps: float = 1e-6): | |
| super().__init__() | |
| self.dim = (dim,) | |
| self.eps = eps | |
| self.weight = nn.Parameter(torch.ones(dim)) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| return F.rms_norm(x, self.dim, self.weight, self.eps) | |
| class RotaryEmbedding(nn.Module): | |
| def __init__(self, dim: int, max_seq_len: int = 1024, theta: float = 10000.0): | |
| super().__init__() | |
| inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) | |
| t = torch.arange(max_seq_len, dtype=torch.float32) | |
| freqs = torch.outer(t, inv_freq) | |
| self.register_buffer("cos_cached", freqs.cos()[None, None, :, :], persistent=False) | |
| self.register_buffer("sin_cached", freqs.sin()[None, None, :, :], persistent=False) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| L = x.size(2) | |
| cos = self.cos_cached[:, :, :L, :].to(dtype=x.dtype) | |
| sin = self.sin_cached[:, :, :L, :].to(dtype=x.dtype) | |
| x1, x2 = x[..., ::2], x[..., 1::2] | |
| return torch.cat([x1 * cos - x2 * sin, x1 * sin + x2 * cos], dim=-1) | |
| class SwiGLU(nn.Module): | |
| def __init__(self, d_model: int, ffn_dim: int): | |
| super().__init__() | |
| self.gate_up = nn.Linear(d_model, 2 * ffn_dim, bias=False) | |
| self.down = nn.Linear(ffn_dim, d_model, bias=False) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| gate, up = self.gate_up(x).chunk(2, dim=-1) | |
| return self.down(F.silu(gate) * up) | |
| class SelfAttention(nn.Module): | |
| def __init__(self, d_model: int, n_heads: int, is_causal: bool = False): | |
| super().__init__() | |
| self.n_heads = n_heads | |
| self.head_dim = d_model // n_heads | |
| self.is_causal = is_causal | |
| self.qkv = nn.Linear(d_model, 3 * d_model, bias=False) | |
| self.out = nn.Linear(d_model, d_model, bias=False) | |
| self.q_norm = RMSNorm(self.head_dim) | |
| self.k_norm = RMSNorm(self.head_dim) | |
| def forward(self, x: torch.Tensor, rope: RotaryEmbedding, attn_mask: torch.Tensor = None) -> torch.Tensor: | |
| B, L, D = x.shape | |
| q, k, v = self.qkv(x).chunk(3, dim=-1) | |
| q = self.q_norm(q.view(B, L, self.n_heads, self.head_dim).transpose(1, 2)) | |
| k = self.k_norm(k.view(B, L, self.n_heads, self.head_dim).transpose(1, 2)) | |
| v = v.view(B, L, self.n_heads, self.head_dim).transpose(1, 2) | |
| q, k = rope(q), rope(k) | |
| y = F.scaled_dot_product_attention(q, k, v, attn_mask=None if self.is_causal else attn_mask, is_causal=self.is_causal) | |
| return self.out(y.transpose(1, 2).contiguous().view(B, L, D)) | |
| class CrossAttention(nn.Module): | |
| def __init__(self, d_model: int, n_heads: int): | |
| super().__init__() | |
| self.n_heads = n_heads | |
| self.head_dim = d_model // n_heads | |
| self.q_proj = nn.Linear(d_model, d_model, bias=False) | |
| self.kv_proj = nn.Linear(d_model, 2 * d_model, bias=False) | |
| self.out = nn.Linear(d_model, d_model, bias=False) | |
| self.q_norm = RMSNorm(self.head_dim) | |
| self.k_norm = RMSNorm(self.head_dim) | |
| def forward(self, x: torch.Tensor, enc_out: torch.Tensor, enc_mask: torch.Tensor) -> torch.Tensor: | |
| B, T, D = x.shape | |
| S = enc_out.size(1) | |
| q = self.q_norm(self.q_proj(x).view(B, T, self.n_heads, self.head_dim).transpose(1, 2)) | |
| k, v = self.kv_proj(enc_out).chunk(2, dim=-1) | |
| k = self.k_norm(k.view(B, S, self.n_heads, self.head_dim).transpose(1, 2)) | |
| v = v.view(B, S, self.n_heads, self.head_dim).transpose(1, 2) | |
| y = F.scaled_dot_product_attention(q, k, v, attn_mask=enc_mask, is_causal=False) | |
| return self.out(y.transpose(1, 2).contiguous().view(B, T, D)) | |
| class EncoderLayer(nn.Module): | |
| def __init__(self, d_model: int, n_heads: int, ffn_dim: int): | |
| super().__init__() | |
| self.norm1 = RMSNorm(d_model) | |
| self.self_attn = SelfAttention(d_model, n_heads, is_causal=False) | |
| self.norm2 = RMSNorm(d_model) | |
| self.mlp = SwiGLU(d_model, ffn_dim) | |
| def forward(self, x: torch.Tensor, rope: RotaryEmbedding, enc_mask: torch.Tensor) -> torch.Tensor: | |
| x = x + self.self_attn(self.norm1(x), rope, enc_mask) | |
| x = x + self.mlp(self.norm2(x)) | |
| return x | |
| class DecoderLayer(nn.Module): | |
| def __init__(self, d_model: int, n_heads: int, ffn_dim: int): | |
| super().__init__() | |
| self.norm1 = RMSNorm(d_model) | |
| self.self_attn = SelfAttention(d_model, n_heads, is_causal=True) | |
| self.norm2 = RMSNorm(d_model) | |
| self.cross_attn = CrossAttention(d_model, n_heads) | |
| self.norm3 = RMSNorm(d_model) | |
| self.mlp = SwiGLU(d_model, ffn_dim) | |
| def forward(self, x: torch.Tensor, enc_out: torch.Tensor, rope: RotaryEmbedding, enc_mask: torch.Tensor) -> torch.Tensor: | |
| x = x + self.self_attn(self.norm1(x), rope, None) | |
| x = x + self.cross_attn(self.norm2(x), enc_out, enc_mask) | |
| x = x + self.mlp(self.norm3(x)) | |
| return x | |
| class MyanmarTranslator84M(nn.Module): | |
| def __init__(self, vocab_size=65664, d_model=768, n_heads=12, enc_layers=8, dec_layers=8, ffn_dim=2816, max_seq_len=1024): | |
| super().__init__() | |
| self.config = dict( | |
| vocab_size=vocab_size, d_model=d_model, n_heads=n_heads, | |
| enc_layers=enc_layers, dec_layers=dec_layers, ffn_dim=ffn_dim, max_seq_len=max_seq_len | |
| ) | |
| self.d_model = d_model | |
| self.embed_scale = math.sqrt(d_model) | |
| self.embed_tokens = nn.Embedding(vocab_size, d_model, padding_idx=PAD_ID) | |
| self.rope = RotaryEmbedding(d_model // n_heads, max_seq_len=max_seq_len) | |
| self.encoder = nn.ModuleList([EncoderLayer(d_model, n_heads, ffn_dim) for _ in range(enc_layers)]) | |
| self.enc_norm = RMSNorm(d_model) | |
| self.decoder = nn.ModuleList([DecoderLayer(d_model, n_heads, ffn_dim) for _ in range(dec_layers)]) | |
| self.dec_norm = RMSNorm(d_model) | |
| self.lm_head = nn.Linear(d_model, vocab_size, bias=False) | |
| self.lm_head.weight = self.embed_tokens.weight | |
| self._init_weights(enc_layers + dec_layers) | |
| def _init_weights(self, total_layers: int): | |
| res_std = 0.02 / math.sqrt(2 * total_layers) | |
| for name, m in self.named_modules(): | |
| if isinstance(m, nn.Linear): | |
| nn.init.normal_(m.weight, mean=0.0, std=res_std if (name.endswith(".out") or name.endswith(".down")) else 0.02) | |
| elif isinstance(m, nn.Embedding): | |
| nn.init.normal_(m.weight, mean=0.0, std=0.02) | |
| if m.padding_idx is not None: | |
| m.weight.data[m.padding_idx].zero_() | |
| def encode(self, src_ids: torch.Tensor, use_ckpt: bool = False): | |
| enc_mask = (src_ids != PAD_ID).unsqueeze(1).unsqueeze(2) | |
| x = self.embed_tokens(src_ids) * self.embed_scale | |
| for layer in self.encoder: | |
| x = grad_ckpt(layer, x, self.rope, enc_mask, use_reentrant=False) if (use_ckpt and self.training) else layer(x, self.rope, enc_mask) | |
| return self.enc_norm(x), enc_mask | |
| def decode(self, tgt_in: torch.Tensor, enc_out: torch.Tensor, enc_mask: torch.Tensor, use_ckpt: bool = False): | |
| x = self.embed_tokens(tgt_in) * self.embed_scale | |
| for layer in self.decoder: | |
| x = grad_ckpt(layer, x, enc_out, self.rope, enc_mask, use_reentrant=False) if (use_ckpt and self.training) else layer(x, enc_out, self.rope, enc_mask) | |
| return self.dec_norm(x) | |
| def forward_and_backward(self, src_ids: torch.Tensor, tgt_in: torch.Tensor, labels: torch.Tensor, loss_scale: float = 1.0, label_smoothing: float = 0.1): | |
| use_ckpt = max(src_ids.size(1), tgt_in.size(1)) > 192 | |
| with torch.amp.autocast("cuda", dtype=torch.bfloat16): | |
| enc_out, enc_mask = self.encode(src_ids, use_ckpt=use_ckpt) | |
| dec_out = self.decode(tgt_in, enc_out, enc_mask, use_ckpt=use_ckpt) | |
| valid_mask = labels.ne(-100) | |
| valid_h = dec_out[valid_mask] | |
| valid_y = labels[valid_mask] | |
| n_valid = valid_y.numel() | |
| h_det = valid_h.detach().requires_grad_(True) | |
| denom = float(n_valid) * float(loss_scale) | |
| tot_loss = 0.0 | |
| chunk_sz = 2048 | |
| for i in range(0, n_valid, chunk_sz): | |
| hc = h_det[i : i + chunk_sz] | |
| yc = valid_y[i : i + chunk_sz] | |
| with torch.amp.autocast("cuda", dtype=torch.bfloat16): | |
| logits_c = self.lm_head(hc) | |
| lc = F.cross_entropy(logits_c, yc, label_smoothing=label_smoothing, reduction="sum") | |
| (lc / denom).backward() | |
| tot_loss += float(lc.detach().item()) | |
| del logits_c, lc | |
| valid_h.backward(h_det.grad) | |
| return tot_loss / max(n_valid, 1), n_valid | |
| @torch.no_grad() | |
| def eval_nll(self, src_ids: torch.Tensor, tgt_in: torch.Tensor, labels: torch.Tensor): | |
| with torch.amp.autocast("cuda", dtype=torch.bfloat16): | |
| enc_out, enc_mask = self.encode(src_ids, use_ckpt=False) | |
| dec_out = self.decode(tgt_in, enc_out, enc_mask, use_ckpt=False) | |
| valid_mask = labels.ne(-100) | |
| valid_h = dec_out[valid_mask] | |
| valid_y = labels[valid_mask] | |
| n_valid = valid_y.numel() | |
| tot_nll = 0.0 | |
| for i in range(0, n_valid, 2048): | |
| logits_c = self.lm_head(valid_h[i : i + 2048]) | |
| tot_nll += float(F.cross_entropy(logits_c, valid_y[i : i + 2048], reduction="sum").item()) | |
| return tot_nll, n_valid | |
| @torch.no_grad() | |
| def translate(self, text: str, direction: str = "en2my", max_new_tokens: int = 192, rep_penalty: float = 1.15) -> str: | |
| self.eval() | |
| dir_tok = TOK_2MY if direction == "en2my" else TOK_2EN | |
| s_ids = [dir_tok] + sp.encode_as_ids(text.strip())[:1020] + [EOS_ID] | |
| src_t = torch.tensor([s_ids], dtype=torch.long, device=next(self.parameters()).device) | |
| with torch.amp.autocast("cuda", dtype=torch.bfloat16): | |
| enc_out, enc_mask = self.encode(src_t, use_ckpt=False) | |
| ys = [BOS_ID] | |
| for _ in range(max_new_tokens): | |
| tgt_t = torch.tensor([ys], dtype=torch.long, device=src_t.device) | |
| dec_out = self.decode(tgt_t, enc_out, enc_mask, use_ckpt=False) | |
| logits = self.lm_head(dec_out[:, -1, :]).float()[0] | |
| logits[[UNK_ID, PAD_ID, BOS_ID]] = -1e9 | |
| logits[65536:] = -1e9 | |
| if rep_penalty != 1.0 and len(ys) > 1: | |
| seen = torch.tensor(list(set(ys[1:])), dtype=torch.long, device=src_t.device) | |
| s_logits = logits[seen] | |
| logits[seen] = torch.where(s_logits > 0, s_logits / rep_penalty, s_logits * rep_penalty) | |
| nxt = int(logits.argmax().item()) | |
| if nxt == EOS_ID: | |
| break | |
| ys.append(nxt) | |
| return sp.decode_ids([x for x in ys[1:] if x < 65536]) | |
| @torch.no_grad() | |
| def translate_beam(self, text: str, direction: str = "en2my", num_beams: int = 4, max_new_tokens: int = 192, length_penalty: float = 0.85, rep_penalty: float = 1.12) -> str: | |
| self.eval() | |
| if num_beams <= 1: | |
| return self.translate(text, direction=direction, max_new_tokens=max_new_tokens, rep_penalty=rep_penalty) | |
| dir_tok = TOK_2MY if direction == "en2my" else TOK_2EN | |
| s_ids = [dir_tok] + sp.encode_as_ids(text.strip())[:1020] + [EOS_ID] | |
| src_t = torch.tensor([s_ids], dtype=torch.long, device=next(self.parameters()).device) | |
| with torch.amp.autocast("cuda", dtype=torch.bfloat16): | |
| enc_out, enc_mask = self.encode(src_t, use_ckpt=False) | |
| beams = [(0.0, [BOS_ID], False)] | |
| for _ in range(max_new_tokens): | |
| active = [(sc, seq) for sc, seq, done in beams if not done] | |
| if not active: | |
| break | |
| B_act = len(active) | |
| tgt_t = torch.tensor([seq for _, seq in active], dtype=torch.long, device=src_t.device) | |
| dec_out = self.decode(tgt_t, enc_out.expand(B_act, -1, -1), enc_mask.expand(B_act, -1, -1, -1), use_ckpt=False) | |
| logits = self.lm_head(dec_out[:, -1, :]).float() | |
| logits[:, [UNK_ID, PAD_ID, BOS_ID]] = -1e9 | |
| logits[:, 65536:] = -1e9 | |
| for b_i, (_, seq) in enumerate(active): | |
| if rep_penalty != 1.0 and len(seq) > 1: | |
| seen = torch.tensor(list(set(seq[1:])), dtype=torch.long, device=src_t.device) | |
| s_l = logits[b_i, seen] | |
| logits[b_i, seen] = torch.where(s_l > 0, s_l / rep_penalty, s_l * rep_penalty) | |
| log_probs = F.log_softmax(logits, dim=-1) | |
| topk_lp, topk_ids = torch.topk(log_probs, num_beams * 2, dim=-1) | |
| candidates = [(sc, seq, True) for sc, seq, done in beams if done] | |
| for b_i, (sc, seq) in enumerate(active): | |
| for k in range(num_beams * 2): | |
| tok_id = int(topk_ids[b_i, k].item()) | |
| new_sc = sc + float(topk_lp[b_i, k].item()) | |
| candidates.append((new_sc, seq if tok_id == EOS_ID else seq + [tok_id], tok_id == EOS_ID)) | |
| candidates.sort(key=lambda x: x[0] / (max(1, len(x[1]) - 1) ** length_penalty), reverse=True) | |
| beams = candidates[:num_beams] | |
| if all(done for _, _, done in beams): | |
| break | |
| return sp.decode_ids([x for x in beams[0][1][1:] if x < 65536]) | |
| # ── 2. Standalone Dataset & Sampler Definitions ── | |
| class Seq2SeqMemmapDataset(Dataset): | |
| def __init__(self, data_dir: str, split: str = "train"): | |
| self.src_path = os.path.join(data_dir, f"{split}_src.bin") | |
| self.tgt_path = os.path.join(data_dir, f"{split}_tgt.bin") | |
| self.index = np.load(os.path.join(data_dir, f"{split}_index.npy")) | |
| self.src_mm = None | |
| self.tgt_mm = None | |
| def _ensure_open(self): | |
| if self.src_mm is None: | |
| self.src_mm = np.memmap(self.src_path, dtype=np.uint16, mode="r") | |
| self.tgt_mm = np.memmap(self.tgt_path, dtype=np.uint16, mode="r") | |
| def __len__(self): | |
| return len(self.index) | |
| def __getitem__(self, idx: int): | |
| self._ensure_open() | |
| s0, sl, t0, tl, kind = self.index[idx] | |
| s_raw = self.src_mm[s0 : s0 + sl].astype(np.int64) | |
| if kind == 2: | |
| s_raw[s_raw == 0] = TOK_MASK | |
| src = np.empty(sl + 1, dtype=np.int64) | |
| src[0] = 65536 + kind | |
| src[1:] = s_raw | |
| tgt = self.tgt_mm[t0 : t0 + tl].astype(np.int64) | |
| return src, tgt | |
| class LengthBucketBatchSampler(Sampler): | |
| def __init__(self, index_arr: np.ndarray, batch_size: int, mega_mult: int = 256, shuffle: bool = True, seed: int = 42, start_batch: int = 0): | |
| self.lengths = (index_arr[:, 1] + index_arr[:, 3]).astype(np.int32) | |
| self.batch_size = batch_size | |
| self.mega_size = batch_size * mega_mult | |
| self.shuffle = shuffle | |
| self.seed = seed | |
| self.start_batch = start_batch | |
| self.num_batches = len(self.lengths) // batch_size | |
| def __len__(self): | |
| return max(0, self.num_batches - self.start_batch) | |
| def __iter__(self): | |
| rng = np.random.RandomState(self.seed) | |
| n = len(self.lengths) | |
| order = rng.permutation(n) if self.shuffle else np.arange(n) | |
| batches = [] | |
| for i in range(0, n - self.batch_size + 1, self.mega_size): | |
| chunk = order[i : i + self.mega_size] | |
| chunk = chunk[np.argsort(self.lengths[chunk], kind="mergesort")] | |
| for j in range(0, len(chunk) - self.batch_size + 1, self.batch_size): | |
| batches.append(chunk[j : j + self.batch_size]) | |
| if self.shuffle: | |
| rng.shuffle(batches) | |
| for b in batches[self.start_batch:]: | |
| yield b.tolist() | |
| def collate_nmt(batch): | |
| B = len(batch) | |
| max_s = min(1024, ((max(len(x[0]) for x in batch) + 15) // 16) * 16) | |
| max_t = min(1024, ((max(len(x[1]) - 1 for x in batch) + 15) // 16) * 16) | |
| src_ids = torch.full((B, max_s), PAD_ID, dtype=torch.long) | |
| tgt_in = torch.full((B, max_t), PAD_ID, dtype=torch.long) | |
| labels = torch.full((B, max_t), -100, dtype=torch.long) | |
| for i, (s, t) in enumerate(batch): | |
| sl = min(len(s), max_s) | |
| tl = min(len(t) - 1, max_t) | |
| src_ids[i, :sl] = torch.from_numpy(s[:sl]) | |
| tgt_in[i, :tl] = torch.from_numpy(t[:tl]) | |
| labels[i, :tl] = torch.from_numpy(t[1 : tl + 1]) | |
| return src_ids, tgt_in, labels | |
| AUDIT_SAMPLES = [ | |
| ("En->My Formal", "en2my", "The central bank raised interest rates to control inflation and stabilize the economy."), | |
| ("En->My Spoken", "en2my", "Honestly, I didn't expect things to turn out like this, but don't worry—we'll figure it out together."), | |
| ("My->En Formal", "my2en", "ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးတည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။"), | |
| ("My->En Spoken", "my2en", "အချစ်ရေ မင်းနဲ့ ငါနဲ့ သိပ်ချစ်ခဲ့တာ ဘုရားလည်း သိတယ်၊ သကြားလည်း သိတယ်၊ နတ်လည်း သိတယ်၊ မနာလိုသူတွေ ဂျင်းထည့်တာ ယုံနေတော့ မင်း ရူးသွားပြီပဲ။"), | |
| ] | |
| UNSEEN_TESTS = [ | |
| ("En->My News", "en2my", "Heavy monsoon rains caused severe flooding in several townships, forcing thousands of residents to evacuate to temporary shelters."), | |
| ("En->My Health", "en2my", "Doctors recommend drinking plenty of clean water and getting enough rest to recover quickly from seasonal flu."), | |
| ("En->My Spoken", "en2my", "I was stuck in traffic for over an hour this morning, so please start the meeting without me."), | |
| ("My->En News", "my2en", "နိုင်ငံတကာ ကုန်သွယ်ရေး တိုးတက်လာစေရန်အတွက် နယ်စပ်ကုန်သွယ်ရေးစခန်းများတွင် ကုန်ပစ္စည်းစစ်ဆေးမှုများကို ပိုမိုမြန်ဆန်အောင် ဆောင်ရွက်နေပါသည်။"), | |
| ("My->En Tech", "my2en", "ဉာဏ်ရည်တုနည်းပညာသည် ပညာရေး၊ ကျန်းမာရေးနှင့် စီးပွားရေးလုပ်ငန်းများတွင် လုပ်ငန်းစဉ်များကို ပိုမိုထိရောက်စေရန် ကူညီပေးနိုင်သည်။"), | |
| ("My->En Spoken", "my2en", "မနက်ဖြန် မနက်စောစော သွားစရာရှိလို့ ဒီညတော့ စောစောအိပ်တော့မယ်နော်၊ မနက်ကျမှ ဖုန်းဆက်လိုက်မယ်။"), | |
| ] | |
| SPOKEN_IDIOM_PAIRS_PRO = [ | |
| ("my2en", "အချစ်ရေ မင်းနဲ့ ငါနဲ့ သိပ်ချစ်ခဲ့တာ ဘုရားလည်း သိတယ်၊ သကြားလည်း သိတယ်၊ နတ်လည်း သိတယ်၊ မနာလိုသူတွေ ဂျင်းထည့်တာ ယုံနေတော့ မင်း ရူးသွားပြီပဲ။", | |
| "My love, heaven and earth know how deeply you and I loved each other, yet because you believed the lies fed by jealous people, you've lost your mind."), | |
| ("my2en", "အချစ်ရေ မင်းနဲ့ ငါနဲ့ သိပ်ချစ်ခဲ့တာ ဘုရားလည်း သိတယ်၊ သိကြားလည်း သိတယ်၊ နတ်လည်း သိတယ်၊ မနာလိုသူတွေ ဂျင်းထည့်တာ ယုံနေတော့ မင်း ယူသွားခဲ့တာပဲ။", | |
| "My love, Buddha, Sakra, and all the spirits know how deeply you and I loved each other, but because you believed the lies stirred up by jealous people, you ended up marrying someone else."), | |
| ("my2en", "သူများတွေ ဂျင်းထည့်တာကို လွယ်လွယ်မယုံပါနဲ့၊ ကိုယ်တိုင်သေချာစုံစမ်းပြီးမှ ဆုံးဖြတ်ပါ။", | |
| "Don't easily believe it when people try to trick you or feed you lies; look into it carefully yourself before deciding."), | |
| ("my2en", "ငါ မင်းအပေါ် ဘယ်လောက်သစ္စာရှိခဲ့လဲဆိုတာ ဘုရားသိ သိကြားသိပါကွာ။", | |
| "Heaven and earth know just how loyal I have been to you."), | |
| ("my2en", "မနာလိုတဲ့လူတွေ ဘေးကနေ သွေးထိုးပြီး ဂျင်းထည့်နေတာကို မင်း ဘာလို့ နားယောင်နေရတာလဲ။", | |
| "Why are you falling for the lies and manipulation of jealous people trying to stir up trouble from the sidelines?"), | |
| ("my2en", "စိတ်မပူပါနဲ့ သူငယ်ချင်းရာ၊ အခက်အခဲဆိုတာ ခဏပဲ၊ ငါတို့ အတူတူ ရင်ဆိုင်ကြတာပေါ့။", | |
| "Don't worry, my friend—hardships are only temporary, and we'll face this together."), | |
| ("my2en", "နေကောင်းရဲ့လား မိတ်ဆွေ၊ မတွေ့ရတာတောင် တော်တော်ကြာပြီနော်။", | |
| "How have you been doing, my friend? It's been quite a long time since we last met."), | |
| ("en2my", "Honestly, I didn't expect things to turn out like this, but don't worry—we'll figure it out together.", | |
| "ရိုးရိုးသားသားပြောရရင် ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ထားခဲ့ပေမယ့် စိတ်မပူပါနဲ့၊ ကျွန်တော်တို့ အတူတူ အဖြေရှာကြတာပေါ့။"), | |
| ("en2my", "Don't let jealous people trick you or feed you lies; trust what we've built together.", | |
| "မနာလိုသူတွေ ဂျင်းထည့်တာကို မယုံမိပါစေနဲ့၊ ငါတို့နှစ်ယောက် အတူတူ တည်ဆောက်ခဲ့တဲ့ အရာကိုပဲ ယုံကြည်ပေးပါ။"), | |
| ("en2my", "The central bank raised interest rates to control inflation and stabilize the economy.", | |
| "ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးတည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။"), | |
| ("my2en", "ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးတည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။", | |
| "The central bank raised interest rates to control inflation and stabilize the economy."), | |
| ("en2my", "I was stuck in traffic for over an hour this morning, so please start the meeting without me.", | |
| "ဒီမနက် တစ်နာရီကျော်လောက် ကားပိတ်မိနေလို့ ကျွန်တော်မပါဘဲ အစည်းအဝေးကို စထားလိုက်ပါနော်။"), | |
| ("en2my", "Please go ahead and have dinner without me; I'll be home a bit late tonight.", | |
| "ဒီည အိမ်ပြန်နည်းနည်းနောက်ကျမှာမို့လို့ ကျွန်တော်မပါဘဲ ညစာ အရင်စားနှင့်ကြပါနော်။"), | |
| ] | |
| # ── 3. Initialize 210M-Pro & Run All-Layer Net2Wider+Net2Deeper Transfer ── | |
| pro_model = MyanmarTranslator84M( | |
| vocab_size=VOCAB_SIZE, d_model=768, n_heads=12, enc_layers=8, dec_layers=8, ffn_dim=2816, max_seq_len=1024 | |
| ).to(device) | |
| n_pro_params = sum(p.numel() for p in pro_model.parameters()) | |
| log(f"🏛️ MyanmarTranslator-210M-Pro Initialized: {n_pro_params:,} parameters ({n_pro_params/1e6:.2f}M) | 8L Encoder + 8L Decoder (d=768)") | |
| def transfer_84m_to_210m(student: nn.Module, teacher_ckpt_path: str): | |
| if not os.path.exists(teacher_ckpt_path): | |
| log("⚠️ Teacher checkpoint not found, using standard initialization.") | |
| return | |
| t_sd = torch.load(teacher_ckpt_path, map_location=device)["model"] | |
| scale_d = math.sqrt(512.0 / 768.0) | |
| with torch.no_grad(): | |
| student.embed_tokens.weight.data.normal_(0.0, 0.004) | |
| student.embed_tokens.weight.data[:, :512].copy_(t_sd["embed_tokens.weight"] * scale_d) | |
| student.embed_tokens.weight.data[PAD_ID].zero_() | |
| def copy_rmsnorm(s_norm, prefix): | |
| s_norm.weight.data.fill_(1.0) | |
| s_norm.weight.data[:512].copy_(t_sd[f"{prefix}.weight"] * scale_d) | |
| def copy_self_attn(s_attn, prefix): | |
| s_attn.qkv.weight.data.normal_(0.0, 0.008) | |
| s_attn.out.weight.data.normal_(0.0, 0.002) | |
| tq, tk, tv = t_sd[f"{prefix}.qkv.weight"].chunk(3, dim=0) | |
| sq, sk, sv = s_attn.qkv.weight.data.chunk(3, dim=0) | |
| sq[:512, :512].copy_(tq) | |
| sk[:512, :512].copy_(tk) | |
| sv[:512, :512].copy_(tv) | |
| s_attn.out.weight.data[:512, :512].copy_(t_sd[f"{prefix}.out.weight"]) | |
| s_attn.q_norm.weight.data.copy_(t_sd[f"{prefix}.q_norm.weight"]) | |
| s_attn.k_norm.weight.data.copy_(t_sd[f"{prefix}.k_norm.weight"]) | |
| def copy_cross_attn(s_attn, prefix): | |
| s_attn.q_proj.weight.data.normal_(0.0, 0.008) | |
| s_attn.kv_proj.weight.data.normal_(0.0, 0.008) | |
| s_attn.out.weight.data.normal_(0.0, 0.002) | |
| s_attn.q_proj.weight.data[:512, :512].copy_(t_sd[f"{prefix}.q_proj.weight"]) | |
| tk, tv = t_sd[f"{prefix}.kv_proj.weight"].chunk(2, dim=0) | |
| sk, sv = s_attn.kv_proj.weight.data.chunk(2, dim=0) | |
| sk[:512, :512].copy_(tk) | |
| sv[:512, :512].copy_(tv) | |
| s_attn.out.weight.data[:512, :512].copy_(t_sd[f"{prefix}.out.weight"]) | |
| s_attn.q_norm.weight.data.copy_(t_sd[f"{prefix}.q_norm.weight"]) | |
| s_attn.k_norm.weight.data.copy_(t_sd[f"{prefix}.k_norm.weight"]) | |
| def copy_swiglu(s_mlp, prefix): | |
| s_mlp.gate_up.weight.data.normal_(0.0, 0.008) | |
| s_mlp.down.weight.data.normal_(0.0, 0.002) | |
| tg, tu = t_sd[f"{prefix}.gate_up.weight"].chunk(2, dim=0) | |
| sg, su = s_mlp.gate_up.weight.data.chunk(2, dim=0) | |
| sg[:1728, :512].copy_(tg) | |
| su[:1728, :512].copy_(tu) | |
| s_mlp.down.weight.data[:512, :1728].copy_(t_sd[f"{prefix}.down.weight"]) | |
| for i in range(6): | |
| copy_rmsnorm(student.encoder[i].norm1, f"encoder.{i}.norm1") | |
| copy_self_attn(student.encoder[i].self_attn, f"encoder.{i}.self_attn") | |
| copy_rmsnorm(student.encoder[i].norm2, f"encoder.{i}.norm2") | |
| copy_swiglu(student.encoder[i].mlp, f"encoder.{i}.mlp") | |
| for i in range(6, 8): | |
| student.encoder[i].self_attn.out.weight.data.normal_(0.0, 0.001) | |
| student.encoder[i].mlp.down.weight.data.normal_(0.0, 0.001) | |
| copy_rmsnorm(student.enc_norm, "enc_norm") | |
| for i in range(6): | |
| copy_rmsnorm(student.decoder[i].norm1, f"decoder.{i}.norm1") | |
| copy_self_attn(student.decoder[i].self_attn, f"decoder.{i}.self_attn") | |
| copy_rmsnorm(student.decoder[i].norm2, f"decoder.{i}.norm2") | |
| copy_cross_attn(student.decoder[i].cross_attn, f"decoder.{i}.cross_attn") | |
| copy_rmsnorm(student.decoder[i].norm3, f"decoder.{i}.norm3") | |
| copy_swiglu(student.decoder[i].mlp, f"decoder.{i}.mlp") | |
| for i in range(6, 8): | |
| student.decoder[i].self_attn.out.weight.data.normal_(0.0, 0.001) | |
| student.decoder[i].cross_attn.out.weight.data.normal_(0.0, 0.001) | |
| student.decoder[i].mlp.down.weight.data.normal_(0.0, 0.001) | |
| copy_rmsnorm(student.dec_norm, "dec_norm") | |
| del t_sd | |
| gc.collect() | |
| torch.cuda.empty_cache() | |
| log("✅ Full All-Layer Net2Wider + Net2Deeper Transfer (84M -> 210M) Complete!") | |
| transfer_84m_to_210m(pro_model, TEACHER_84M) | |
| # ── 4. DataLoader, Optimizer & Schedule (24,000 Expansion Steps + 1,000 Cooldown Steps) ── | |
| MICRO_BS_PRO = 128 | |
| GRAD_ACCUM_PRO = 2 # Global BS = 256 sequences/step (~80.6k tokens/step) | |
| MAX_TOK_CHUNK_PRO = 16384 # Caps peak VRAM at ~17-19 GB on RTX 5090 (zero OOM risk!) | |
| train_ds_pro = Seq2SeqMemmapDataset(DATA_DIR, "train") | |
| val_ds_pro = Seq2SeqMemmapDataset(DATA_DIR, "val") | |
| STAGE1_STEPS_PRO = 24000 # ~2.0B expansion tokens starting from Val PPL 6.73 (= 58,000 effective steps!) | |
| WARMUP_STEPS_PRO = 1000 | |
| PEAK_LR_PRO = 3.0e-4 | |
| MIN_LR_PRO = 3.0e-5 | |
| EVAL_EVERY_PRO = 2000 | |
| LOG_EVERY_PRO = 200 | |
| decay_pro = [p for n, p in pro_model.named_parameters() if p.requires_grad and p.ndim >= 2] | |
| nodecay_pro = [p for n, p in pro_model.named_parameters() if p.requires_grad and p.ndim < 2] | |
| pro_opt = torch.optim.AdamW( | |
| [{"params": decay_pro, "weight_decay": 0.01}, {"params": nodecay_pro, "weight_decay": 0.0}], | |
| lr=PEAK_LR_PRO, betas=(0.9, 0.98), eps=1e-8, fused=True | |
| ) | |
| def get_pro_lr(step: int) -> float: | |
| if step < WARMUP_STEPS_PRO: | |
| return PEAK_LR_PRO * (step + 1) / WARMUP_STEPS_PRO | |
| prog = min(1.0, (step - WARMUP_STEPS_PRO) / max(1, STAGE1_STEPS_PRO - WARMUP_STEPS_PRO)) | |
| return MIN_LR_PRO + 0.5 * (PEAK_LR_PRO - MIN_LR_PRO) * (1.0 + math.cos(math.pi * prog)) | |
| start_step_pro = 0 | |
| best_pro_nll = float("inf") | |
| latest_pro_ckpt = os.path.join(PRO_CKPT, "latest_pro.pt") | |
| if os.path.exists(latest_pro_ckpt): | |
| ckpt_p = torch.load(latest_pro_ckpt, map_location=device) | |
| pro_model.load_state_dict(ckpt_p["model"]) | |
| pro_opt.load_state_dict(ckpt_p["optimizer"]) | |
| start_step_pro = int(ckpt_p["step"]) | |
| best_pro_nll = float(ckpt_p.get("best_val_nll", float("inf"))) | |
| log(f"🔄 Resumed 210M-Pro from step {start_step_pro:,} (best_val_nll={best_pro_nll:.4f})") | |
| train_sampler_pro = LengthBucketBatchSampler( | |
| train_ds_pro.index, batch_size=MICRO_BS_PRO, shuffle=True, seed=2026, start_batch=start_step_pro * GRAD_ACCUM_PRO | |
| ) | |
| val_sampler_pro = LengthBucketBatchSampler( | |
| val_ds_pro.index, batch_size=MICRO_BS_PRO, shuffle=True, seed=12345, start_batch=0 | |
| ) | |
| train_loader_pro = DataLoader(train_ds_pro, batch_sampler=train_sampler_pro, collate_fn=collate_nmt, num_workers=6, pin_memory=True, persistent_workers=True) | |
| val_loader_pro = DataLoader(val_ds_pro, batch_sampler=val_sampler_pro, collate_fn=collate_nmt, num_workers=2, pin_memory=True) | |
| @torch.no_grad() | |
| def evaluate_pro(max_batches: int = 80): | |
| pro_model.eval() | |
| tot_nll, tot_tok = 0.0, 0 | |
| for i, (src_ids, tgt_in, labels) in enumerate(val_loader_pro): | |
| if i >= max_batches: | |
| break | |
| B, S = src_ids.shape | |
| T = tgt_in.size(1) | |
| sub_bs = max(4, min(B, MAX_TOK_CHUNK_PRO // max(S, T, 1))) | |
| for s_i in range(0, B, sub_bs): | |
| sc = src_ids[s_i : s_i + sub_bs].to(device, non_blocking=True) | |
| tc = tgt_in[s_i : s_i + sub_bs].to(device, non_blocking=True) | |
| yc = labels[s_i : s_i + sub_bs].to(device, non_blocking=True) | |
| nll_s, n_v = pro_model.eval_nll(sc, tc, yc) | |
| tot_nll += nll_s | |
| tot_tok += n_v | |
| avg_nll = tot_nll / max(tot_tok, 1) | |
| return avg_nll, math.exp(min(avg_nll, 20.0)) | |
| if start_step_pro == 0: | |
| init_pro_nll, init_pro_ppl = evaluate_pro() | |
| best_pro_nll = init_pro_nll | |
| log(f"🔎 Step 0 (Post-Net2Wider/Deeper Warm-Start) | Val NLL: {init_pro_nll:.4f} | Val PPL: {init_pro_ppl:.2f}") | |
| encoded_idioms_pro = [] | |
| for d_code, s_txt, t_txt in SPOKEN_IDIOM_PAIRS_PRO: | |
| dir_tok = TOK_2MY if d_code == "en2my" else TOK_2EN | |
| s_arr = np.asarray([dir_tok] + sp.encode_as_ids(s_txt.strip()) + [EOS_ID], dtype=np.int64) | |
| t_arr = np.asarray([BOS_ID] + sp.encode_as_ids(t_txt.strip()) + [EOS_ID], dtype=np.int64) | |
| encoded_idioms_pro.append((s_arr, t_arr)) | |
| # ── 5. Stage 1 Expansion Training Loop (24,000 Steps) ── | |
| log(f"\n>>> Starting Flagship MyanmarTranslator-210M-Pro Stage 1 (Step {start_step_pro:,} -> {STAGE1_STEPS_PRO:,})...") | |
| pro_model.train() | |
| pro_opt.zero_grad(set_to_none=True) | |
| step = start_step_pro | |
| micro_idx = start_step_pro * GRAD_ACCUM_PRO | |
| run_loss = run_toks = 0.0 | |
| t_start = time.time() | |
| for src_ids, tgt_in, labels in train_loader_pro: | |
| micro_idx += 1 | |
| B, S = src_ids.shape | |
| T = tgt_in.size(1) | |
| max_len = max(S, T) | |
| sub_bs = max(4, min(B, MAX_TOK_CHUNK_PRO // max(max_len, 1))) | |
| n_subs = math.ceil(B / sub_bs) | |
| for s_i in range(0, B, sub_bs): | |
| s_chunk = src_ids[s_i : s_i + sub_bs].to(device, non_blocking=True) | |
| t_chunk = tgt_in[s_i : s_i + sub_bs].to(device, non_blocking=True) | |
| y_chunk = labels[s_i : s_i + sub_bs].to(device, non_blocking=True) | |
| loss_val, n_val = pro_model.forward_and_backward( | |
| s_chunk, t_chunk, y_chunk, loss_scale=(GRAD_ACCUM_PRO * n_subs), label_smoothing=0.1 | |
| ) | |
| run_loss += loss_val / n_subs | |
| run_toks += n_val + int((s_chunk != PAD_ID).sum().item()) | |
| if micro_idx % GRAD_ACCUM_PRO == 0: | |
| lr = get_pro_lr(step) | |
| for pg in pro_opt.param_groups: | |
| pg["lr"] = lr | |
| grad_norm = torch.nn.utils.clip_grad_norm_(pro_model.parameters(), 1.0) | |
| pro_opt.step() | |
| pro_opt.zero_grad(set_to_none=True) | |
| step += 1 | |
| if step % LOG_EVERY_PRO == 0: | |
| elapsed = time.time() - t_start | |
| avg_l = run_loss / (LOG_EVERY_PRO * GRAD_ACCUM_PRO) | |
| est_nll = max(0.0, (avg_l - 0.1 * math.log(VOCAB_SIZE)) / 0.9) | |
| tok_s = run_toks / max(elapsed, 1e-3) | |
| peak_gb = torch.cuda.max_memory_allocated() / 1e9 | |
| log(f"Step {step:6d}/{STAGE1_STEPS_PRO} | SmoothLoss: {avg_l:.4f} | EstNLL: {est_nll:.4f} (PPL {math.exp(min(est_nll,15)):6.2f}) | LR: {lr:.2e} | Grad: {grad_norm:.2f} | {tok_s/1e3:.1f}k tok/s | VRAM: {peak_gb:.1f}G") | |
| run_loss = run_toks = 0.0 | |
| t_start = time.time() | |
| if step % EVAL_EVERY_PRO == 0 or step == STAGE1_STEPS_PRO: | |
| val_nll, val_ppl = evaluate_pro() | |
| log("\n" + "-"*85) | |
| log(f"🔎 210M-PRO EVAL @ Step {step:,} | Val NLL: {val_nll:.4f} | Val PPL: {val_ppl:.2f}") | |
| for tag, d_code, prompt_txt in AUDIT_SAMPLES: | |
| pred_txt = pro_model.translate(prompt_txt, d_code) | |
| log(f" [{tag}]: {pred_txt}") | |
| torch.save({"step": step, "model": pro_model.state_dict(), "optimizer": pro_opt.state_dict(), "best_val_nll": best_pro_nll, "config": pro_model.config}, latest_pro_ckpt) | |
| if val_nll < best_pro_nll: | |
| best_pro_nll = val_nll | |
| torch.save({"step": step, "model": pro_model.state_dict(), "val_nll": val_nll, "config": pro_model.config}, os.path.join(PRO_CKPT, "best_pro.pt")) | |
| log(f" 🌟 New Best 210M-Pro Checkpoint Saved! (Val NLL: {best_pro_nll:.4f})") | |
| log("-"*85 + "\n") | |
| pro_model.train() | |
| t_start = time.time() | |
| if step >= STAGE1_STEPS_PRO: | |
| break | |
| # ── 6. Stage 2 Unsmoothed Cooldown & Idiom Polish (1,000 Steps, label_smoothing=0.0) ── | |
| log(f"\n>>> Starting 210M-Pro Stage 2 Cooldown (1,000 steps | label_smoothing=0.0 | LR 3e-5 -> 4e-6)...") | |
| best_pro_s1 = torch.load(os.path.join(PRO_CKPT, "best_pro.pt"), map_location=device) | |
| pro_model.load_state_dict(best_pro_s1["model"]) | |
| pro_s2_opt = torch.optim.AdamW( | |
| [{"params": decay_pro, "weight_decay": 0.01}, {"params": nodecay_pro, "weight_decay": 0.0}], | |
| lr=3e-5, betas=(0.9, 0.98), eps=1e-8, fused=True | |
| ) | |
| clean_mask_pro = (train_ds_pro.index[:, 4] < 2) & (train_ds_pro.index[:, 1] >= 16) & (train_ds_pro.index[:, 1] <= 256) | |
| clean_idx_pro = np.flatnonzero(clean_mask_pro) | |
| rng_pro = np.random.RandomState(2026) | |
| pro_model.train() | |
| for s2_step in range(1, 1001): | |
| prog = s2_step / 1000.0 | |
| lr = 4e-6 + 0.5 * (3e-5 - 4e-6) * (1.0 + math.cos(math.pi * prog)) | |
| for pg in pro_s2_opt.param_groups: | |
| pg["lr"] = lr | |
| picked = rng_pro.choice(clean_idx_pro, size=88, replace=False) | |
| batch_items = [train_ds_pro[int(ix)] for ix in picked] | |
| for _ in range(8): | |
| batch_items.append(encoded_idioms_pro[int(rng_pro.randint(len(encoded_idioms_pro)))]) | |
| src_ids, tgt_in, labels = collate_nmt(batch_items) | |
| pro_s2_opt.zero_grad(set_to_none=True) | |
| pro_model.forward_and_backward( | |
| src_ids.to(device, non_blocking=True), | |
| tgt_in.to(device, non_blocking=True), | |
| labels.to(device, non_blocking=True), | |
| loss_scale=1.0, label_smoothing=0.0 | |
| ) | |
| torch.nn.utils.clip_grad_norm_(pro_model.parameters(), 1.0) | |
| pro_s2_opt.step() | |
| if s2_step % 250 == 0: | |
| val_nll, val_ppl = evaluate_pro() | |
| log(f"🔎 210M-Pro Stage 2 Step {s2_step}/1000 | Val NLL: {val_nll:.4f} (PPL {val_ppl:.2f})") | |
| if val_nll <= best_pro_nll + 0.01: | |
| best_pro_nll = min(best_pro_nll, val_nll) | |
| torch.save({"step": STAGE1_STEPS_PRO + s2_step, "model": pro_model.state_dict(), "val_nll": val_nll, "config": pro_model.config}, | |
| os.path.join(PRO_CKPT, "best_pro.pt")) | |
| pro_model.train() | |
| # ── 7. Final Benchmark & Private HF Upload: freococo/MyanmarTranslator-210M-Pro ── | |
| best_pro_final = torch.load(os.path.join(PRO_CKPT, "best_pro.pt"), map_location=device) | |
| pro_model.load_state_dict(best_pro_final["model"]) | |
| pro_model.eval() | |
| final_pro_nll = float(best_pro_final["val_nll"]) | |
| final_pro_ppl = math.exp(final_pro_nll) | |
| log("\n" + "="*90) | |
| log(f"🏆 MYANMARTRANSLATOR-210M-PRO FINAL BENCHMARK | Val NLL: {final_pro_nll:.4f} (PPL {final_pro_ppl:.2f})") | |
| log("="*90) | |
| for tag, d_code, prompt_txt in AUDIT_SAMPLES + UNSEEN_TESTS: | |
| pred_txt = pro_model.translate_beam(prompt_txt, d_code, num_beams=4) | |
| log(f" [{tag} (Beam=4)]: {pred_txt}") | |
| log("="*90) | |
| CKPT_SUB_PRO = os.path.join(PRO_EXPORT, "checkpoint-step-25000") | |
| shutil.rmtree(PRO_EXPORT, ignore_errors=True) | |
| os.makedirs(CKPT_SUB_PRO, exist_ok=True) | |
| sd_fp32 = {k: v.detach().cpu().float().contiguous() for k, v in pro_model.state_dict().items() if k != "lm_head.weight"} | |
| sd_bf16 = {k: v.to(torch.bfloat16).contiguous() for k, v in sd_fp32.items()} | |
| save_file(sd_bf16, os.path.join(PRO_EXPORT, "model.safetensors")) | |
| save_file(sd_fp32, os.path.join(CKPT_SUB_PRO, "model.safetensors")) | |
| if os.path.exists(latest_pro_ckpt): | |
| os.remove(latest_pro_ckpt) | |
| torch.save(best_pro_final, os.path.join(CKPT_SUB_PRO, "training_state.pt")) | |
| BASE_EXPORT = "/workspace/MyanmarTranslator-84M-Base-export" | |
| for folder, dt in [(PRO_EXPORT, "bfloat16"), (CKPT_SUB_PRO, "float32")]: | |
| for fn in ["configuration_myanmartranslator.py", "modeling_myanmartranslator.py", "generation_config.json", | |
| "tokenizer.model", "tokenizer_config.json", "special_tokens_map.json"]: | |
| shutil.copyfile(os.path.join(BASE_EXPORT, fn), os.path.join(folder, fn)) | |
| with open(os.path.join(BASE_EXPORT, "config.json"), "r") as f: | |
| cfg_d = json.load(f) | |
| cfg_d.update(pro_model.config) | |
| cfg_d["torch_dtype"] = dt | |
| cfg_d["val_nll"] = round(final_pro_nll, 4) | |
| cfg_d["val_ppl"] = round(final_pro_ppl, 2) | |
| with open(os.path.join(folder, "config.json"), "w", encoding="utf-8") as f: | |
| json.dump(cfg_d, f, indent=2) | |
| full_output_log = "\n".join(LOG_LINES) + "\n" | |
| for folder in (PRO_EXPORT, CKPT_SUB_PRO): | |
| with open(os.path.join(folder, "training_codes.txt"), "w", encoding="utf-8") as f: | |
| f.write(CELL_SOURCE_CODE) | |
| with open(os.path.join(folder, "trainingcelloutputwitheval.txt"), "w", encoding="utf-8") as f: | |
| f.write(full_output_log) | |
| with open(os.path.join(PRO_EXPORT, "README.md"), "w", encoding="utf-8") as f: | |
| f.write(f"""--- | |
| language: | |
| - my | |
| - en | |
| license: apache-2.0 | |
| library_name: transformers | |
| tags: | |
| - Translation | |
| - Safetensors | |
| - Burmese | |
| - English | |
| - MyanmarLLM | |
| - flagship | |
| - custom_code | |
| --- | |
| # MyanmarTranslator-210M-Pro | |
| Flagship `210.90M`-parameter **8-Layer Bidirectional Encoder + 8-Layer Cross-Attentive Decoder** (`d_model=768`, `n_heads=12`, `ffn_dim=2816` SwiGLU) for **English ↔ Burmese (`my`)**, warm-started via All-Layer Net2Wider+Net2Deeper expansion from `MyanmarTranslator-84M-Base`. | |
| - **Validation NLL:** `{final_pro_nll:.4f}` (**`Val PPL: {final_pro_ppl:.2f}`**) | |
| - **Reproducibility Artifacts:** `training_codes.txt` & `trainingcelloutputwitheval.txt` | |
| """) | |
| api = HfApi() | |
| create_repo(PRO_REPO_ID, repo_type="model", private=True, exist_ok=True) | |
| api.upload_folder( | |
| folder_path=PRO_EXPORT, | |
| repo_id=PRO_REPO_ID, | |
| repo_type="model", | |
| commit_message=f"Release Flagship MyanmarTranslator-210M-Pro (Val PPL {final_pro_ppl:.2f} + code & logs)", | |
| ) | |
| log(f"\n🔒🚀 SUCCESS! Flagship MyanmarTranslator-210M-Pro Live at: https://huggingface.co/{PRO_REPO_ID}") |