--- language: - my - en license: apache-2.0 library_name: transformers tags: - Translation - Safetensors - Burmese - English - MyanmarLLM - flagship - custom_code - myanmar datasets: - HuggingFaceFW/finetranslations pipeline_tag: translation --- # MyanmarTranslator-210M-Pro **MyanmarTranslator-210M-Pro** is a `210.90M`-parameter bidirectional **English ↔ Burmese (`en` ↔ `my`)** neural machine translation model built on a custom **8-Layer Bidirectional Encoder + 8-Layer Cross-Attentive Decoder** Transformer architecture. It was warm-started via All-Layer **Net2Wider + Net2Deeper** expansion from `MyanmarTranslator-84M-Base` and trained on `HuggingFaceFW/finetranslations`. - **Validation NLL:** `1.6724` (**`Val PPL: 5.33`**) - **Precision:** `bfloat16` (`model.safetensors`) - **Tokenizer:** SentencePiece (`65,664` vocab, `tokenizer.model`) - **Reproducibility Artifacts:** `training_codes.txt` & `trainingcelloutputwitheval.txt` --- ## Architecture Specifications | Component | Specification | |---|---| | **Architecture** | Custom Encoder-Decoder Transformer | | **Parameters** | `210.90M` | | **Encoder / Decoder Layers** | `8` Encoder + `8` Decoder | | **Hidden Size (`d_model`)** | `768` | | **Attention Heads (`n_heads`)** | `12` (`head_dim = 64`) | | **Feed-Forward (`ffn_dim`)** | `2816` (`SwiGLU`) | | **Normalization** | `RMSNorm` (`eps=1e-6`, Pre-Norm + QK-Norm) | | **Positional Encoding** | Rotary Position Embeddings (`RoPE`, `theta=10000.0`, `max_seq_len=1024`) | | **Vocabulary Size** | `65,664` (Tied `embed_tokens` & `lm_head`, scaled by `√d_model`) | | **Special Control Tokens** | `UNK=0`, `BOS=1`, `EOS=2`, `PAD=3`, `<2my>=65536`, `<2en>=65537` | --- ## Installation ```bash pip install -U torch sentencepiece safetensors huggingface_hub numpy pyarrow datasets ``` --- ## Quick Start: Bidirectional Translation (`my` ↔ `en`) The standalone PyTorch + SentencePiece implementation below uses **pre-allocated KV-caching**, **PyTorch SDPA** (FlashAttention / Memory-Efficient Attention), **dynamic active-sequence pruning**, and **Burmese Unicode normalization + clause-aware chunking** for long text. ```python import math import os import re import numpy as np import sentencepiece as spm import torch import torch.nn as nn import torch.nn.functional as F from huggingface_hub import snapshot_download from safetensors.torch import load_file MODEL_REPO = "freococo/MyanmarTranslator-210M-Pro" MODEL_DIR = snapshot_download( repo_id=MODEL_REPO, ignore_patterns=["checkpoint-*/*", "*.pt", "*.txt"], ) DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") COMPUTE_DTYPE = ( torch.bfloat16 if (torch.cuda.is_available() and torch.cuda.get_device_capability()[0] >= 8) else (torch.float16 if torch.cuda.is_available() else torch.float32) ) VOCAB_SIZE, UNK_ID, BOS_ID, EOS_ID, PAD_ID = 65664, 0, 1, 2, 3 TOK_2MY, TOK_2EN = 65536, 65537 class RMSNorm(nn.Module): def __init__(self, dim: int, eps: float = 1e-6): super().__init__() self.dim, self.eps = (dim,), eps self.weight = nn.Parameter(torch.ones(dim)) def forward(self, x: torch.Tensor) -> torch.Tensor: return F.rms_norm(x, self.dim, self.weight, self.eps) class RotaryEmbedding(nn.Module): def __init__(self, dim: int, max_seq_len: int = 1024, theta: float = 10000.0): super().__init__() inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) t = torch.arange(max_seq_len, dtype=torch.float32) freqs = torch.outer(t, inv_freq) self.register_buffer("cos_cached", freqs.cos()[None, None, :, :], persistent=False) self.register_buffer("sin_cached", freqs.sin()[None, None, :, :], persistent=False) def forward(self, x: torch.Tensor) -> torch.Tensor: L = x.size(2) cos, sin = self.cos_cached[:, :, :L, :], self.sin_cached[:, :, :L, :] x1, x2 = x[..., ::2], x[..., 1::2] return torch.cat((x1 * cos - x2 * sin, x1 * sin + x2 * cos), dim=-1) class SwiGLU(nn.Module): def __init__(self, d_model: int, ffn_dim: int): super().__init__() self.gate_up = nn.Linear(d_model, 2 * ffn_dim, bias=False) self.down = nn.Linear(ffn_dim, d_model, bias=False) def forward(self, x: torch.Tensor) -> torch.Tensor: gate, up = self.gate_up(x).chunk(2, dim=-1) return self.down(F.silu(gate) * up) class SelfAttention(nn.Module): def __init__(self, d_model: int, n_heads: int, is_causal: bool = False): super().__init__() self.n_heads, self.head_dim, self.is_causal = n_heads, d_model // n_heads, is_causal self.qkv = nn.Linear(d_model, 3 * d_model, bias=False) self.out = nn.Linear(d_model, d_model, bias=False) self.q_norm, self.k_norm = RMSNorm(self.head_dim), RMSNorm(self.head_dim) def forward(self, x: torch.Tensor, rope: RotaryEmbedding, attn_mask: torch.Tensor = None) -> torch.Tensor: B, L, D = x.shape q, k, v = self.qkv(x).chunk(3, dim=-1) q = self.q_norm(q.view(B, L, self.n_heads, self.head_dim).transpose(1, 2)) k = self.k_norm(k.view(B, L, self.n_heads, self.head_dim).transpose(1, 2)) v = v.view(B, L, self.n_heads, self.head_dim).transpose(1, 2) q, k = rope(q), rope(k) y = F.scaled_dot_product_attention( q, k, v, attn_mask=None if self.is_causal else attn_mask, is_causal=self.is_causal ) return self.out(y.transpose(1, 2).reshape(B, L, D)) def step_kv(self, x_step, cos_p, sin_p, pos, k_cache, v_cache) -> torch.Tensor: B, _, D = x_step.shape q, k, v = self.qkv(x_step).chunk(3, dim=-1) q = self.q_norm(q.view(B, 1, self.n_heads, self.head_dim).transpose(1, 2)) k = self.k_norm(k.view(B, 1, self.n_heads, self.head_dim).transpose(1, 2)) v = v.view(B, 1, self.n_heads, self.head_dim).transpose(1, 2) q1, q2 = q[..., ::2], q[..., 1::2] k1, k2 = k[..., ::2], k[..., 1::2] q = torch.cat((q1 * cos_p - q2 * sin_p, q1 * sin_p + q2 * cos_p), dim=-1) k = torch.cat((k1 * cos_p - k2 * sin_p, k1 * sin_p + k2 * cos_p), dim=-1) k_cache[:, :, pos : pos + 1, :] = k v_cache[:, :, pos : pos + 1, :] = v y = F.scaled_dot_product_attention( q, k_cache[:, :, : pos + 1, :], v_cache[:, :, : pos + 1, :], is_causal=False ) return self.out(y.transpose(1, 2).reshape(B, 1, D)) class CrossAttention(nn.Module): def __init__(self, d_model: int, n_heads: int): super().__init__() self.n_heads, self.head_dim = n_heads, d_model // n_heads self.q_proj = nn.Linear(d_model, d_model, bias=False) self.kv_proj = nn.Linear(d_model, 2 * d_model, bias=False) self.out = nn.Linear(d_model, d_model, bias=False) self.q_norm, self.k_norm = RMSNorm(self.head_dim), RMSNorm(self.head_dim) def precompute_kv(self, enc_out: torch.Tensor): B, S, _ = enc_out.shape k, v = self.kv_proj(enc_out).chunk(2, dim=-1) return ( self.k_norm(k.view(B, S, self.n_heads, self.head_dim).transpose(1, 2)), v.view(B, S, self.n_heads, self.head_dim).transpose(1, 2), ) def step_kv(self, x_step, k_cross, v_cross, enc_mask) -> torch.Tensor: B, T, D = x_step.shape q = self.q_norm(self.q_proj(x_step).view(B, T, self.n_heads, self.head_dim).transpose(1, 2)) y = F.scaled_dot_product_attention(q, k_cross, v_cross, attn_mask=enc_mask, is_causal=False) return self.out(y.transpose(1, 2).reshape(B, T, D)) class EncoderLayer(nn.Module): def __init__(self, d_model: int, n_heads: int, ffn_dim: int): super().__init__() self.norm1, self.self_attn = RMSNorm(d_model), SelfAttention(d_model, n_heads, is_causal=False) self.norm2, self.mlp = RMSNorm(d_model), SwiGLU(d_model, ffn_dim) def forward(self, x, rope, enc_mask): x = x + self.self_attn(self.norm1(x), rope, enc_mask) return x + self.mlp(self.norm2(x)) class DecoderLayer(nn.Module): def __init__(self, d_model: int, n_heads: int, ffn_dim: int): super().__init__() self.norm1, self.self_attn = RMSNorm(d_model), SelfAttention(d_model, n_heads, is_causal=True) self.norm2, self.cross_attn = RMSNorm(d_model), CrossAttention(d_model, n_heads) self.norm3, self.mlp = RMSNorm(d_model), SwiGLU(d_model, ffn_dim) def step_kv(self, x_step, cos_p, sin_p, pos, k_self, v_self, k_cross, v_cross, enc_mask): x_step = x_step + self.self_attn.step_kv(self.norm1(x_step), cos_p, sin_p, pos, k_self, v_self) x_step = x_step + self.cross_attn.step_kv(self.norm2(x_step), k_cross, v_cross, enc_mask) return x_step + self.mlp(self.norm3(x_step)) class FastKVTranslator(nn.Module): def __init__( self, vocab_size=65664, d_model=768, n_heads=12, enc_layers=8, dec_layers=8, ffn_dim=2816, max_seq_len=1024, kv_alloc=16384, ): super().__init__() self.d_model, self.n_heads, self.head_dim, self.dec_layers = ( d_model, n_heads, d_model // n_heads, dec_layers, ) self.kv_alloc = kv_alloc self.embed_scale = math.sqrt(d_model) self.embed_tokens = nn.Embedding(vocab_size, d_model, padding_idx=PAD_ID) self.rope = RotaryEmbedding(self.head_dim, max_seq_len=max_seq_len) self.encoder = nn.ModuleList([EncoderLayer(d_model, n_heads, ffn_dim) for _ in range(enc_layers)]) self.enc_norm = RMSNorm(d_model) self.decoder = nn.ModuleList([DecoderLayer(d_model, n_heads, ffn_dim) for _ in range(dec_layers)]) self.dec_norm = RMSNorm(d_model) self.lm_head = nn.Linear(d_model, vocab_size, bias=False) self._k_pool = self._v_pool = None def _get_kv_caches(self, B: int, max_new_tokens: int, dtype: torch.dtype, dev: torch.device): needed = self.dec_layers * B * self.n_heads * max_new_tokens * self.head_dim if self._k_pool is None or self._k_pool.numel() < needed: self._k_pool = self._v_pool = None alloc = max(needed, self.dec_layers * self.kv_alloc * self.n_heads * self.head_dim) self._k_pool = torch.empty(alloc, dtype=dtype, device=dev) self._v_pool = torch.empty(alloc, dtype=dtype, device=dev) shape = (self.dec_layers, B, self.n_heads, max_new_tokens, self.head_dim) return self._k_pool[:needed].view(shape), self._v_pool[:needed].view(shape) @torch.inference_mode() def translate_batch_kv( self, token_id_lists: list, direction: str = "my2en", max_new_tokens: int = 192, rep_penalty: float = 1.12, ) -> list: B = len(token_id_lists) if B == 0: return [] dir_tok = TOK_2EN if direction == "my2en" else TOK_2MY max_len = min(512, max(len(s) for s in token_id_lists) + 2) min_len = min(512, min(len(s) for s in token_id_lists) + 2) max_s = min(512, ((max_len + 7) // 8) * 8) src_np = np.full((B, max_s), PAD_ID, dtype=np.int64) for i, s in enumerate(token_id_lists): trunc = s[:510] L = len(trunc) src_np[i, 0] = dir_tok src_np[i, 1 : 1 + L] = trunc src_np[i, 1 + L] = EOS_ID src_ids = torch.from_numpy(src_np).to(DEVICE) enc_mask = (src_ids != PAD_ID).unsqueeze(1).unsqueeze(2) if (min_len < max_s) else None x_enc = self.embed_tokens(src_ids) * self.embed_scale for layer in self.encoder: x_enc = layer(x_enc, self.rope, enc_mask) enc_out = self.enc_norm(x_enc) cross_kv = [layer.cross_attn.precompute_kv(enc_out) for layer in self.decoder] k_self_cache, v_self_cache = self._get_kv_caches(B, max_new_tokens, COMPUTE_DTYPE, DEVICE) lm_w = self.lm_head.weight[:65536] cos_all, sin_all = self.rope.cos_cached, self.rope.sin_cached eos_tensor = torch.tensor(EOS_ID, dtype=torch.long, device=DEVICE) active_orig_idx = torch.arange(B, dtype=torch.long, device=DEVICE) active_done = torch.zeros(B, dtype=torch.bool, device=DEVICE) active_gen = torch.empty((B, max_new_tokens), dtype=torch.long, device=DEVICE) finished_gen = torch.full((B, max_new_tokens), EOS_ID, dtype=torch.long, device=DEVICE) cur_tok = torch.full((B, 1), BOS_ID, dtype=torch.long, device=DEVICE) actual_max_pos = max_new_tokens for pos in range(max_new_tokens): cos_p, sin_p = cos_all[:, :, pos : pos + 1, :], sin_all[:, :, pos : pos + 1, :] x_dec = self.embed_tokens(cur_tok) * self.embed_scale for l_idx, layer in enumerate(self.decoder): kc, vc = cross_kv[l_idx] x_dec = layer.step_kv( x_dec, cos_p, sin_p, pos, k_self_cache[l_idx], v_self_cache[l_idx], kc, vc, enc_mask ) logits = F.linear(self.dec_norm(x_dec)[:, 0, :], lm_w) logits[:, :2] = -30000.0 logits[:, 3] = -30000.0 if rep_penalty != 1.0 and pos > 0: prev_toks = active_gen[:, :pos] s_logits = torch.gather(logits, 1, prev_toks) s_logits = torch.where(s_logits > 0, s_logits / rep_penalty, s_logits * rep_penalty) logits.scatter_(1, prev_toks, s_logits) nxt = torch.where(active_done, eos_tensor, logits.argmax(dim=-1)) active_gen[:, pos] = nxt active_done = active_done | (nxt == EOS_ID) cur_tok = nxt.unsqueeze(1) if (pos + 1) & 3 == 0: keep_mask = ~active_done n_keep = int(keep_mask.sum().item()) if n_keep == 0: finished_gen[active_orig_idx, : pos + 1] = active_gen[:, : pos + 1] actual_max_pos = pos + 1 break cur_B = active_orig_idx.size(0) if cur_B - n_keep >= 16 or n_keep * 10 <= cur_B * 8: finished_gen[active_orig_idx[active_done], : pos + 1] = active_gen[active_done, : pos + 1] keep_idx = torch.nonzero(keep_mask, as_tuple=False).squeeze(-1) active_orig_idx = active_orig_idx[keep_idx] active_done = active_done[keep_idx] active_gen = active_gen[keep_idx] cur_tok = cur_tok[keep_idx] if enc_mask is not None: enc_mask = enc_mask[keep_idx] cross_kv = [(kc[keep_idx], vc[keep_idx]) for kc, vc in cross_kv] k_self_cache = k_self_cache[:, keep_idx] v_self_cache = v_self_cache[:, keep_idx] else: finished_gen[active_orig_idx, :actual_max_pos] = active_gen[:, :actual_max_pos] gen_cpu = finished_gen[:, :actual_max_pos].cpu().tolist() out_ids = [] for row in gen_cpu: try: out_ids.append(row[: row.index(EOS_ID)]) except ValueError: out_ids.append(row) return out_ids sp = spm.SentencePieceProcessor() sp.load(os.path.join(MODEL_DIR, "tokenizer.model")) model = FastKVTranslator() sd = load_file(os.path.join(MODEL_DIR, "model.safetensors")) model.load_state_dict(sd, strict=False) model.lm_head.weight = model.embed_tokens.weight model = model.to(device=DEVICE, dtype=COMPUTE_DTYPE).eval() # --- Usage Examples --- WS_RE = re.compile(r"[ \t]+") def normalize_myanmar(text: str) -> str: return WS_RE.sub( " ", text.replace("\u103a\u1037", "\u1037\u103a").replace("\u1036\u102f", "\u102f\u1036") ).strip() def translate(texts: list[str], direction: str = "my2en") -> list[str]: cleaned = [normalize_myanmar(t) if direction == "my2en" else t.strip() for t in texts] toks = [sp.encode_as_ids(t) for t in cleaned] max_src = min(510, max(len(t) for t in toks)) max_new = min(256, max(48, int(max_src * 1.4) + 16)) out_ids = model.translate_batch_kv(toks, direction=direction, max_new_tokens=max_new) return [sp.decode_ids(ids).strip() for ids in out_ids] # Burmese -> English print(translate(["အီလက်ထရွန်းနစ် သည် လျှပ်စစ်ပတ်လမ်းများကို လေ့လာသည့် ပညာ ဖြစ်သည်။"], direction="my2en")) # English -> Burmese print(translate(["Electronics is the study of electrical circuits and how they interact."], direction="en2my")) ``` --- ## High-Throughput Batch Dataset Translation Script For large-scale corpus translation, this production runner adds: - **Segment Deduplication (`seg_to_uid`)** so repeated headings, titles, or authors are translated only once per chunk. - **Dynamic Token-Budget Batching** (`MAX_BATCH_SEGS=1024`, `MAX_BATCH_TOKENS=57344`) sorted by token length to eliminate padding waste. - **Clause- & Sentence-Aware Burmese Chunking (`split_burmese_v3`)** for arbitrarily long articles and books. - **Automatic Resume & Background Parquet Upload** directly to a Hugging Face dataset repository. ### Verified Real-World Throughput (`NVIDIA GeForce RTX 5090`, `bfloat16`) - **`freococo/jw_myanmar_bible_dataset`** (`31,078` verses): completed in a few seconds (`5.71 MB` Parquet). - **`kalixlouiis/myanmar-literature-corpus`** (`371,917` rows across 3 combined columns): `~49s` per `50,000`-row shard (`8.2–8.3 GB` peak VRAM). ```python import os, socket os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True" _orig_getaddrinfo = socket.getaddrinfo def _ipv4_first(*args, **kwargs): res = _orig_getaddrinfo(*args, **kwargs) v4 = [r for r in res if r[0] == socket.AF_INET] return v4 if v4 else res socket.getaddrinfo = _ipv4_first import re, gc, glob, time, math, shutil from concurrent.futures import ThreadPoolExecutor import numpy as np import pyarrow as pa import pyarrow.parquet as pq import sentencepiece as spm import torch import torch.nn as nn import torch.nn.functional as F from safetensors.torch import load_file from huggingface_hub import HfApi, snapshot_download, get_token from datasets import load_dataset, concatenate_datasets # ============================================================================== # CONFIGURATION # ============================================================================== TARGET_PRIVATE_REPO = "your-username/myanmar-english-corpus" SOURCE_REPO_ID = "kalixlouiis/myanmar-literature-corpus" TEXT_COLUMNS = ["text", "author", "book_title"] # Joined with "\n" per row HF_TOKEN = os.environ.get("HF_TOKEN") or get_token() assert HF_TOKEN, "Not logged in! Run: huggingface-cli login (or set HF_TOKEN)" BASE_DIR = "/workspace" if os.path.exists("/workspace") else os.path.expanduser("~") MODEL_REPO = "freococo/MyanmarTranslator-210M-Pro" MODEL_DIR = os.path.join(BASE_DIR, "MyanmarTranslator-210M-Pro") OUT_DIR = os.path.join(BASE_DIR, "translated_parquets") os.makedirs(OUT_DIR, exist_ok=True) MAX_BATCH_SEGS, MAX_BATCH_TOKENS, KV_ALLOC = 1024, 57_344, 110_000 BIG_SHARD_ROWS, CUR_SUPER_CHUNK = 50_000, 50_000 DELETE_LOCAL_AFTER_UPLOAD = True device = torch.device("cuda" if torch.cuda.is_available() else "cpu") COMPUTE_DTYPE = torch.bfloat16 if (torch.cuda.is_available() and torch.cuda.get_device_capability()[0] >= 8) else torch.float16 if torch.cuda.is_available(): torch.backends.cuda.enable_flash_sdp(True) torch.backends.cuda.enable_mem_efficient_sdp(True) torch.backends.cuda.enable_math_sdp(True) print(f"🖥️ GPU: {torch.cuda.get_device_name(0)} | Dtype: {COMPUTE_DTYPE} | Batch: {MAX_BATCH_SEGS}/{MAX_BATCH_TOKENS}") def hf_retry(fn, *args, **kwargs): for attempt in range(1, 7): try: return fn(*args, **kwargs) except Exception as e: print(f" ⚠️ HF retry {attempt}/6: {e}") if attempt == 6: raise time.sleep(5 * attempt) api = HfApi(token=HF_TOKEN) hf_retry(api.create_repo, repo_id=TARGET_PRIVATE_REPO, repo_type="dataset", private=True, exist_ok=True) existing_remote_files = set(hf_retry(api.list_repo_files, repo_id=TARGET_PRIVATE_REPO, repo_type="dataset")) VOCAB_SIZE, UNK_ID, BOS_ID, EOS_ID, PAD_ID = 65664, 0, 1, 2, 3 TOK_2MY, TOK_2EN = 65536, 65537 class RMSNorm(nn.Module): def __init__(self, dim: int, eps: float = 1e-6): super().__init__() self.dim, self.eps = (dim,), eps self.weight = nn.Parameter(torch.ones(dim)) def forward(self, x: torch.Tensor) -> torch.Tensor: return F.rms_norm(x, self.dim, self.weight, self.eps) class RotaryEmbedding(nn.Module): def __init__(self, dim: int, max_seq_len: int = 1024, theta: float = 10000.0): super().__init__() inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) t = torch.arange(max_seq_len, dtype=torch.float32) freqs = torch.outer(t, inv_freq) self.register_buffer("cos_cached", freqs.cos()[None, None, :, :], persistent=False) self.register_buffer("sin_cached", freqs.sin()[None, None, :, :], persistent=False) def forward(self, x: torch.Tensor) -> torch.Tensor: L = x.size(2) cos, sin = self.cos_cached[:, :, :L, :], self.sin_cached[:, :, :L, :] x1, x2 = x[..., ::2], x[..., 1::2] return torch.cat((x1 * cos - x2 * sin, x1 * sin + x2 * cos), dim=-1) class SwiGLU(nn.Module): def __init__(self, d_model: int, ffn_dim: int): super().__init__() self.gate_up = nn.Linear(d_model, 2 * ffn_dim, bias=False) self.down = nn.Linear(ffn_dim, d_model, bias=False) def forward(self, x: torch.Tensor) -> torch.Tensor: gate, up = self.gate_up(x).chunk(2, dim=-1) return self.down(F.silu(gate) * up) class SelfAttention(nn.Module): def __init__(self, d_model: int, n_heads: int, is_causal: bool = False): super().__init__() self.n_heads, self.head_dim, self.is_causal = n_heads, d_model // n_heads, is_causal self.qkv = nn.Linear(d_model, 3 * d_model, bias=False) self.out = nn.Linear(d_model, d_model, bias=False) self.q_norm, self.k_norm = RMSNorm(self.head_dim), RMSNorm(self.head_dim) def forward(self, x: torch.Tensor, rope: RotaryEmbedding, attn_mask: torch.Tensor = None) -> torch.Tensor: B, L, D = x.shape q, k, v = self.qkv(x).chunk(3, dim=-1) q = self.q_norm(q.view(B, L, self.n_heads, self.head_dim).transpose(1, 2)) k = self.k_norm(k.view(B, L, self.n_heads, self.head_dim).transpose(1, 2)) v = v.view(B, L, self.n_heads, self.head_dim).transpose(1, 2) q, k = rope(q), rope(k) y = F.scaled_dot_product_attention(q, k, v, attn_mask=None if self.is_causal else attn_mask, is_causal=self.is_causal) return self.out(y.transpose(1, 2).reshape(B, L, D)) def step_kv(self, x_step, cos_p, sin_p, pos, k_cache, v_cache) -> torch.Tensor: B, _, D = x_step.shape q, k, v = self.qkv(x_step).chunk(3, dim=-1) q = self.q_norm(q.view(B, 1, self.n_heads, self.head_dim).transpose(1, 2)) k = self.k_norm(k.view(B, 1, self.n_heads, self.head_dim).transpose(1, 2)) v = v.view(B, 1, self.n_heads, self.head_dim).transpose(1, 2) q1, q2 = q[..., ::2], q[..., 1::2] k1, k2 = k[..., ::2], k[..., 1::2] q = torch.cat((q1 * cos_p - q2 * sin_p, q1 * sin_p + q2 * cos_p), dim=-1) k = torch.cat((k1 * cos_p - k2 * sin_p, k1 * sin_p + k2 * cos_p), dim=-1) k_cache[:, :, pos : pos + 1, :] = k v_cache[:, :, pos : pos + 1, :] = v y = F.scaled_dot_product_attention(q, k_cache[:, :, : pos + 1, :], v_cache[:, :, : pos + 1, :], is_causal=False) return self.out(y.transpose(1, 2).reshape(B, 1, D)) class CrossAttention(nn.Module): def __init__(self, d_model: int, n_heads: int): super().__init__() self.n_heads, self.head_dim = n_heads, d_model // n_heads self.q_proj = nn.Linear(d_model, d_model, bias=False) self.kv_proj = nn.Linear(d_model, 2 * d_model, bias=False) self.out = nn.Linear(d_model, d_model, bias=False) self.q_norm, self.k_norm = RMSNorm(self.head_dim), RMSNorm(self.head_dim) def precompute_kv(self, enc_out: torch.Tensor): B, S, _ = enc_out.shape k, v = self.kv_proj(enc_out).chunk(2, dim=-1) return self.k_norm(k.view(B, S, self.n_heads, self.head_dim).transpose(1, 2)), v.view(B, S, self.n_heads, self.head_dim).transpose(1, 2) def step_kv(self, x_step, k_cross, v_cross, enc_mask) -> torch.Tensor: B, T, D = x_step.shape q = self.q_norm(self.q_proj(x_step).view(B, T, self.n_heads, self.head_dim).transpose(1, 2)) y = F.scaled_dot_product_attention(q, k_cross, v_cross, attn_mask=enc_mask, is_causal=False) return self.out(y.transpose(1, 2).reshape(B, T, D)) class EncoderLayer(nn.Module): def __init__(self, d_model: int, n_heads: int, ffn_dim: int): super().__init__() self.norm1, self.self_attn = RMSNorm(d_model), SelfAttention(d_model, n_heads, is_causal=False) self.norm2, self.mlp = RMSNorm(d_model), SwiGLU(d_model, ffn_dim) def forward(self, x, rope, enc_mask): x = x + self.self_attn(self.norm1(x), rope, enc_mask) return x + self.mlp(self.norm2(x)) class DecoderLayer(nn.Module): def __init__(self, d_model: int, n_heads: int, ffn_dim: int): super().__init__() self.norm1, self.self_attn = RMSNorm(d_model), SelfAttention(d_model, n_heads, is_causal=True) self.norm2, self.cross_attn = RMSNorm(d_model), CrossAttention(d_model, n_heads) self.norm3, self.mlp = RMSNorm(d_model), SwiGLU(d_model, ffn_dim) def step_kv(self, x_step, cos_p, sin_p, pos, k_self, v_self, k_cross, v_cross, enc_mask): x_step = x_step + self.self_attn.step_kv(self.norm1(x_step), cos_p, sin_p, pos, k_self, v_self) x_step = x_step + self.cross_attn.step_kv(self.norm2(x_step), k_cross, v_cross, enc_mask) return x_step + self.mlp(self.norm3(x_step)) class FastKVTranslator(nn.Module): def __init__(self, vocab_size=65664, d_model=768, n_heads=12, enc_layers=8, dec_layers=8, ffn_dim=2816, max_seq_len=1024): super().__init__() self.d_model, self.n_heads, self.head_dim, self.dec_layers = d_model, n_heads, d_model // n_heads, dec_layers self.embed_scale = math.sqrt(d_model) self.embed_tokens = nn.Embedding(vocab_size, d_model, padding_idx=PAD_ID) self.rope = RotaryEmbedding(self.head_dim, max_seq_len=max_seq_len) self.encoder = nn.ModuleList([EncoderLayer(d_model, n_heads, ffn_dim) for _ in range(enc_layers)]) self.enc_norm = RMSNorm(d_model) self.decoder = nn.ModuleList([DecoderLayer(d_model, n_heads, ffn_dim) for _ in range(dec_layers)]) self.dec_norm = RMSNorm(d_model) self.lm_head = nn.Linear(d_model, vocab_size, bias=False) self._k_pool = self._v_pool = None def _get_kv_caches(self, B: int, max_new_tokens: int, dtype: torch.dtype, dev: torch.device): needed = self.dec_layers * B * self.n_heads * max_new_tokens * self.head_dim if self._k_pool is None or self._k_pool.numel() < needed: if self._k_pool is not None: del self._k_pool, self._v_pool self._k_pool = self._v_pool = None torch.cuda.empty_cache() alloc = max(needed, self.dec_layers * KV_ALLOC * self.n_heads * self.head_dim) self._k_pool = torch.empty(alloc, dtype=dtype, device=dev) self._v_pool = torch.empty(alloc, dtype=dtype, device=dev) shape = (self.dec_layers, B, self.n_heads, max_new_tokens, self.head_dim) return self._k_pool[:needed].view(shape), self._v_pool[:needed].view(shape) @torch.inference_mode() def translate_batch_kv(self, token_id_lists: list, direction: str = "my2en", max_new_tokens: int = 192, rep_penalty: float = 1.12) -> list: B = len(token_id_lists) if B == 0: return [] dir_tok = TOK_2EN if direction == "my2en" else TOK_2MY max_len = min(512, max(len(s) for s in token_id_lists) + 2) min_len = min(512, min(len(s) for s in token_id_lists) + 2) max_s = min(512, ((max_len + 7) // 8) * 8) src_np = np.full((B, max_s), PAD_ID, dtype=np.int64) for i, s in enumerate(token_id_lists): trunc = s[:510] L = len(trunc) src_np[i, 0] = dir_tok src_np[i, 1 : 1 + L] = trunc src_np[i, 1 + L] = EOS_ID src_ids = torch.from_numpy(src_np).to(device) enc_mask = (src_ids != PAD_ID).unsqueeze(1).unsqueeze(2) if (min_len < max_s) else None x_enc = self.embed_tokens(src_ids) * self.embed_scale for layer in self.encoder: x_enc = layer(x_enc, self.rope, enc_mask) enc_out = self.enc_norm(x_enc) cross_kv = [layer.cross_attn.precompute_kv(enc_out) for layer in self.decoder] k_self_cache, v_self_cache = self._get_kv_caches(B, max_new_tokens, COMPUTE_DTYPE, device) lm_w = self.lm_head.weight[:65536] cos_all, sin_all = self.rope.cos_cached, self.rope.sin_cached eos_tensor = torch.tensor(EOS_ID, dtype=torch.long, device=device) active_orig_idx = torch.arange(B, dtype=torch.long, device=device) active_done = torch.zeros(B, dtype=torch.bool, device=device) active_gen = torch.empty((B, max_new_tokens), dtype=torch.long, device=device) finished_gen = torch.full((B, max_new_tokens), EOS_ID, dtype=torch.long, device=device) cur_tok = torch.full((B, 1), BOS_ID, dtype=torch.long, device=device) actual_max_pos = max_new_tokens for pos in range(max_new_tokens): cos_p, sin_p = cos_all[:, :, pos : pos + 1, :], sin_all[:, :, pos : pos + 1, :] x_dec = self.embed_tokens(cur_tok) * self.embed_scale for l_idx, layer in enumerate(self.decoder): kc, vc = cross_kv[l_idx] x_dec = layer.step_kv(x_dec, cos_p, sin_p, pos, k_self_cache[l_idx], v_self_cache[l_idx], kc, vc, enc_mask) logits = F.linear(self.dec_norm(x_dec)[:, 0, :], lm_w) logits[:, :2] = -30000.0 logits[:, 3] = -30000.0 if rep_penalty != 1.0 and pos > 0: prev_toks = active_gen[:, :pos] s_logits = torch.gather(logits, 1, prev_toks) s_logits = torch.where(s_logits > 0, s_logits / rep_penalty, s_logits * rep_penalty) logits.scatter_(1, prev_toks, s_logits) nxt = torch.where(active_done, eos_tensor, logits.argmax(dim=-1)) active_gen[:, pos] = nxt active_done = active_done | (nxt == EOS_ID) cur_tok = nxt.unsqueeze(1) if (pos + 1) & 3 == 0: keep_mask = ~active_done n_keep = int(keep_mask.sum().item()) if n_keep == 0: finished_gen[active_orig_idx, : pos + 1] = active_gen[:, : pos + 1] actual_max_pos = pos + 1 break cur_B = active_orig_idx.size(0) if cur_B - n_keep >= 16 or n_keep * 10 <= cur_B * 8: finished_gen[active_orig_idx[active_done], : pos + 1] = active_gen[active_done, : pos + 1] keep_idx = torch.nonzero(keep_mask, as_tuple=False).squeeze(-1) active_orig_idx = active_orig_idx[keep_idx] active_done = active_done[keep_idx] active_gen = active_gen[keep_idx] cur_tok = cur_tok[keep_idx] if enc_mask is not None: enc_mask = enc_mask[keep_idx] cross_kv = [(kc[keep_idx], vc[keep_idx]) for kc, vc in cross_kv] k_self_cache = k_self_cache[:, keep_idx] v_self_cache = v_self_cache[:, keep_idx] else: finished_gen[active_orig_idx, :actual_max_pos] = active_gen[:, :actual_max_pos] gen_cpu = finished_gen[:, :actual_max_pos].cpu().tolist() out_ids = [] for row in gen_cpu: try: out_ids.append(row[: row.index(EOS_ID)]) except ValueError: out_ids.append(row) return out_ids if not os.path.exists(os.path.join(MODEL_DIR, "model.safetensors")): hf_retry(snapshot_download, repo_id=MODEL_REPO, local_dir=MODEL_DIR, ignore_patterns=["checkpoint-*/*", "*.pt", "*.txt"], token=HF_TOKEN) sp = spm.SentencePieceProcessor() sp.load(os.path.join(MODEL_DIR, "tokenizer.model")) model = FastKVTranslator() sd = load_file(os.path.join(MODEL_DIR, "model.safetensors")) model.load_state_dict(sd, strict=False) model.lm_head.weight = model.embed_tokens.weight model = model.to(device=device, dtype=COMPUTE_DTYPE).eval() del sd; gc.collect(); torch.cuda.empty_cache() MYANMAR_CHAR_RE = re.compile(r"[\u1000-\u109F]") WS_RE, PARA_SPLIT_RE, SENT_SPLIT_RE, CLAUSE_SPLIT_RE = re.compile(r"[ \t]+"), re.compile(r"။\s*။|\n+"), re.compile(r"(?<=။)"), re.compile(r"(?<=၊)") N_CPU_THREADS = min(6, os.cpu_count() or 4) PARQUET_SCHEMA = pa.schema([("myanmar_text_original", pa.string()), ("english_text_translation", pa.string())]) def normalize_myanmar_unicode(text: str) -> str: return WS_RE.sub(" ", text.replace("\u103a\u1037", "\u1037\u103a").replace("\u1036\u102f", "\u102f\u1036")).strip() def _hard_split(text: str, max_chars: int) -> list: if len(text) <= max_chars: return [text] words = text.split(" ") if len(words) > 1: out, cur = [], "" for w in words: if not cur: cur = w if len(w) <= max_chars else w[:max_chars] elif len(cur) + len(w) + 1 <= max_chars: cur = cur + " " + w else: out.append(cur); cur = w if cur: out.append(cur) if all(len(x) <= max_chars for x in out): return out return [text[i : i + max_chars] for i in range(0, len(text), max_chars)] def split_burmese_v3(text: str, target_chars: int = 240, max_chars: int = 320) -> list: para_chunks = [] for para in (p.strip() for p in PARA_SPLIT_RE.split(text) if p.strip()): raw_sents = [s.strip() for s in SENT_SPLIT_RE.split(para) if s.strip()] merged_sents, i, n = [], 0, len(raw_sents) while i < n: if len(raw_sents[i]) < 42 and i + 1 < n: raw_sents[i + 1] = raw_sents[i] + " " + raw_sents[i + 1] i += 1; continue merged_sents.append(raw_sents[i]); i += 1 pieces = [] for s in merged_sents: if len(s) <= max_chars: pieces.append(s) else: sub = "" for cl in (c.strip() for c in CLAUSE_SPLIT_RE.split(s) if c.strip()): if not sub: sub = cl elif len(sub) + len(cl) + 1 <= target_chars or len(cl) < 18: sub = sub + " " + cl else: pieces.extend(_hard_split(sub, max_chars)); sub = cl if sub: pieces.extend(_hard_split(sub, max_chars)) chunks, cur = [], "" for p in pieces: if not cur: cur = p elif len(cur) + len(p) + 1 <= target_chars: cur = cur + " " + p else: chunks.append(cur); cur = p if cur: chunks.append(cur) if chunks: para_chunks.append(chunks) return para_chunks def sp_encode_batch(texts: list) -> list: try: return sp.encode(texts, out_type=int, num_threads=N_CPU_THREADS) except TypeError: return [sp.encode_as_ids(t) for t in texts] def sp_decode_batch(id_lists: list) -> list: try: return [s.strip() for s in sp.decode(id_lists)] except Exception: return [sp.decode_ids(ids).strip() for ids in id_lists] def find_uncovered_gaps(short_name: str, total_src_rows: int, remote_files: set): part_re = re.compile(rf"^data/{re.escape(short_name)}/part-(\d+)-(\d+)\.parquet$") covered = [] for fn in remote_files: m_p = part_re.match(fn) if m_p: covered.append((int(m_p.group(1)), int(m_p.group(2)))) covered.sort() merged = [] for s, e in covered: s, e = max(0, s), min(total_src_rows, e) if s >= e: continue if merged and s <= merged[-1][1]: merged[-1] = (merged[-1][0], max(merged[-1][1], e)) else: merged.append((s, e)) gaps, cur = [], 0 for s, e in merged: if cur < s: gaps.append((cur, s)) cur = max(cur, e) if cur < total_src_rows: gaps.append((cur, total_src_rows)) return gaps, sum(e - s for s, e in merged) def upload_parquet_worker(local_path: str, remote_path: str, repo_id_src: str): for attempt in range(1, 7): try: t_up = time.time() api.upload_file( path_or_fileobj=local_path, path_in_repo=remote_path, repo_id=TARGET_PRIVATE_REPO, repo_type="dataset", commit_message=f"Add {remote_path} from {repo_id_src}", ) print(f" ☁️ Uploaded {remote_path} in {time.time()-t_up:.1f}s!") if DELETE_LOCAL_AFTER_UPLOAD and os.path.exists(local_path): os.remove(local_path) return except Exception as e: print(f" ⚠️ Upload retry {attempt}/6 for {remote_path} ({e})...") if attempt == 6: raise time.sleep(5 * attempt) short_name = SOURCE_REPO_ID.split("/")[-1] ds_dict = hf_retry(load_dataset, SOURCE_REPO_ID, token=HF_TOKEN) cur_ds = concatenate_datasets([ds_dict[k] for k in ds_dict.keys()]) if hasattr(ds_dict, "keys") else ds_dict total_src_rows = len(cur_ds) for c in TEXT_COLUMNS: assert c in cur_ds.column_names, f"'{c}' not found! Columns: {cur_ds.column_names}" gaps, covered_rows = find_uncovered_gaps(short_name, total_src_rows, existing_remote_files) if not gaps: print(f"✅ All {total_src_rows:,} rows already uploaded!") else: shard_ranges = [(s, min(s + BIG_SHARD_ROWS, g_end)) for g_start, g_end in gaps for s in range(g_start, g_end, BIG_SHARD_ROWS)] upload_executor = ThreadPoolExecutor(max_workers=1) pending_uploads = [] for sh_idx, (sh_start, sh_end) in enumerate(shard_ranges, 1): remote_path = f"data/{short_name}/part-{sh_start:09d}-{sh_end:09d}.parquet" local_path = os.path.join(OUT_DIR, f"{short_name}_part-{sh_start:09d}-{sh_end:09d}.parquet") if remote_path in existing_remote_files: continue t_sh = time.time() writer = pq.ParquetWriter(local_path, PARQUET_SCHEMA, compression="snappy", use_dictionary=True) shard_out_rows = 0 for sc_start in range(sh_start, sh_end, CUR_SUPER_CHUNK): sc_end = min(sc_start + CUR_SUPER_CHUNK, sh_end) sub_ds = cur_ds.select(range(sc_start, sc_end)) cols_data = {c: sub_ds[c] for c in TEXT_COLUMNS} doc_originals, doc_para_lens, all_segments = [], [], [] for r in range(sc_end - sc_start): parts = [cols_data[c][r].strip() for c in TEXT_COLUMNS if isinstance(cols_data[c][r], str) and cols_data[c][r].strip()] if not parts: continue raw_val = "\n".join(parts) if not MYANMAR_CHAR_RE.search(raw_val): continue norm_val = normalize_myanmar_unicode(raw_val) if not norm_val: continue p_chunks = split_burmese_v3(norm_val) if not p_chunks: continue doc_originals.append(norm_val) doc_para_lens.append([len(p) for p in p_chunks]) for p in p_chunks: all_segments.extend(p) del sub_ds, cols_data if not all_segments: continue seg_to_uid, unique_segs = {}, [] seg_uids = np.empty(len(all_segments), dtype=np.int32) for idx, s in enumerate(all_segments): uid = seg_to_uid.get(s) if uid is None: uid = len(unique_segs); seg_to_uid[s] = uid; unique_segs.append(s) seg_uids[idx] = uid del seg_to_uid, all_segments tokenized_unique = sp_encode_batch(unique_segs) del unique_segs lengths = np.array([len(t) for t in tokenized_unique], dtype=np.int32) sorted_order = np.argsort(lengths) translated_unique = [""] * len(tokenized_unique) dyn_batches, cur_b, cur_max_l = [], [], 0 for uid in sorted_order: l = min(512, int(lengths[uid])) new_max = max(cur_max_l, l) if cur_b and (len(cur_b) >= MAX_BATCH_SEGS or (len(cur_b) + 1) * new_max > MAX_BATCH_TOKENS): dyn_batches.append(cur_b); cur_b, cur_max_l = [int(uid)], l else: cur_b.append(int(uid)); cur_max_l = new_max if cur_b: dyn_batches.append(cur_b) for batch_uids in dyn_batches: batch_toks = [tokenized_unique[u] for u in batch_uids] max_src_in_b = min(510, max(len(t) for t in batch_toks)) dyn_max_new = min(208, max(40, int(max_src_in_b * 1.35) + 12)) out_id_lists = model.translate_batch_kv(batch_toks, direction="my2en", max_new_tokens=dyn_max_new) for u, dec in zip(batch_uids, sp_decode_batch(out_id_lists)): translated_unique[u] = dec out_en_list, cursor = [], 0 for p_lens in doc_para_lens: paras_en = [] for plen in p_lens: paras_en.append(" ".join(translated_unique[seg_uids[cursor + k]] for k in range(plen))) cursor += plen out_en_list.append("\n\n".join(paras_en).strip()) writer.write_table(pa.Table.from_pydict({ "myanmar_text_original": doc_originals, "english_text_translation": out_en_list, }, schema=PARQUET_SCHEMA)) shard_out_rows += len(doc_originals) del doc_originals, doc_para_lens, out_en_list, seg_uids, translated_unique, tokenized_unique, lengths, sorted_order, dyn_batches torch.cuda.empty_cache() writer.close() gc.collect() if shard_out_rows > 0: pending_uploads.append(upload_executor.submit(upload_parquet_worker, local_path, remote_path, SOURCE_REPO_ID)) else: os.remove(local_path) for fut in pending_uploads: fut.result() upload_executor.shutdown(wait=True) print(f"🎉 DONE {SOURCE_REPO_ID} -> https://huggingface.co/datasets/{TARGET_PRIVATE_REPO}") ```