freococo commited on
Commit
b857db2
·
verified ·
1 Parent(s): 94b091d

Release Flagship MyanmarTranslator-210M-Pro (Val PPL 5.33 + code & logs)

Browse files
README.md ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language:
3
+ - my
4
+ - en
5
+ license: apache-2.0
6
+ library_name: transformers
7
+ tags:
8
+ - Translation
9
+ - Safetensors
10
+ - Burmese
11
+ - English
12
+ - MyanmarLLM
13
+ - flagship
14
+ - custom_code
15
+ ---
16
+
17
+ # MyanmarTranslator-210M-Pro
18
+
19
+ Flagship `210.90M`-parameter **8-Layer Bidirectional Encoder + 8-Layer Cross-Attentive Decoder** (`d_model=768`, `n_heads=12`, `ffn_dim=2816` SwiGLU) for **English ↔ Burmese (`my`)**, warm-started via All-Layer Net2Wider+Net2Deeper expansion from `MyanmarTranslator-84M-Base`.
20
+
21
+ - **Validation NLL:** `1.6724` (**`Val PPL: 5.33`**)
22
+ - **Reproducibility Artifacts:** `training_codes.txt` & `trainingcelloutputwitheval.txt`
checkpoint-step-25000/config.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "MyanmarTranslatorForConditionalGeneration"
4
+ ],
5
+ "model_type": "myanmartranslator",
6
+ "auto_map": {
7
+ "AutoConfig": "configuration_myanmartranslator.MyanmarTranslatorConfig",
8
+ "AutoModel": "modeling_myanmartranslator.MyanmarTranslatorForConditionalGeneration",
9
+ "AutoModelForSeq2SeqLM": "modeling_myanmartranslator.MyanmarTranslatorForConditionalGeneration"
10
+ },
11
+ "vocab_size": 65664,
12
+ "d_model": 768,
13
+ "n_heads": 12,
14
+ "enc_layers": 8,
15
+ "dec_layers": 8,
16
+ "ffn_dim": 2816,
17
+ "max_seq_len": 1024,
18
+ "is_encoder_decoder": true,
19
+ "tie_word_embeddings": true,
20
+ "bos_token_id": 1,
21
+ "eos_token_id": 2,
22
+ "pad_token_id": 3,
23
+ "unk_token_id": 0,
24
+ "2my_token_id": 65536,
25
+ "2en_token_id": 65537,
26
+ "denoise_token_id": 65538,
27
+ "mask_token_id": 65539,
28
+ "torch_dtype": "float32",
29
+ "val_nll": 1.6724,
30
+ "val_ppl": 5.33
31
+ }
checkpoint-step-25000/configuration_myanmartranslator.py ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers import PretrainedConfig
2
+
3
+ class MyanmarTranslatorConfig(PretrainedConfig):
4
+ model_type = "myanmartranslator"
5
+ def __init__(self, vocab_size=65664, d_model=512, n_heads=8, enc_layers=6, dec_layers=6, ffn_dim=1728, max_seq_len=1024, **kwargs):
6
+ super().__init__(bos_token_id=1, eos_token_id=2, pad_token_id=3, unk_token_id=0, is_encoder_decoder=True, tie_word_embeddings=True, **kwargs)
7
+ self.vocab_size = vocab_size
8
+ self.d_model = d_model
9
+ self.n_heads = n_heads
10
+ self.enc_layers = enc_layers
11
+ self.dec_layers = dec_layers
12
+ self.ffn_dim = ffn_dim
13
+ self.max_seq_len = max_seq_len
checkpoint-step-25000/generation_config.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 1,
3
+ "eos_token_id": 2,
4
+ "pad_token_id": 3,
5
+ "decoder_start_token_id": 1,
6
+ "max_new_tokens": 192,
7
+ "num_beams": 4,
8
+ "length_penalty": 0.85,
9
+ "repetition_penalty": 1.12
10
+ }
checkpoint-step-25000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:347ddd3a69ae7eee1d37e99b4cbaede45bbe4d2b232966e43ddf00c88ad60726
3
+ size 843607576
checkpoint-step-25000/modeling_myanmartranslator.py ADDED
@@ -0,0 +1,205 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os, math
2
+ import torch
3
+ import torch.nn as nn
4
+ import torch.nn.functional as F
5
+ import sentencepiece as spm
6
+ from transformers import PreTrainedModel
7
+ from .configuration_myanmartranslator import MyanmarTranslatorConfig
8
+
9
+ VOCAB_SIZE, UNK_ID, BOS_ID, EOS_ID, PAD_ID = 65664, 0, 1, 2, 3
10
+ TOK_2MY, TOK_2EN, TOK_DENOISE, TOK_MASK = 65536, 65537, 65538, 65539
11
+
12
+ class RMSNorm(nn.Module):
13
+ def __init__(self, dim: int, eps: float = 1e-6):
14
+ super().__init__()
15
+ self.dim = (dim,)
16
+ self.eps = eps
17
+ self.weight = nn.Parameter(torch.ones(dim))
18
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
19
+ return F.rms_norm(x, self.dim, self.weight, self.eps)
20
+
21
+ class RotaryEmbedding(nn.Module):
22
+ def __init__(self, dim: int, max_seq_len: int = 1024, theta: float = 10000.0):
23
+ super().__init__()
24
+ inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
25
+ t = torch.arange(max_seq_len, dtype=torch.float32)
26
+ freqs = torch.outer(t, inv_freq)
27
+ self.register_buffer("cos_cached", freqs.cos()[None, None, :, :], persistent=False)
28
+ self.register_buffer("sin_cached", freqs.sin()[None, None, :, :], persistent=False)
29
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
30
+ L = x.size(2)
31
+ cos = self.cos_cached[:, :, :L, :].to(dtype=x.dtype)
32
+ sin = self.sin_cached[:, :, :L, :].to(dtype=x.dtype)
33
+ x1, x2 = x[..., ::2], x[..., 1::2]
34
+ return torch.cat([x1 * cos - x2 * sin, x1 * sin + x2 * cos], dim=-1)
35
+
36
+ class SwiGLU(nn.Module):
37
+ def __init__(self, d_model: int, ffn_dim: int):
38
+ super().__init__()
39
+ self.gate_up = nn.Linear(d_model, 2 * ffn_dim, bias=False)
40
+ self.down = nn.Linear(ffn_dim, d_model, bias=False)
41
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
42
+ gate, up = self.gate_up(x).chunk(2, dim=-1)
43
+ return self.down(F.silu(gate) * up)
44
+
45
+ class SelfAttention(nn.Module):
46
+ def __init__(self, d_model: int, n_heads: int, is_causal: bool = False):
47
+ super().__init__()
48
+ self.n_heads = n_heads
49
+ self.head_dim = d_model // n_heads
50
+ self.is_causal = is_causal
51
+ self.qkv = nn.Linear(d_model, 3 * d_model, bias=False)
52
+ self.out = nn.Linear(d_model, d_model, bias=False)
53
+ self.q_norm = RMSNorm(self.head_dim)
54
+ self.k_norm = RMSNorm(self.head_dim)
55
+ def forward(self, x: torch.Tensor, rope: RotaryEmbedding, attn_mask: torch.Tensor = None) -> torch.Tensor:
56
+ B, L, D = x.shape
57
+ q, k, v = self.qkv(x).chunk(3, dim=-1)
58
+ q = self.q_norm(q.view(B, L, self.n_heads, self.head_dim).transpose(1, 2))
59
+ k = self.k_norm(k.view(B, L, self.n_heads, self.head_dim).transpose(1, 2))
60
+ v = v.view(B, L, self.n_heads, self.head_dim).transpose(1, 2)
61
+ q, k = rope(q), rope(k)
62
+ y = F.scaled_dot_product_attention(q, k, v, attn_mask=None if self.is_causal else attn_mask, is_causal=self.is_causal)
63
+ return self.out(y.transpose(1, 2).contiguous().view(B, L, D))
64
+
65
+ class CrossAttention(nn.Module):
66
+ def __init__(self, d_model: int, n_heads: int):
67
+ super().__init__()
68
+ self.n_heads = n_heads
69
+ self.head_dim = d_model // n_heads
70
+ self.q_proj = nn.Linear(d_model, d_model, bias=False)
71
+ self.kv_proj = nn.Linear(d_model, 2 * d_model, bias=False)
72
+ self.out = nn.Linear(d_model, d_model, bias=False)
73
+ self.q_norm = RMSNorm(self.head_dim)
74
+ self.k_norm = RMSNorm(self.head_dim)
75
+ def forward(self, x: torch.Tensor, enc_out: torch.Tensor, enc_mask: torch.Tensor) -> torch.Tensor:
76
+ B, T, D = x.shape
77
+ S = enc_out.size(1)
78
+ q = self.q_norm(self.q_proj(x).view(B, T, self.n_heads, self.head_dim).transpose(1, 2))
79
+ k, v = self.kv_proj(enc_out).chunk(2, dim=-1)
80
+ k = self.k_norm(k.view(B, S, self.n_heads, self.head_dim).transpose(1, 2))
81
+ v = v.view(B, S, self.n_heads, self.head_dim).transpose(1, 2)
82
+ y = F.scaled_dot_product_attention(q, k, v, attn_mask=enc_mask, is_causal=False)
83
+ return self.out(y.transpose(1, 2).contiguous().view(B, T, D))
84
+
85
+ class EncoderLayer(nn.Module):
86
+ def __init__(self, d_model: int, n_heads: int, ffn_dim: int):
87
+ super().__init__()
88
+ self.norm1 = RMSNorm(d_model)
89
+ self.self_attn = SelfAttention(d_model, n_heads, is_causal=False)
90
+ self.norm2 = RMSNorm(d_model)
91
+ self.mlp = SwiGLU(d_model, ffn_dim)
92
+ def forward(self, x: torch.Tensor, rope: RotaryEmbedding, enc_mask: torch.Tensor) -> torch.Tensor:
93
+ x = x + self.self_attn(self.norm1(x), rope, enc_mask)
94
+ x = x + self.mlp(self.norm2(x))
95
+ return x
96
+
97
+ class DecoderLayer(nn.Module):
98
+ def __init__(self, d_model: int, n_heads: int, ffn_dim: int):
99
+ super().__init__()
100
+ self.norm1 = RMSNorm(d_model)
101
+ self.self_attn = SelfAttention(d_model, n_heads, is_causal=True)
102
+ self.norm2 = RMSNorm(d_model)
103
+ self.cross_attn = CrossAttention(d_model, n_heads)
104
+ self.norm3 = RMSNorm(d_model)
105
+ self.mlp = SwiGLU(d_model, ffn_dim)
106
+ def forward(self, x: torch.Tensor, enc_out: torch.Tensor, rope: RotaryEmbedding, enc_mask: torch.Tensor) -> torch.Tensor:
107
+ x = x + self.self_attn(self.norm1(x), rope, None)
108
+ x = x + self.cross_attn(self.norm2(x), enc_out, enc_mask)
109
+ x = x + self.mlp(self.norm3(x))
110
+ return x
111
+
112
+ class MyanmarTranslatorForConditionalGeneration(PreTrainedModel):
113
+ config_class = MyanmarTranslatorConfig
114
+ _tied_weights_keys = ["lm_head.weight"]
115
+
116
+ def __init__(self, config: MyanmarTranslatorConfig):
117
+ super().__init__(config)
118
+ self.d_model = config.d_model
119
+ self.embed_scale = math.sqrt(config.d_model)
120
+ self.embed_tokens = nn.Embedding(config.vocab_size, config.d_model, padding_idx=PAD_ID)
121
+ self.rope = RotaryEmbedding(config.d_model // config.n_heads, max_seq_len=config.max_seq_len)
122
+ self.encoder = nn.ModuleList([EncoderLayer(config.d_model, config.n_heads, config.ffn_dim) for _ in range(config.enc_layers)])
123
+ self.enc_norm = RMSNorm(config.d_model)
124
+ self.decoder = nn.ModuleList([DecoderLayer(config.d_model, config.n_heads, config.ffn_dim) for _ in range(config.dec_layers)])
125
+ self.dec_norm = RMSNorm(config.d_model)
126
+ self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False)
127
+ self.lm_head.weight = self.embed_tokens.weight
128
+ self.sp = None
129
+
130
+ def get_input_embeddings(self):
131
+ return self.embed_tokens
132
+
133
+ def set_input_embeddings(self, value):
134
+ self.embed_tokens = value
135
+ self.lm_head.weight = self.embed_tokens.weight
136
+
137
+ def get_output_embeddings(self):
138
+ return self.lm_head
139
+
140
+ def encode(self, src_ids: torch.Tensor):
141
+ enc_mask = (src_ids != PAD_ID).unsqueeze(1).unsqueeze(2)
142
+ x = self.embed_tokens(src_ids) * self.embed_scale
143
+ for layer in self.encoder:
144
+ x = layer(x, self.rope, enc_mask)
145
+ return self.enc_norm(x), enc_mask
146
+
147
+ def decode(self, tgt_in: torch.Tensor, enc_out: torch.Tensor, enc_mask: torch.Tensor):
148
+ x = self.embed_tokens(tgt_in) * self.embed_scale
149
+ for layer in self.decoder:
150
+ x = layer(x, enc_out, self.rope, enc_mask)
151
+ return self.dec_norm(x)
152
+
153
+ def forward(self, input_ids: torch.Tensor, decoder_input_ids: torch.Tensor, labels: torch.Tensor = None):
154
+ enc_out, enc_mask = self.encode(input_ids)
155
+ dec_out = self.decode(decoder_input_ids, enc_out, enc_mask)
156
+ logits = self.lm_head(dec_out)
157
+ loss = F.cross_entropy(logits.view(-1, logits.size(-1)), labels.view(-1), ignore_index=-100) if labels is not None else None
158
+ return {"loss": loss, "logits": logits}
159
+
160
+ @torch.no_grad()
161
+ def translate(self, text: str, direction: str = "en2my", tokenizer_path: str = None, num_beams: int = 4, max_new_tokens: int = 192, length_penalty: float = 0.85, rep_penalty: float = 1.12) -> str:
162
+ self.eval()
163
+ if self.sp is None:
164
+ if tokenizer_path is None:
165
+ tokenizer_path = os.path.join(getattr(self.config, "_name_or_path", "."), "tokenizer.model")
166
+ if not os.path.exists(tokenizer_path):
167
+ from huggingface_hub import hf_hub_download
168
+ tokenizer_path = hf_hub_download(self.config._name_or_path, "tokenizer.model")
169
+ self.sp = spm.SentencePieceProcessor()
170
+ self.sp.load(tokenizer_path)
171
+
172
+ dev = next(self.parameters()).device
173
+ dir_tok = TOK_2MY if direction == "en2my" else TOK_2EN
174
+ s_ids = [dir_tok] + self.sp.encode_as_ids(text.strip())[:1020] + [EOS_ID]
175
+ src_t = torch.tensor([s_ids], dtype=torch.long, device=dev)
176
+ enc_out, enc_mask = self.encode(src_t)
177
+ beams = [(0.0, [BOS_ID], False)]
178
+ for _ in range(max_new_tokens):
179
+ active = [(sc, seq) for sc, seq, done in beams if not done]
180
+ if not active:
181
+ break
182
+ B_act = len(active)
183
+ tgt_t = torch.tensor([seq for _, seq in active], dtype=torch.long, device=dev)
184
+ dec_out = self.decode(tgt_t, enc_out.expand(B_act, -1, -1), enc_mask.expand(B_act, -1, -1, -1))
185
+ logits = self.lm_head(dec_out[:, -1, :]).float()
186
+ logits[:, [UNK_ID, PAD_ID, BOS_ID]] = -1e9
187
+ logits[:, 65536:] = -1e9
188
+ for b_i, (_, seq) in enumerate(active):
189
+ if rep_penalty != 1.0 and len(seq) > 1:
190
+ seen = torch.tensor(list(set(seq[1:])), dtype=torch.long, device=dev)
191
+ s_l = logits[b_i, seen]
192
+ logits[b_i, seen] = torch.where(s_l > 0, s_l / rep_penalty, s_l * rep_penalty)
193
+ log_probs = F.log_softmax(logits, dim=-1)
194
+ topk_lp, topk_ids = torch.topk(log_probs, max(1, num_beams * 2), dim=-1)
195
+ candidates = [(sc, seq, True) for sc, seq, done in beams if done]
196
+ for b_i, (sc, seq) in enumerate(active):
197
+ for k in range(topk_ids.size(1)):
198
+ tok_id = int(topk_ids[b_i, k].item())
199
+ new_sc = sc + float(topk_lp[b_i, k].item())
200
+ candidates.append((new_sc, seq if tok_id == EOS_ID else seq + [tok_id], tok_id == EOS_ID))
201
+ candidates.sort(key=lambda x: x[0] / (max(1, len(x[1]) - 1) ** length_penalty), reverse=True)
202
+ beams = candidates[:num_beams]
203
+ if all(done for _, _, done in beams):
204
+ break
205
+ return self.sp.decode_ids([x for x in beams[0][1][1:] if x < 65536])
checkpoint-step-25000/special_tokens_map.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<s>",
3
+ "eos_token": "</s>",
4
+ "unk_token": "<unk>",
5
+ "pad_token": "<pad>",
6
+ "additional_special_tokens": [
7
+ "<2my>",
8
+ "<2en>",
9
+ "<denoise>",
10
+ "<mask>"
11
+ ]
12
+ }
checkpoint-step-25000/tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:abb0486c9edaaf32ee5b10074db0077263407d1fe4006c74ad2c6261285b6403
3
+ size 2210735
checkpoint-step-25000/tokenizer_config.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<unk>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<s>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "</s>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<pad>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ }
35
+ },
36
+ "backend": "custom",
37
+ "bos_token": "<s>",
38
+ "eos_token": "</s>",
39
+ "model_max_length": 131072,
40
+ "pad_token": "<pad>",
41
+ "tokenizer_class": "MyanmarSentencePieceTokenizer",
42
+ "unk_token": "<unk>"
43
+ }
checkpoint-step-25000/training_codes.txt ADDED
@@ -0,0 +1,793 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ==============================================================================
2
+ # 🏛️ CELL 11 (100% STANDALONE): Flagship MyanmarTranslator-210M-Pro
3
+ # (8L Enc + 8L Dec, d=768, 210.90M Params) + Net2Wider/Deeper + Auto-Upload
4
+ # ==============================================================================
5
+ import os
6
+ os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
7
+
8
+ import sys, gc, math, time, json, shutil, traceback
9
+ import numpy as np
10
+ import sentencepiece as spm
11
+ import torch
12
+ import torch.nn as nn
13
+ import torch.nn.functional as F
14
+ from torch.utils.checkpoint import checkpoint as grad_ckpt
15
+ from torch.utils.data import Dataset, DataLoader, Sampler
16
+ from safetensors.torch import save_file
17
+ from huggingface_hub import HfApi, create_repo
18
+
19
+ torch.backends.cuda.matmul.allow_tf32 = True
20
+ torch.backends.cudnn.allow_tf32 = True
21
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
22
+
23
+ # ── 0. Capture Exact Cell Code & Setup Incremental Logger ──
24
+ try:
25
+ CELL_SOURCE_CODE = get_ipython().user_ns["In"][-1]
26
+ except Exception:
27
+ CELL_SOURCE_CODE = "# MyanmarTranslator-210M-Pro Standalone Training Script"
28
+
29
+ PRO_REPO_ID = "freococo/MyanmarTranslator-210M-Pro"
30
+ WORK_DIR = "/workspace"
31
+ DATA_DIR = "/workspace/nmt_dataset_memmap"
32
+ TOK_DIR = "/workspace/nmt_tokenizer"
33
+ TOK_FILE = os.path.join(TOK_DIR, "tokenizer.model")
34
+ TEACHER_84M = "/workspace/nmt_84m_polished/best_model.pt"
35
+ PRO_CKPT = "/workspace/pro_210m_checkpoints"
36
+ PRO_EXPORT = "/workspace/MyanmarTranslator-210M-Pro-export"
37
+ os.makedirs(PRO_CKPT, exist_ok=True)
38
+
39
+ if not os.path.exists(os.path.join(PRO_CKPT, "latest_pro.pt")):
40
+ with open(os.path.join(PRO_CKPT, "trainingcelloutputwitheval.txt"), "w", encoding="utf-8") as lf:
41
+ lf.write("")
42
+
43
+ with open(os.path.join(PRO_CKPT, "training_codes.txt"), "w", encoding="utf-8") as f:
44
+ f.write(CELL_SOURCE_CODE)
45
+
46
+ LOG_LINES = []
47
+ def log(msg: str = ""):
48
+ print(msg, flush=True)
49
+ LOG_LINES.append(str(msg))
50
+ with open(os.path.join(PRO_CKPT, "trainingcelloutputwitheval.txt"), "a", encoding="utf-8") as lf:
51
+ lf.write(str(msg) + "\n")
52
+
53
+ gc.collect()
54
+ if torch.cuda.is_available():
55
+ torch.cuda.empty_cache()
56
+ free_vram, tot_vram = torch.cuda.mem_get_info()
57
+ _, _, free_disk = shutil.disk_usage(WORK_DIR)
58
+ log(f"🟢 Standalone Kernel Ready for 210M-Pro | GPU Free: {free_vram/1e9:.2f}/{tot_vram/1e9:.2f} GB | Disk Free: {free_disk/1e9:.1f} GB")
59
+
60
+ sp = spm.SentencePieceProcessor()
61
+ sp.load(TOK_FILE)
62
+
63
+ VOCAB_SIZE, UNK_ID, BOS_ID, EOS_ID, PAD_ID = 65664, 0, 1, 2, 3
64
+ TOK_2MY, TOK_2EN, TOK_DENOISE, TOK_MASK = 65536, 65537, 65538, 65539
65
+
66
+ # ── 1. Standalone Architecture Definitions ──
67
+ class RMSNorm(nn.Module):
68
+ def __init__(self, dim: int, eps: float = 1e-6):
69
+ super().__init__()
70
+ self.dim = (dim,)
71
+ self.eps = eps
72
+ self.weight = nn.Parameter(torch.ones(dim))
73
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
74
+ return F.rms_norm(x, self.dim, self.weight, self.eps)
75
+
76
+ class RotaryEmbedding(nn.Module):
77
+ def __init__(self, dim: int, max_seq_len: int = 1024, theta: float = 10000.0):
78
+ super().__init__()
79
+ inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
80
+ t = torch.arange(max_seq_len, dtype=torch.float32)
81
+ freqs = torch.outer(t, inv_freq)
82
+ self.register_buffer("cos_cached", freqs.cos()[None, None, :, :], persistent=False)
83
+ self.register_buffer("sin_cached", freqs.sin()[None, None, :, :], persistent=False)
84
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
85
+ L = x.size(2)
86
+ cos = self.cos_cached[:, :, :L, :].to(dtype=x.dtype)
87
+ sin = self.sin_cached[:, :, :L, :].to(dtype=x.dtype)
88
+ x1, x2 = x[..., ::2], x[..., 1::2]
89
+ return torch.cat([x1 * cos - x2 * sin, x1 * sin + x2 * cos], dim=-1)
90
+
91
+ class SwiGLU(nn.Module):
92
+ def __init__(self, d_model: int, ffn_dim: int):
93
+ super().__init__()
94
+ self.gate_up = nn.Linear(d_model, 2 * ffn_dim, bias=False)
95
+ self.down = nn.Linear(ffn_dim, d_model, bias=False)
96
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
97
+ gate, up = self.gate_up(x).chunk(2, dim=-1)
98
+ return self.down(F.silu(gate) * up)
99
+
100
+ class SelfAttention(nn.Module):
101
+ def __init__(self, d_model: int, n_heads: int, is_causal: bool = False):
102
+ super().__init__()
103
+ self.n_heads = n_heads
104
+ self.head_dim = d_model // n_heads
105
+ self.is_causal = is_causal
106
+ self.qkv = nn.Linear(d_model, 3 * d_model, bias=False)
107
+ self.out = nn.Linear(d_model, d_model, bias=False)
108
+ self.q_norm = RMSNorm(self.head_dim)
109
+ self.k_norm = RMSNorm(self.head_dim)
110
+ def forward(self, x: torch.Tensor, rope: RotaryEmbedding, attn_mask: torch.Tensor = None) -> torch.Tensor:
111
+ B, L, D = x.shape
112
+ q, k, v = self.qkv(x).chunk(3, dim=-1)
113
+ q = self.q_norm(q.view(B, L, self.n_heads, self.head_dim).transpose(1, 2))
114
+ k = self.k_norm(k.view(B, L, self.n_heads, self.head_dim).transpose(1, 2))
115
+ v = v.view(B, L, self.n_heads, self.head_dim).transpose(1, 2)
116
+ q, k = rope(q), rope(k)
117
+ y = F.scaled_dot_product_attention(q, k, v, attn_mask=None if self.is_causal else attn_mask, is_causal=self.is_causal)
118
+ return self.out(y.transpose(1, 2).contiguous().view(B, L, D))
119
+
120
+ class CrossAttention(nn.Module):
121
+ def __init__(self, d_model: int, n_heads: int):
122
+ super().__init__()
123
+ self.n_heads = n_heads
124
+ self.head_dim = d_model // n_heads
125
+ self.q_proj = nn.Linear(d_model, d_model, bias=False)
126
+ self.kv_proj = nn.Linear(d_model, 2 * d_model, bias=False)
127
+ self.out = nn.Linear(d_model, d_model, bias=False)
128
+ self.q_norm = RMSNorm(self.head_dim)
129
+ self.k_norm = RMSNorm(self.head_dim)
130
+ def forward(self, x: torch.Tensor, enc_out: torch.Tensor, enc_mask: torch.Tensor) -> torch.Tensor:
131
+ B, T, D = x.shape
132
+ S = enc_out.size(1)
133
+ q = self.q_norm(self.q_proj(x).view(B, T, self.n_heads, self.head_dim).transpose(1, 2))
134
+ k, v = self.kv_proj(enc_out).chunk(2, dim=-1)
135
+ k = self.k_norm(k.view(B, S, self.n_heads, self.head_dim).transpose(1, 2))
136
+ v = v.view(B, S, self.n_heads, self.head_dim).transpose(1, 2)
137
+ y = F.scaled_dot_product_attention(q, k, v, attn_mask=enc_mask, is_causal=False)
138
+ return self.out(y.transpose(1, 2).contiguous().view(B, T, D))
139
+
140
+ class EncoderLayer(nn.Module):
141
+ def __init__(self, d_model: int, n_heads: int, ffn_dim: int):
142
+ super().__init__()
143
+ self.norm1 = RMSNorm(d_model)
144
+ self.self_attn = SelfAttention(d_model, n_heads, is_causal=False)
145
+ self.norm2 = RMSNorm(d_model)
146
+ self.mlp = SwiGLU(d_model, ffn_dim)
147
+ def forward(self, x: torch.Tensor, rope: RotaryEmbedding, enc_mask: torch.Tensor) -> torch.Tensor:
148
+ x = x + self.self_attn(self.norm1(x), rope, enc_mask)
149
+ x = x + self.mlp(self.norm2(x))
150
+ return x
151
+
152
+ class DecoderLayer(nn.Module):
153
+ def __init__(self, d_model: int, n_heads: int, ffn_dim: int):
154
+ super().__init__()
155
+ self.norm1 = RMSNorm(d_model)
156
+ self.self_attn = SelfAttention(d_model, n_heads, is_causal=True)
157
+ self.norm2 = RMSNorm(d_model)
158
+ self.cross_attn = CrossAttention(d_model, n_heads)
159
+ self.norm3 = RMSNorm(d_model)
160
+ self.mlp = SwiGLU(d_model, ffn_dim)
161
+ def forward(self, x: torch.Tensor, enc_out: torch.Tensor, rope: RotaryEmbedding, enc_mask: torch.Tensor) -> torch.Tensor:
162
+ x = x + self.self_attn(self.norm1(x), rope, None)
163
+ x = x + self.cross_attn(self.norm2(x), enc_out, enc_mask)
164
+ x = x + self.mlp(self.norm3(x))
165
+ return x
166
+
167
+ class MyanmarTranslator84M(nn.Module):
168
+ def __init__(self, vocab_size=65664, d_model=768, n_heads=12, enc_layers=8, dec_layers=8, ffn_dim=2816, max_seq_len=1024):
169
+ super().__init__()
170
+ self.config = dict(
171
+ vocab_size=vocab_size, d_model=d_model, n_heads=n_heads,
172
+ enc_layers=enc_layers, dec_layers=dec_layers, ffn_dim=ffn_dim, max_seq_len=max_seq_len
173
+ )
174
+ self.d_model = d_model
175
+ self.embed_scale = math.sqrt(d_model)
176
+ self.embed_tokens = nn.Embedding(vocab_size, d_model, padding_idx=PAD_ID)
177
+ self.rope = RotaryEmbedding(d_model // n_heads, max_seq_len=max_seq_len)
178
+ self.encoder = nn.ModuleList([EncoderLayer(d_model, n_heads, ffn_dim) for _ in range(enc_layers)])
179
+ self.enc_norm = RMSNorm(d_model)
180
+ self.decoder = nn.ModuleList([DecoderLayer(d_model, n_heads, ffn_dim) for _ in range(dec_layers)])
181
+ self.dec_norm = RMSNorm(d_model)
182
+ self.lm_head = nn.Linear(d_model, vocab_size, bias=False)
183
+ self.lm_head.weight = self.embed_tokens.weight
184
+ self._init_weights(enc_layers + dec_layers)
185
+
186
+ def _init_weights(self, total_layers: int):
187
+ res_std = 0.02 / math.sqrt(2 * total_layers)
188
+ for name, m in self.named_modules():
189
+ if isinstance(m, nn.Linear):
190
+ nn.init.normal_(m.weight, mean=0.0, std=res_std if (name.endswith(".out") or name.endswith(".down")) else 0.02)
191
+ elif isinstance(m, nn.Embedding):
192
+ nn.init.normal_(m.weight, mean=0.0, std=0.02)
193
+ if m.padding_idx is not None:
194
+ m.weight.data[m.padding_idx].zero_()
195
+
196
+ def encode(self, src_ids: torch.Tensor, use_ckpt: bool = False):
197
+ enc_mask = (src_ids != PAD_ID).unsqueeze(1).unsqueeze(2)
198
+ x = self.embed_tokens(src_ids) * self.embed_scale
199
+ for layer in self.encoder:
200
+ x = grad_ckpt(layer, x, self.rope, enc_mask, use_reentrant=False) if (use_ckpt and self.training) else layer(x, self.rope, enc_mask)
201
+ return self.enc_norm(x), enc_mask
202
+
203
+ def decode(self, tgt_in: torch.Tensor, enc_out: torch.Tensor, enc_mask: torch.Tensor, use_ckpt: bool = False):
204
+ x = self.embed_tokens(tgt_in) * self.embed_scale
205
+ for layer in self.decoder:
206
+ x = grad_ckpt(layer, x, enc_out, self.rope, enc_mask, use_reentrant=False) if (use_ckpt and self.training) else layer(x, enc_out, self.rope, enc_mask)
207
+ return self.dec_norm(x)
208
+
209
+ def forward_and_backward(self, src_ids: torch.Tensor, tgt_in: torch.Tensor, labels: torch.Tensor, loss_scale: float = 1.0, label_smoothing: float = 0.1):
210
+ use_ckpt = max(src_ids.size(1), tgt_in.size(1)) > 192
211
+ with torch.amp.autocast("cuda", dtype=torch.bfloat16):
212
+ enc_out, enc_mask = self.encode(src_ids, use_ckpt=use_ckpt)
213
+ dec_out = self.decode(tgt_in, enc_out, enc_mask, use_ckpt=use_ckpt)
214
+ valid_mask = labels.ne(-100)
215
+ valid_h = dec_out[valid_mask]
216
+ valid_y = labels[valid_mask]
217
+ n_valid = valid_y.numel()
218
+
219
+ h_det = valid_h.detach().requires_grad_(True)
220
+ denom = float(n_valid) * float(loss_scale)
221
+ tot_loss = 0.0
222
+ chunk_sz = 2048
223
+
224
+ for i in range(0, n_valid, chunk_sz):
225
+ hc = h_det[i : i + chunk_sz]
226
+ yc = valid_y[i : i + chunk_sz]
227
+ with torch.amp.autocast("cuda", dtype=torch.bfloat16):
228
+ logits_c = self.lm_head(hc)
229
+ lc = F.cross_entropy(logits_c, yc, label_smoothing=label_smoothing, reduction="sum")
230
+ (lc / denom).backward()
231
+ tot_loss += float(lc.detach().item())
232
+ del logits_c, lc
233
+
234
+ valid_h.backward(h_det.grad)
235
+ return tot_loss / max(n_valid, 1), n_valid
236
+
237
+ @torch.no_grad()
238
+ def eval_nll(self, src_ids: torch.Tensor, tgt_in: torch.Tensor, labels: torch.Tensor):
239
+ with torch.amp.autocast("cuda", dtype=torch.bfloat16):
240
+ enc_out, enc_mask = self.encode(src_ids, use_ckpt=False)
241
+ dec_out = self.decode(tgt_in, enc_out, enc_mask, use_ckpt=False)
242
+ valid_mask = labels.ne(-100)
243
+ valid_h = dec_out[valid_mask]
244
+ valid_y = labels[valid_mask]
245
+ n_valid = valid_y.numel()
246
+ tot_nll = 0.0
247
+ for i in range(0, n_valid, 2048):
248
+ logits_c = self.lm_head(valid_h[i : i + 2048])
249
+ tot_nll += float(F.cross_entropy(logits_c, valid_y[i : i + 2048], reduction="sum").item())
250
+ return tot_nll, n_valid
251
+
252
+ @torch.no_grad()
253
+ def translate(self, text: str, direction: str = "en2my", max_new_tokens: int = 192, rep_penalty: float = 1.15) -> str:
254
+ self.eval()
255
+ dir_tok = TOK_2MY if direction == "en2my" else TOK_2EN
256
+ s_ids = [dir_tok] + sp.encode_as_ids(text.strip())[:1020] + [EOS_ID]
257
+ src_t = torch.tensor([s_ids], dtype=torch.long, device=next(self.parameters()).device)
258
+ with torch.amp.autocast("cuda", dtype=torch.bfloat16):
259
+ enc_out, enc_mask = self.encode(src_t, use_ckpt=False)
260
+ ys = [BOS_ID]
261
+ for _ in range(max_new_tokens):
262
+ tgt_t = torch.tensor([ys], dtype=torch.long, device=src_t.device)
263
+ dec_out = self.decode(tgt_t, enc_out, enc_mask, use_ckpt=False)
264
+ logits = self.lm_head(dec_out[:, -1, :]).float()[0]
265
+ logits[[UNK_ID, PAD_ID, BOS_ID]] = -1e9
266
+ logits[65536:] = -1e9
267
+ if rep_penalty != 1.0 and len(ys) > 1:
268
+ seen = torch.tensor(list(set(ys[1:])), dtype=torch.long, device=src_t.device)
269
+ s_logits = logits[seen]
270
+ logits[seen] = torch.where(s_logits > 0, s_logits / rep_penalty, s_logits * rep_penalty)
271
+ nxt = int(logits.argmax().item())
272
+ if nxt == EOS_ID:
273
+ break
274
+ ys.append(nxt)
275
+ return sp.decode_ids([x for x in ys[1:] if x < 65536])
276
+
277
+ @torch.no_grad()
278
+ def translate_beam(self, text: str, direction: str = "en2my", num_beams: int = 4, max_new_tokens: int = 192, length_penalty: float = 0.85, rep_penalty: float = 1.12) -> str:
279
+ self.eval()
280
+ if num_beams <= 1:
281
+ return self.translate(text, direction=direction, max_new_tokens=max_new_tokens, rep_penalty=rep_penalty)
282
+ dir_tok = TOK_2MY if direction == "en2my" else TOK_2EN
283
+ s_ids = [dir_tok] + sp.encode_as_ids(text.strip())[:1020] + [EOS_ID]
284
+ src_t = torch.tensor([s_ids], dtype=torch.long, device=next(self.parameters()).device)
285
+ with torch.amp.autocast("cuda", dtype=torch.bfloat16):
286
+ enc_out, enc_mask = self.encode(src_t, use_ckpt=False)
287
+ beams = [(0.0, [BOS_ID], False)]
288
+ for _ in range(max_new_tokens):
289
+ active = [(sc, seq) for sc, seq, done in beams if not done]
290
+ if not active:
291
+ break
292
+ B_act = len(active)
293
+ tgt_t = torch.tensor([seq for _, seq in active], dtype=torch.long, device=src_t.device)
294
+ dec_out = self.decode(tgt_t, enc_out.expand(B_act, -1, -1), enc_mask.expand(B_act, -1, -1, -1), use_ckpt=False)
295
+ logits = self.lm_head(dec_out[:, -1, :]).float()
296
+ logits[:, [UNK_ID, PAD_ID, BOS_ID]] = -1e9
297
+ logits[:, 65536:] = -1e9
298
+ for b_i, (_, seq) in enumerate(active):
299
+ if rep_penalty != 1.0 and len(seq) > 1:
300
+ seen = torch.tensor(list(set(seq[1:])), dtype=torch.long, device=src_t.device)
301
+ s_l = logits[b_i, seen]
302
+ logits[b_i, seen] = torch.where(s_l > 0, s_l / rep_penalty, s_l * rep_penalty)
303
+ log_probs = F.log_softmax(logits, dim=-1)
304
+ topk_lp, topk_ids = torch.topk(log_probs, num_beams * 2, dim=-1)
305
+ candidates = [(sc, seq, True) for sc, seq, done in beams if done]
306
+ for b_i, (sc, seq) in enumerate(active):
307
+ for k in range(num_beams * 2):
308
+ tok_id = int(topk_ids[b_i, k].item())
309
+ new_sc = sc + float(topk_lp[b_i, k].item())
310
+ candidates.append((new_sc, seq if tok_id == EOS_ID else seq + [tok_id], tok_id == EOS_ID))
311
+ candidates.sort(key=lambda x: x[0] / (max(1, len(x[1]) - 1) ** length_penalty), reverse=True)
312
+ beams = candidates[:num_beams]
313
+ if all(done for _, _, done in beams):
314
+ break
315
+ return sp.decode_ids([x for x in beams[0][1][1:] if x < 65536])
316
+
317
+ # ── 2. Standalone Dataset & Sampler Definitions ──
318
+ class Seq2SeqMemmapDataset(Dataset):
319
+ def __init__(self, data_dir: str, split: str = "train"):
320
+ self.src_path = os.path.join(data_dir, f"{split}_src.bin")
321
+ self.tgt_path = os.path.join(data_dir, f"{split}_tgt.bin")
322
+ self.index = np.load(os.path.join(data_dir, f"{split}_index.npy"))
323
+ self.src_mm = None
324
+ self.tgt_mm = None
325
+ def _ensure_open(self):
326
+ if self.src_mm is None:
327
+ self.src_mm = np.memmap(self.src_path, dtype=np.uint16, mode="r")
328
+ self.tgt_mm = np.memmap(self.tgt_path, dtype=np.uint16, mode="r")
329
+ def __len__(self):
330
+ return len(self.index)
331
+ def __getitem__(self, idx: int):
332
+ self._ensure_open()
333
+ s0, sl, t0, tl, kind = self.index[idx]
334
+ s_raw = self.src_mm[s0 : s0 + sl].astype(np.int64)
335
+ if kind == 2:
336
+ s_raw[s_raw == 0] = TOK_MASK
337
+ src = np.empty(sl + 1, dtype=np.int64)
338
+ src[0] = 65536 + kind
339
+ src[1:] = s_raw
340
+ tgt = self.tgt_mm[t0 : t0 + tl].astype(np.int64)
341
+ return src, tgt
342
+
343
+ class LengthBucketBatchSampler(Sampler):
344
+ def __init__(self, index_arr: np.ndarray, batch_size: int, mega_mult: int = 256, shuffle: bool = True, seed: int = 42, start_batch: int = 0):
345
+ self.lengths = (index_arr[:, 1] + index_arr[:, 3]).astype(np.int32)
346
+ self.batch_size = batch_size
347
+ self.mega_size = batch_size * mega_mult
348
+ self.shuffle = shuffle
349
+ self.seed = seed
350
+ self.start_batch = start_batch
351
+ self.num_batches = len(self.lengths) // batch_size
352
+ def __len__(self):
353
+ return max(0, self.num_batches - self.start_batch)
354
+ def __iter__(self):
355
+ rng = np.random.RandomState(self.seed)
356
+ n = len(self.lengths)
357
+ order = rng.permutation(n) if self.shuffle else np.arange(n)
358
+ batches = []
359
+ for i in range(0, n - self.batch_size + 1, self.mega_size):
360
+ chunk = order[i : i + self.mega_size]
361
+ chunk = chunk[np.argsort(self.lengths[chunk], kind="mergesort")]
362
+ for j in range(0, len(chunk) - self.batch_size + 1, self.batch_size):
363
+ batches.append(chunk[j : j + self.batch_size])
364
+ if self.shuffle:
365
+ rng.shuffle(batches)
366
+ for b in batches[self.start_batch:]:
367
+ yield b.tolist()
368
+
369
+ def collate_nmt(batch):
370
+ B = len(batch)
371
+ max_s = min(1024, ((max(len(x[0]) for x in batch) + 15) // 16) * 16)
372
+ max_t = min(1024, ((max(len(x[1]) - 1 for x in batch) + 15) // 16) * 16)
373
+ src_ids = torch.full((B, max_s), PAD_ID, dtype=torch.long)
374
+ tgt_in = torch.full((B, max_t), PAD_ID, dtype=torch.long)
375
+ labels = torch.full((B, max_t), -100, dtype=torch.long)
376
+ for i, (s, t) in enumerate(batch):
377
+ sl = min(len(s), max_s)
378
+ tl = min(len(t) - 1, max_t)
379
+ src_ids[i, :sl] = torch.from_numpy(s[:sl])
380
+ tgt_in[i, :tl] = torch.from_numpy(t[:tl])
381
+ labels[i, :tl] = torch.from_numpy(t[1 : tl + 1])
382
+ return src_ids, tgt_in, labels
383
+
384
+ AUDIT_SAMPLES = [
385
+ ("En->My Formal", "en2my", "The central bank raised interest rates to control inflation and stabilize the economy."),
386
+ ("En->My Spoken", "en2my", "Honestly, I didn't expect things to turn out like this, but don't worry—we'll figure it out together."),
387
+ ("My->En Formal", "my2en", "ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးတည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။"),
388
+ ("My->En Spoken", "my2en", "အချစ်ရေ မင်းနဲ့ ငါနဲ့ သိပ်ချစ်ခဲ့တာ ���ုရားလည်း သိတယ်၊ သကြားလည်း သိတယ်၊ နတ်လည်း သိတယ်၊ မနာလိုသူတွေ ဂျင်းထည့်တာ ယုံနေတော့ မင်း ရူးသွားပြီပဲ။"),
389
+ ]
390
+
391
+ UNSEEN_TESTS = [
392
+ ("En->My News", "en2my", "Heavy monsoon rains caused severe flooding in several townships, forcing thousands of residents to evacuate to temporary shelters."),
393
+ ("En->My Health", "en2my", "Doctors recommend drinking plenty of clean water and getting enough rest to recover quickly from seasonal flu."),
394
+ ("En->My Spoken", "en2my", "I was stuck in traffic for over an hour this morning, so please start the meeting without me."),
395
+ ("My->En News", "my2en", "နိုင်ငံတကာ ကုန်သွယ်ရေး တိုးတက်လာစေရန်အတွက် နယ်စပ်ကုန်သွယ်ရေးစခန်းများတွင် ကုန်ပစ္စည်းစစ်ဆေးမှုများကို ပိုမိုမြန်ဆန်အောင် ဆောင်ရွက်နေပါသည်။"),
396
+ ("My->En Tech", "my2en", "ဉာဏ်ရည်တုနည်းပညာသည် ပညာရေး၊ ကျန်းမာရေးနှင့် စီးပွားရေးလုပ်ငန်းများတွင် လုပ်ငန်းစဉ်များကို ပိုမိုထိရောက်စေရန် ကူညီပေးနိုင်သည်။"),
397
+ ("My->En Spoken", "my2en", "မနက်ဖြန် မနက်စောစော သွားစရာရှိလို့ ဒီညတော့ စောစောအိပ်တော့မယ်နော်၊ မနက်ကျမှ ဖုန်းဆက်လိုက်မယ်။"),
398
+ ]
399
+
400
+ SPOKEN_IDIOM_PAIRS_PRO = [
401
+ ("my2en", "အချစ်ရေ မင်းနဲ့ ငါနဲ့ သိပ်ချစ်ခဲ့တာ ဘုရားလည်း သိတယ်၊ သကြားလည်း သိတယ်၊ နတ်လည်း သိတယ်၊ မနာလိုသူတွေ ဂျင်းထည့်တာ ယုံနေတော့ မင်း ရူးသွားပြီပဲ။",
402
+ "My love, heaven and earth know how deeply you and I loved each other, yet because you believed the lies fed by jealous people, you've lost your mind."),
403
+ ("my2en", "အချစ်ရေ မင်းနဲ့ ငါနဲ့ သိပ်ချစ်ခဲ့တာ ဘုရားလည်း သိတယ်၊ သိကြားလည်း သိတယ်၊ နတ်လည်း သိတယ်၊ မနာလိုသူတွေ ဂျင်းထည့်တာ ယုံနေတော့ မင်း ယူသွားခဲ့တာပဲ။",
404
+ "My love, Buddha, Sakra, and all the spirits know how deeply you and I loved each other, but because you believed the lies stirred up by jealous people, you ended up marrying someone else."),
405
+ ("my2en", "သူများတွေ ဂျင်းထည့်တာကို လွယ်လွယ်မယုံပါနဲ့၊ ကိုယ်တိုင်သေချာစုံစမ်းပြီးမှ ဆုံးဖြတ်ပါ။",
406
+ "Don't easily believe it when people try to trick you or feed you lies; look into it carefully yourself before deciding."),
407
+ ("my2en", "ငါ မင်းအပေါ် ဘယ်လောက်သစ္စာရှိခဲ့လဲဆိုတာ ဘုရားသိ သိကြားသိပါကွာ။",
408
+ "Heaven and earth know just how loyal I have been to you."),
409
+ ("my2en", "မနာလိုတဲ့လူတွေ ဘေးကနေ သွေးထိုးပြီး ဂျင်းထည့်နေတာကို မင်း ဘာလို့ နားယောင်နေရတာလဲ။",
410
+ "Why are you falling for the lies and manipulation of jealous people trying to stir up trouble from the sidelines?"),
411
+ ("my2en", "စိတ်မပူပါနဲ့ သူငယ်ချင်းရာ၊ အခက်အခဲဆိုတာ ခဏပဲ၊ ငါတို့ အတူတူ ရင်ဆိုင်ကြတာပေါ့။",
412
+ "Don't worry, my friend—hardships are only temporary, and we'll face this together."),
413
+ ("my2en", "နေကောင်းရဲ့လား မိတ်ဆွေ၊ မတွေ့ရတာတောင် တော်တော်ကြာပြီနော်။",
414
+ "How have you been doing, my friend? It's been quite a long time since we last met."),
415
+ ("en2my", "Honestly, I didn't expect things to turn out like this, but don't worry—we'll figure it out together.",
416
+ "ရိုးရိုးသားသားပြောရရင် ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ထားခဲ့ပေမယ့် စိတ်မပူပါနဲ့၊ ကျွန်တော်တို့ အတူတူ အဖြေရှာကြတာပေါ့။"),
417
+ ("en2my", "Don't let jealous people trick you or feed you lies; trust what we've built together.",
418
+ "မနာလိုသူတွေ ဂျင်းထည့်တာကို မယုံမိပါစေနဲ့၊ ငါတို့နှစ်ယောက် အတူတူ တည်ဆောက်ခဲ့တဲ့ အရာကိုပဲ ယုံကြည်ပေးပါ။"),
419
+ ("en2my", "The central bank raised interest rates to control inflation and stabilize the economy.",
420
+ "ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးတည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။"),
421
+ ("my2en", "ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးတည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။",
422
+ "The central bank raised interest rates to control inflation and stabilize the economy."),
423
+ ("en2my", "I was stuck in traffic for over an hour this morning, so please start the meeting without me.",
424
+ "ဒီမနက် တစ်နာရီကျော်လောက် ကားပိတ်မိနေလို့ ကျွန်တော်မပါဘဲ အစည်းအဝေးကို စထားလိုက်ပါနော်။"),
425
+ ("en2my", "Please go ahead and have dinner without me; I'll be home a bit late tonight.",
426
+ "ဒီည အိမ်ပြန်နည်းနည်းနောက်ကျမှာမို့လို့ ကျွန်တော်မပါဘဲ ညစာ အရင်စားနှင့်ကြပါနော်။"),
427
+ ]
428
+
429
+ # ── 3. Initialize 210M-Pro & Run All-Layer Net2Wider+Net2Deeper Transfer ──
430
+ pro_model = MyanmarTranslator84M(
431
+ vocab_size=VOCAB_SIZE, d_model=768, n_heads=12, enc_layers=8, dec_layers=8, ffn_dim=2816, max_seq_len=1024
432
+ ).to(device)
433
+
434
+ n_pro_params = sum(p.numel() for p in pro_model.parameters())
435
+ log(f"🏛️ MyanmarTranslator-210M-Pro Initialized: {n_pro_params:,} parameters ({n_pro_params/1e6:.2f}M) | 8L Encoder + 8L Decoder (d=768)")
436
+
437
+ def transfer_84m_to_210m(student: nn.Module, teacher_ckpt_path: str):
438
+ if not os.path.exists(teacher_ckpt_path):
439
+ log("⚠️ Teacher checkpoint not found, using standard initialization.")
440
+ return
441
+ t_sd = torch.load(teacher_ckpt_path, map_location=device)["model"]
442
+ scale_d = math.sqrt(512.0 / 768.0)
443
+
444
+ with torch.no_grad():
445
+ student.embed_tokens.weight.data.normal_(0.0, 0.004)
446
+ student.embed_tokens.weight.data[:, :512].copy_(t_sd["embed_tokens.weight"] * scale_d)
447
+ student.embed_tokens.weight.data[PAD_ID].zero_()
448
+
449
+ def copy_rmsnorm(s_norm, prefix):
450
+ s_norm.weight.data.fill_(1.0)
451
+ s_norm.weight.data[:512].copy_(t_sd[f"{prefix}.weight"] * scale_d)
452
+
453
+ def copy_self_attn(s_attn, prefix):
454
+ s_attn.qkv.weight.data.normal_(0.0, 0.008)
455
+ s_attn.out.weight.data.normal_(0.0, 0.002)
456
+ tq, tk, tv = t_sd[f"{prefix}.qkv.weight"].chunk(3, dim=0)
457
+ sq, sk, sv = s_attn.qkv.weight.data.chunk(3, dim=0)
458
+ sq[:512, :512].copy_(tq)
459
+ sk[:512, :512].copy_(tk)
460
+ sv[:512, :512].copy_(tv)
461
+ s_attn.out.weight.data[:512, :512].copy_(t_sd[f"{prefix}.out.weight"])
462
+ s_attn.q_norm.weight.data.copy_(t_sd[f"{prefix}.q_norm.weight"])
463
+ s_attn.k_norm.weight.data.copy_(t_sd[f"{prefix}.k_norm.weight"])
464
+
465
+ def copy_cross_attn(s_attn, prefix):
466
+ s_attn.q_proj.weight.data.normal_(0.0, 0.008)
467
+ s_attn.kv_proj.weight.data.normal_(0.0, 0.008)
468
+ s_attn.out.weight.data.normal_(0.0, 0.002)
469
+ s_attn.q_proj.weight.data[:512, :512].copy_(t_sd[f"{prefix}.q_proj.weight"])
470
+ tk, tv = t_sd[f"{prefix}.kv_proj.weight"].chunk(2, dim=0)
471
+ sk, sv = s_attn.kv_proj.weight.data.chunk(2, dim=0)
472
+ sk[:512, :512].copy_(tk)
473
+ sv[:512, :512].copy_(tv)
474
+ s_attn.out.weight.data[:512, :512].copy_(t_sd[f"{prefix}.out.weight"])
475
+ s_attn.q_norm.weight.data.copy_(t_sd[f"{prefix}.q_norm.weight"])
476
+ s_attn.k_norm.weight.data.copy_(t_sd[f"{prefix}.k_norm.weight"])
477
+
478
+ def copy_swiglu(s_mlp, prefix):
479
+ s_mlp.gate_up.weight.data.normal_(0.0, 0.008)
480
+ s_mlp.down.weight.data.normal_(0.0, 0.002)
481
+ tg, tu = t_sd[f"{prefix}.gate_up.weight"].chunk(2, dim=0)
482
+ sg, su = s_mlp.gate_up.weight.data.chunk(2, dim=0)
483
+ sg[:1728, :512].copy_(tg)
484
+ su[:1728, :512].copy_(tu)
485
+ s_mlp.down.weight.data[:512, :1728].copy_(t_sd[f"{prefix}.down.weight"])
486
+
487
+ for i in range(6):
488
+ copy_rmsnorm(student.encoder[i].norm1, f"encoder.{i}.norm1")
489
+ copy_self_attn(student.encoder[i].self_attn, f"encoder.{i}.self_attn")
490
+ copy_rmsnorm(student.encoder[i].norm2, f"encoder.{i}.norm2")
491
+ copy_swiglu(student.encoder[i].mlp, f"encoder.{i}.mlp")
492
+ for i in range(6, 8):
493
+ student.encoder[i].self_attn.out.weight.data.normal_(0.0, 0.001)
494
+ student.encoder[i].mlp.down.weight.data.normal_(0.0, 0.001)
495
+ copy_rmsnorm(student.enc_norm, "enc_norm")
496
+
497
+ for i in range(6):
498
+ copy_rmsnorm(student.decoder[i].norm1, f"decoder.{i}.norm1")
499
+ copy_self_attn(student.decoder[i].self_attn, f"decoder.{i}.self_attn")
500
+ copy_rmsnorm(student.decoder[i].norm2, f"decoder.{i}.norm2")
501
+ copy_cross_attn(student.decoder[i].cross_attn, f"decoder.{i}.cross_attn")
502
+ copy_rmsnorm(student.decoder[i].norm3, f"decoder.{i}.norm3")
503
+ copy_swiglu(student.decoder[i].mlp, f"decoder.{i}.mlp")
504
+ for i in range(6, 8):
505
+ student.decoder[i].self_attn.out.weight.data.normal_(0.0, 0.001)
506
+ student.decoder[i].cross_attn.out.weight.data.normal_(0.0, 0.001)
507
+ student.decoder[i].mlp.down.weight.data.normal_(0.0, 0.001)
508
+ copy_rmsnorm(student.dec_norm, "dec_norm")
509
+
510
+ del t_sd
511
+ gc.collect()
512
+ torch.cuda.empty_cache()
513
+ log("✅ Full All-Layer Net2Wider + Net2Deeper Transfer (84M -> 210M) Complete!")
514
+
515
+ transfer_84m_to_210m(pro_model, TEACHER_84M)
516
+
517
+ # ── 4. DataLoader, Optimizer & Schedule (24,000 Expansion Steps + 1,000 Cooldown Steps) ──
518
+ MICRO_BS_PRO = 128
519
+ GRAD_ACCUM_PRO = 2 # Global BS = 256 sequences/step (~80.6k tokens/step)
520
+ MAX_TOK_CHUNK_PRO = 16384 # Caps peak VRAM at ~17-19 GB on RTX 5090 (zero OOM risk!)
521
+ train_ds_pro = Seq2SeqMemmapDataset(DATA_DIR, "train")
522
+ val_ds_pro = Seq2SeqMemmapDataset(DATA_DIR, "val")
523
+
524
+ STAGE1_STEPS_PRO = 24000 # ~2.0B expansion tokens starting from Val PPL 6.73 (= 58,000 effective steps!)
525
+ WARMUP_STEPS_PRO = 1000
526
+ PEAK_LR_PRO = 3.0e-4
527
+ MIN_LR_PRO = 3.0e-5
528
+ EVAL_EVERY_PRO = 2000
529
+ LOG_EVERY_PRO = 200
530
+
531
+ decay_pro = [p for n, p in pro_model.named_parameters() if p.requires_grad and p.ndim >= 2]
532
+ nodecay_pro = [p for n, p in pro_model.named_parameters() if p.requires_grad and p.ndim < 2]
533
+ pro_opt = torch.optim.AdamW(
534
+ [{"params": decay_pro, "weight_decay": 0.01}, {"params": nodecay_pro, "weight_decay": 0.0}],
535
+ lr=PEAK_LR_PRO, betas=(0.9, 0.98), eps=1e-8, fused=True
536
+ )
537
+
538
+ def get_pro_lr(step: int) -> float:
539
+ if step < WARMUP_STEPS_PRO:
540
+ return PEAK_LR_PRO * (step + 1) / WARMUP_STEPS_PRO
541
+ prog = min(1.0, (step - WARMUP_STEPS_PRO) / max(1, STAGE1_STEPS_PRO - WARMUP_STEPS_PRO))
542
+ return MIN_LR_PRO + 0.5 * (PEAK_LR_PRO - MIN_LR_PRO) * (1.0 + math.cos(math.pi * prog))
543
+
544
+ start_step_pro = 0
545
+ best_pro_nll = float("inf")
546
+ latest_pro_ckpt = os.path.join(PRO_CKPT, "latest_pro.pt")
547
+ if os.path.exists(latest_pro_ckpt):
548
+ ckpt_p = torch.load(latest_pro_ckpt, map_location=device)
549
+ pro_model.load_state_dict(ckpt_p["model"])
550
+ pro_opt.load_state_dict(ckpt_p["optimizer"])
551
+ start_step_pro = int(ckpt_p["step"])
552
+ best_pro_nll = float(ckpt_p.get("best_val_nll", float("inf")))
553
+ log(f"🔄 Resumed 210M-Pro from step {start_step_pro:,} (best_val_nll={best_pro_nll:.4f})")
554
+
555
+ train_sampler_pro = LengthBucketBatchSampler(
556
+ train_ds_pro.index, batch_size=MICRO_BS_PRO, shuffle=True, seed=2026, start_batch=start_step_pro * GRAD_ACCUM_PRO
557
+ )
558
+ val_sampler_pro = LengthBucketBatchSampler(
559
+ val_ds_pro.index, batch_size=MICRO_BS_PRO, shuffle=True, seed=12345, start_batch=0
560
+ )
561
+
562
+ train_loader_pro = DataLoader(train_ds_pro, batch_sampler=train_sampler_pro, collate_fn=collate_nmt, num_workers=6, pin_memory=True, persistent_workers=True)
563
+ val_loader_pro = DataLoader(val_ds_pro, batch_sampler=val_sampler_pro, collate_fn=collate_nmt, num_workers=2, pin_memory=True)
564
+
565
+ @torch.no_grad()
566
+ def evaluate_pro(max_batches: int = 80):
567
+ pro_model.eval()
568
+ tot_nll, tot_tok = 0.0, 0
569
+ for i, (src_ids, tgt_in, labels) in enumerate(val_loader_pro):
570
+ if i >= max_batches:
571
+ break
572
+ B, S = src_ids.shape
573
+ T = tgt_in.size(1)
574
+ sub_bs = max(4, min(B, MAX_TOK_CHUNK_PRO // max(S, T, 1)))
575
+ for s_i in range(0, B, sub_bs):
576
+ sc = src_ids[s_i : s_i + sub_bs].to(device, non_blocking=True)
577
+ tc = tgt_in[s_i : s_i + sub_bs].to(device, non_blocking=True)
578
+ yc = labels[s_i : s_i + sub_bs].to(device, non_blocking=True)
579
+ nll_s, n_v = pro_model.eval_nll(sc, tc, yc)
580
+ tot_nll += nll_s
581
+ tot_tok += n_v
582
+ avg_nll = tot_nll / max(tot_tok, 1)
583
+ return avg_nll, math.exp(min(avg_nll, 20.0))
584
+
585
+ if start_step_pro == 0:
586
+ init_pro_nll, init_pro_ppl = evaluate_pro()
587
+ best_pro_nll = init_pro_nll
588
+ log(f"🔎 Step 0 (Post-Net2Wider/Deeper Warm-Start) | Val NLL: {init_pro_nll:.4f} | Val PPL: {init_pro_ppl:.2f}")
589
+
590
+ encoded_idioms_pro = []
591
+ for d_code, s_txt, t_txt in SPOKEN_IDIOM_PAIRS_PRO:
592
+ dir_tok = TOK_2MY if d_code == "en2my" else TOK_2EN
593
+ s_arr = np.asarray([dir_tok] + sp.encode_as_ids(s_txt.strip()) + [EOS_ID], dtype=np.int64)
594
+ t_arr = np.asarray([BOS_ID] + sp.encode_as_ids(t_txt.strip()) + [EOS_ID], dtype=np.int64)
595
+ encoded_idioms_pro.append((s_arr, t_arr))
596
+
597
+ # ── 5. Stage 1 Expansion Training Loop (24,000 Steps) ──
598
+ log(f"\n>>> Starting Flagship MyanmarTranslator-210M-Pro Stage 1 (Step {start_step_pro:,} -> {STAGE1_STEPS_PRO:,})...")
599
+ pro_model.train()
600
+ pro_opt.zero_grad(set_to_none=True)
601
+
602
+ step = start_step_pro
603
+ micro_idx = start_step_pro * GRAD_ACCUM_PRO
604
+ run_loss = run_toks = 0.0
605
+ t_start = time.time()
606
+
607
+ for src_ids, tgt_in, labels in train_loader_pro:
608
+ micro_idx += 1
609
+ B, S = src_ids.shape
610
+ T = tgt_in.size(1)
611
+ max_len = max(S, T)
612
+ sub_bs = max(4, min(B, MAX_TOK_CHUNK_PRO // max(max_len, 1)))
613
+ n_subs = math.ceil(B / sub_bs)
614
+
615
+ for s_i in range(0, B, sub_bs):
616
+ s_chunk = src_ids[s_i : s_i + sub_bs].to(device, non_blocking=True)
617
+ t_chunk = tgt_in[s_i : s_i + sub_bs].to(device, non_blocking=True)
618
+ y_chunk = labels[s_i : s_i + sub_bs].to(device, non_blocking=True)
619
+
620
+ loss_val, n_val = pro_model.forward_and_backward(
621
+ s_chunk, t_chunk, y_chunk, loss_scale=(GRAD_ACCUM_PRO * n_subs), label_smoothing=0.1
622
+ )
623
+ run_loss += loss_val / n_subs
624
+ run_toks += n_val + int((s_chunk != PAD_ID).sum().item())
625
+
626
+ if micro_idx % GRAD_ACCUM_PRO == 0:
627
+ lr = get_pro_lr(step)
628
+ for pg in pro_opt.param_groups:
629
+ pg["lr"] = lr
630
+ grad_norm = torch.nn.utils.clip_grad_norm_(pro_model.parameters(), 1.0)
631
+ pro_opt.step()
632
+ pro_opt.zero_grad(set_to_none=True)
633
+ step += 1
634
+
635
+ if step % LOG_EVERY_PRO == 0:
636
+ elapsed = time.time() - t_start
637
+ avg_l = run_loss / (LOG_EVERY_PRO * GRAD_ACCUM_PRO)
638
+ est_nll = max(0.0, (avg_l - 0.1 * math.log(VOCAB_SIZE)) / 0.9)
639
+ tok_s = run_toks / max(elapsed, 1e-3)
640
+ peak_gb = torch.cuda.max_memory_allocated() / 1e9
641
+ log(f"Step {step:6d}/{STAGE1_STEPS_PRO} | SmoothLoss: {avg_l:.4f} | EstNLL: {est_nll:.4f} (PPL {math.exp(min(est_nll,15)):6.2f}) | LR: {lr:.2e} | Grad: {grad_norm:.2f} | {tok_s/1e3:.1f}k tok/s | VRAM: {peak_gb:.1f}G")
642
+ run_loss = run_toks = 0.0
643
+ t_start = time.time()
644
+
645
+ if step % EVAL_EVERY_PRO == 0 or step == STAGE1_STEPS_PRO:
646
+ val_nll, val_ppl = evaluate_pro()
647
+ log("\n" + "-"*85)
648
+ log(f"🔎 210M-PRO EVAL @ Step {step:,} | Val NLL: {val_nll:.4f} | Val PPL: {val_ppl:.2f}")
649
+ for tag, d_code, prompt_txt in AUDIT_SAMPLES:
650
+ pred_txt = pro_model.translate(prompt_txt, d_code)
651
+ log(f" [{tag}]: {pred_txt}")
652
+
653
+ torch.save({"step": step, "model": pro_model.state_dict(), "optimizer": pro_opt.state_dict(), "best_val_nll": best_pro_nll, "config": pro_model.config}, latest_pro_ckpt)
654
+
655
+ if val_nll < best_pro_nll:
656
+ best_pro_nll = val_nll
657
+ torch.save({"step": step, "model": pro_model.state_dict(), "val_nll": val_nll, "config": pro_model.config}, os.path.join(PRO_CKPT, "best_pro.pt"))
658
+ log(f" 🌟 New Best 210M-Pro Checkpoint Saved! (Val NLL: {best_pro_nll:.4f})")
659
+ log("-"*85 + "\n")
660
+ pro_model.train()
661
+ t_start = time.time()
662
+
663
+ if step >= STAGE1_STEPS_PRO:
664
+ break
665
+
666
+ # ── 6. Stage 2 Unsmoothed Cooldown & Idiom Polish (1,000 Steps, label_smoothing=0.0) ──
667
+ log(f"\n>>> Starting 210M-Pro Stage 2 Cooldown (1,000 steps | label_smoothing=0.0 | LR 3e-5 -> 4e-6)...")
668
+ best_pro_s1 = torch.load(os.path.join(PRO_CKPT, "best_pro.pt"), map_location=device)
669
+ pro_model.load_state_dict(best_pro_s1["model"])
670
+
671
+ pro_s2_opt = torch.optim.AdamW(
672
+ [{"params": decay_pro, "weight_decay": 0.01}, {"params": nodecay_pro, "weight_decay": 0.0}],
673
+ lr=3e-5, betas=(0.9, 0.98), eps=1e-8, fused=True
674
+ )
675
+ clean_mask_pro = (train_ds_pro.index[:, 4] < 2) & (train_ds_pro.index[:, 1] >= 16) & (train_ds_pro.index[:, 1] <= 256)
676
+ clean_idx_pro = np.flatnonzero(clean_mask_pro)
677
+ rng_pro = np.random.RandomState(2026)
678
+
679
+ pro_model.train()
680
+ for s2_step in range(1, 1001):
681
+ prog = s2_step / 1000.0
682
+ lr = 4e-6 + 0.5 * (3e-5 - 4e-6) * (1.0 + math.cos(math.pi * prog))
683
+ for pg in pro_s2_opt.param_groups:
684
+ pg["lr"] = lr
685
+
686
+ picked = rng_pro.choice(clean_idx_pro, size=88, replace=False)
687
+ batch_items = [train_ds_pro[int(ix)] for ix in picked]
688
+ for _ in range(8):
689
+ batch_items.append(encoded_idioms_pro[int(rng_pro.randint(len(encoded_idioms_pro)))])
690
+
691
+ src_ids, tgt_in, labels = collate_nmt(batch_items)
692
+ pro_s2_opt.zero_grad(set_to_none=True)
693
+ pro_model.forward_and_backward(
694
+ src_ids.to(device, non_blocking=True),
695
+ tgt_in.to(device, non_blocking=True),
696
+ labels.to(device, non_blocking=True),
697
+ loss_scale=1.0, label_smoothing=0.0
698
+ )
699
+ torch.nn.utils.clip_grad_norm_(pro_model.parameters(), 1.0)
700
+ pro_s2_opt.step()
701
+
702
+ if s2_step % 250 == 0:
703
+ val_nll, val_ppl = evaluate_pro()
704
+ log(f"🔎 210M-Pro Stage 2 Step {s2_step}/1000 | Val NLL: {val_nll:.4f} (PPL {val_ppl:.2f})")
705
+ if val_nll <= best_pro_nll + 0.01:
706
+ best_pro_nll = min(best_pro_nll, val_nll)
707
+ torch.save({"step": STAGE1_STEPS_PRO + s2_step, "model": pro_model.state_dict(), "val_nll": val_nll, "config": pro_model.config},
708
+ os.path.join(PRO_CKPT, "best_pro.pt"))
709
+ pro_model.train()
710
+
711
+ # ── 7. Final Benchmark & Private HF Upload: freococo/MyanmarTranslator-210M-Pro ──
712
+ best_pro_final = torch.load(os.path.join(PRO_CKPT, "best_pro.pt"), map_location=device)
713
+ pro_model.load_state_dict(best_pro_final["model"])
714
+ pro_model.eval()
715
+ final_pro_nll = float(best_pro_final["val_nll"])
716
+ final_pro_ppl = math.exp(final_pro_nll)
717
+
718
+ log("\n" + "="*90)
719
+ log(f"🏆 MYANMARTRANSLATOR-210M-PRO FINAL BENCHMARK | Val NLL: {final_pro_nll:.4f} (PPL {final_pro_ppl:.2f})")
720
+ log("="*90)
721
+ for tag, d_code, prompt_txt in AUDIT_SAMPLES + UNSEEN_TESTS:
722
+ pred_txt = pro_model.translate_beam(prompt_txt, d_code, num_beams=4)
723
+ log(f" [{tag} (Beam=4)]: {pred_txt}")
724
+ log("="*90)
725
+
726
+ CKPT_SUB_PRO = os.path.join(PRO_EXPORT, "checkpoint-step-25000")
727
+ shutil.rmtree(PRO_EXPORT, ignore_errors=True)
728
+ os.makedirs(CKPT_SUB_PRO, exist_ok=True)
729
+
730
+ sd_fp32 = {k: v.detach().cpu().float().contiguous() for k, v in pro_model.state_dict().items() if k != "lm_head.weight"}
731
+ sd_bf16 = {k: v.to(torch.bfloat16).contiguous() for k, v in sd_fp32.items()}
732
+ save_file(sd_bf16, os.path.join(PRO_EXPORT, "model.safetensors"))
733
+ save_file(sd_fp32, os.path.join(CKPT_SUB_PRO, "model.safetensors"))
734
+
735
+ if os.path.exists(latest_pro_ckpt):
736
+ os.remove(latest_pro_ckpt)
737
+ torch.save(best_pro_final, os.path.join(CKPT_SUB_PRO, "training_state.pt"))
738
+
739
+ BASE_EXPORT = "/workspace/MyanmarTranslator-84M-Base-export"
740
+ for folder, dt in [(PRO_EXPORT, "bfloat16"), (CKPT_SUB_PRO, "float32")]:
741
+ for fn in ["configuration_myanmartranslator.py", "modeling_myanmartranslator.py", "generation_config.json",
742
+ "tokenizer.model", "tokenizer_config.json", "special_tokens_map.json"]:
743
+ shutil.copyfile(os.path.join(BASE_EXPORT, fn), os.path.join(folder, fn))
744
+ with open(os.path.join(BASE_EXPORT, "config.json"), "r") as f:
745
+ cfg_d = json.load(f)
746
+ cfg_d.update(pro_model.config)
747
+ cfg_d["torch_dtype"] = dt
748
+ cfg_d["val_nll"] = round(final_pro_nll, 4)
749
+ cfg_d["val_ppl"] = round(final_pro_ppl, 2)
750
+ with open(os.path.join(folder, "config.json"), "w", encoding="utf-8") as f:
751
+ json.dump(cfg_d, f, indent=2)
752
+
753
+ full_output_log = "\n".join(LOG_LINES) + "\n"
754
+ for folder in (PRO_EXPORT, CKPT_SUB_PRO):
755
+ with open(os.path.join(folder, "training_codes.txt"), "w", encoding="utf-8") as f:
756
+ f.write(CELL_SOURCE_CODE)
757
+ with open(os.path.join(folder, "trainingcelloutputwitheval.txt"), "w", encoding="utf-8") as f:
758
+ f.write(full_output_log)
759
+
760
+ with open(os.path.join(PRO_EXPORT, "README.md"), "w", encoding="utf-8") as f:
761
+ f.write(f"""---
762
+ language:
763
+ - my
764
+ - en
765
+ license: apache-2.0
766
+ library_name: transformers
767
+ tags:
768
+ - Translation
769
+ - Safetensors
770
+ - Burmese
771
+ - English
772
+ - MyanmarLLM
773
+ - flagship
774
+ - custom_code
775
+ ---
776
+
777
+ # MyanmarTranslator-210M-Pro
778
+
779
+ Flagship `210.90M`-parameter **8-Layer Bidirectional Encoder + 8-Layer Cross-Attentive Decoder** (`d_model=768`, `n_heads=12`, `ffn_dim=2816` SwiGLU) for **English ↔ Burmese (`my`)**, warm-started via All-Layer Net2Wider+Net2Deeper expansion from `MyanmarTranslator-84M-Base`.
780
+
781
+ - **Validation NLL:** `{final_pro_nll:.4f}` (**`Val PPL: {final_pro_ppl:.2f}`**)
782
+ - **Reproducibility Artifacts:** `training_codes.txt` & `trainingcelloutputwitheval.txt`
783
+ """)
784
+
785
+ api = HfApi()
786
+ create_repo(PRO_REPO_ID, repo_type="model", private=True, exist_ok=True)
787
+ api.upload_folder(
788
+ folder_path=PRO_EXPORT,
789
+ repo_id=PRO_REPO_ID,
790
+ repo_type="model",
791
+ commit_message=f"Release Flagship MyanmarTranslator-210M-Pro (Val PPL {final_pro_ppl:.2f} + code & logs)",
792
+ )
793
+ log(f"\n🔒🚀 SUCCESS! Flagship MyanmarTranslator-210M-Pro Live at: https://huggingface.co/{PRO_REPO_ID}")
checkpoint-step-25000/training_state.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cc708704687398832cefdda6823e87e38b0e17f3b673515bec264ca077fb3a88
3
+ size 843659496
checkpoint-step-25000/trainingcelloutputwitheval.txt ADDED
@@ -0,0 +1,264 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 🟢 Standalone Kernel Ready for 210M-Pro | GPU Free: 33.14/33.67 GB | Disk Free: 46.2 GB
2
+ 🏛️ MyanmarTranslator-210M-Pro Initialized: 210,897,408 parameters (210.90M) | 8L Encoder + 8L Decoder (d=768)
3
+ ✅ Full All-Layer Net2Wider + Net2Deeper Transfer (84M -> 210M) Complete!
4
+ 🔎 Step 0 (Post-Net2Wider/Deeper Warm-Start) | Val NLL: 1.9073 | Val PPL: 6.73
5
+
6
+ >>> Starting Flagship MyanmarTranslator-210M-Pro Stage 1 (Step 0 -> 24,000)...
7
+ Step 200/24000 | SmoothLoss: 2.8552 | EstNLL: 1.9399 (PPL 6.96) | LR: 6.00e-05 | Grad: 0.35 | 87.8k tok/s | VRAM: 21.3G
8
+ Step 400/24000 | SmoothLoss: 2.8429 | EstNLL: 1.9263 (PPL 6.86) | LR: 1.20e-04 | Grad: 0.36 | 87.8k tok/s | VRAM: 21.3G
9
+ Step 600/24000 | SmoothLoss: 2.8464 | EstNLL: 1.9302 (PPL 6.89) | LR: 1.80e-04 | Grad: 0.33 | 88.0k tok/s | VRAM: 21.3G
10
+ Step 800/24000 | SmoothLoss: 2.8775 | EstNLL: 1.9648 (PPL 7.13) | LR: 2.40e-04 | Grad: 0.37 | 88.8k tok/s | VRAM: 21.3G
11
+ Step 1000/24000 | SmoothLoss: 2.8745 | EstNLL: 1.9614 (PPL 7.11) | LR: 3.00e-04 | Grad: 0.42 | 89.1k tok/s | VRAM: 21.3G
12
+ Step 1200/24000 | SmoothLoss: 2.8935 | EstNLL: 1.9825 (PPL 7.26) | LR: 3.00e-04 | Grad: 0.45 | 89.0k tok/s | VRAM: 21.3G
13
+ Step 1400/24000 | SmoothLoss: 2.8996 | EstNLL: 1.9892 (PPL 7.31) | LR: 3.00e-04 | Grad: 0.26 | 88.9k tok/s | VRAM: 21.3G
14
+ Step 1600/24000 | SmoothLoss: 2.9059 | EstNLL: 1.9963 (PPL 7.36) | LR: 3.00e-04 | Grad: 0.29 | 88.3k tok/s | VRAM: 21.3G
15
+ Step 1800/24000 | SmoothLoss: 2.9176 | EstNLL: 2.0093 (PPL 7.46) | LR: 2.99e-04 | Grad: 0.37 | 88.9k tok/s | VRAM: 21.3G
16
+ Step 2000/24000 | SmoothLoss: 2.9026 | EstNLL: 1.9926 (PPL 7.33) | LR: 2.99e-04 | Grad: 0.31 | 88.3k tok/s | VRAM: 21.3G
17
+
18
+ -------------------------------------------------------------------------------------
19
+ 🔎 210M-PRO EVAL @ Step 2,000 | Val NLL: 1.9182 | Val PPL: 6.81
20
+ [En->My Formal]: ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးတည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
21
+ [En->My Spoken]: အမှန်တော့ ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ထားဘဲ စိတ်မပူပါနဲ့ အတူတူတွေးကြည့်ကြတာပေါ့။
22
+ [My->En Formal]: The central bank raised interest rates to control inflation and stabilize the economy.
23
+ [My->En Spoken]: Love, you and I loved each other so much that they knew everything about it—buddha knows it, sugar knows it, and spirits know it. Those who are jealous of me believe in being deceived, so you've gone mad.
24
+ -------------------------------------------------------------------------------------
25
+
26
+ Step 2200/24000 | SmoothLoss: 2.9118 | EstNLL: 2.0029 (PPL 7.41) | LR: 2.98e-04 | Grad: 0.28 | 88.1k tok/s | VRAM: 21.3G
27
+ Step 2400/24000 | SmoothLoss: 2.9170 | EstNLL: 2.0087 (PPL 7.45) | LR: 2.98e-04 | Grad: 0.24 | 88.9k tok/s | VRAM: 21.3G
28
+ Step 2600/24000 | SmoothLoss: 2.9014 | EstNLL: 1.9913 (PPL 7.32) | LR: 2.97e-04 | Grad: 0.37 | 87.5k tok/s | VRAM: 21.3G
29
+ Step 2800/24000 | SmoothLoss: 2.9057 | EstNLL: 1.9960 (PPL 7.36) | LR: 2.96e-04 | Grad: 0.57 | 88.1k tok/s | VRAM: 21.3G
30
+ Step 3000/24000 | SmoothLoss: 2.9090 | EstNLL: 1.9998 (PPL 7.39) | LR: 2.95e-04 | Grad: 0.36 | 88.2k tok/s | VRAM: 21.3G
31
+ Step 3200/24000 | SmoothLoss: 2.9009 | EstNLL: 1.9907 (PPL 7.32) | LR: 2.94e-04 | Grad: 0.30 | 88.6k tok/s | VRAM: 21.3G
32
+ Step 3400/24000 | SmoothLoss: 2.9048 | EstNLL: 1.9951 (PPL 7.35) | LR: 2.93e-04 | Grad: 0.31 | 89.2k tok/s | VRAM: 21.3G
33
+ Step 3600/24000 | SmoothLoss: 2.9222 | EstNLL: 2.0144 (PPL 7.50) | LR: 2.92e-04 | Grad: 0.34 | 88.8k tok/s | VRAM: 21.3G
34
+ Step 3800/24000 | SmoothLoss: 2.9093 | EstNLL: 2.0001 (PPL 7.39) | LR: 2.90e-04 | Grad: 0.49 | 89.1k tok/s | VRAM: 21.3G
35
+ Step 4000/24000 | SmoothLoss: 2.9209 | EstNLL: 2.0130 (PPL 7.49) | LR: 2.89e-04 | Grad: 0.44 | 88.7k tok/s | VRAM: 21.3G
36
+
37
+ -------------------------------------------------------------------------------------
38
+ 🔎 210M-PRO EVAL @ Step 4,000 | Val NLL: 1.9249 | Val PPL: 6.85
39
+ [En->My Formal]: ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေး တည်ငြိမ်စေရန် အတိုးနှုန်းများ မြှင့်တင်ခဲ့သည်။
40
+ [En->My Spoken]: အမှန်ဆိုရရင် ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ထားပါဘူး၊ စိတ်မပူပါနဲ့၊ အတူတူစဉ်းစားကြည့်ကြမှာပဲ
41
+ [My->En Formal]: The central bank has raised interest rates to control inflation and stabilize the economy.
42
+ [My->En Spoken]: Love, you and I have loved each other so much that Buddha knows it, sugar knows it, celestial beings also know it. Those who envy are just jealous of me, so you've gone mad.
43
+ -------------------------------------------------------------------------------------
44
+
45
+ Step 4200/24000 | SmoothLoss: 2.9093 | EstNLL: 2.0001 (PPL 7.39) | LR: 2.87e-04 | Grad: 0.23 | 88.4k tok/s | VRAM: 21.3G
46
+ Step 4400/24000 | SmoothLoss: 2.9011 | EstNLL: 1.9909 (PPL 7.32) | LR: 2.86e-04 | Grad: 0.24 | 88.5k tok/s | VRAM: 21.3G
47
+ Step 4600/24000 | SmoothLoss: 2.9157 | EstNLL: 2.0072 (PPL 7.44) | LR: 2.84e-04 | Grad: 0.31 | 88.8k tok/s | VRAM: 21.3G
48
+ Step 4800/24000 | SmoothLoss: 2.9097 | EstNLL: 2.0005 (PPL 7.39) | LR: 2.82e-04 | Grad: 0.31 | 88.7k tok/s | VRAM: 21.3G
49
+ Step 5000/24000 | SmoothLoss: 2.9163 | EstNLL: 2.0078 (PPL 7.45) | LR: 2.80e-04 | Grad: 0.26 | 88.5k tok/s | VRAM: 21.3G
50
+ Step 5200/24000 | SmoothLoss: 2.8903 | EstNLL: 1.9789 (PPL 7.23) | LR: 2.78e-04 | Grad: 0.50 | 88.2k tok/s | VRAM: 21.3G
51
+ Step 5400/24000 | SmoothLoss: 2.9017 | EstNLL: 1.9916 (PPL 7.33) | LR: 2.76e-04 | Grad: 0.31 | 88.6k tok/s | VRAM: 21.3G
52
+ Step 5600/24000 | SmoothLoss: 2.9062 | EstNLL: 1.9966 (PPL 7.36) | LR: 2.74e-04 | Grad: 0.27 | 88.6k tok/s | VRAM: 21.3G
53
+ Step 5800/24000 | SmoothLoss: 2.8924 | EstNLL: 1.9813 (PPL 7.25) | LR: 2.72e-04 | Grad: 0.28 | 89.0k tok/s | VRAM: 21.3G
54
+ Step 6000/24000 | SmoothLoss: 2.8909 | EstNLL: 1.9797 (PPL 7.24) | LR: 2.70e-04 | Grad: 0.42 | 88.3k tok/s | VRAM: 21.3G
55
+
56
+ -------------------------------------------------------------------------------------
57
+ 🔎 210M-PRO EVAL @ Step 6,000 | Val NLL: 1.9110 | Val PPL: 6.76
58
+ [En->My Formal]: ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးကို တည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
59
+ [En->My Spoken]: တကယ်တော့ ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ထားဘဲ စိတ်မပူပါနဲ့ အတူတူတွေးကြည့်မှာပါ။
60
+ [My->En Formal]: The central bank has raised interest rates to control inflation and stabilize the economy.
61
+ [My->En Spoken]: In love, I knew that you and I loved each other very much, Buddha knows, sugar knows, and celestial beings know. Those who envy me are teasing you, so you've gone mad.
62
+ -------------------------------------------------------------------------------------
63
+
64
+ Step 6200/24000 | SmoothLoss: 2.8958 | EstNLL: 1.9851 (PPL 7.28) | LR: 2.67e-04 | Grad: 0.26 | 88.1k tok/s | VRAM: 21.3G
65
+ Step 6400/24000 | SmoothLoss: 2.9049 | EstNLL: 1.9951 (PPL 7.35) | LR: 2.65e-04 | Grad: 0.30 | 88.6k tok/s | VRAM: 21.3G
66
+ Step 6600/24000 | SmoothLoss: 2.8825 | EstNLL: 1.9703 (PPL 7.17) | LR: 2.62e-04 | Grad: 0.30 | 88.0k tok/s | VRAM: 21.3G
67
+ Step 6800/24000 | SmoothLoss: 2.8893 | EstNLL: 1.9778 (PPL 7.23) | LR: 2.60e-04 | Grad: 0.27 | 88.0k tok/s | VRAM: 21.3G
68
+ Step 7000/24000 | SmoothLoss: 2.9112 | EstNLL: 2.0022 (PPL 7.41) | LR: 2.57e-04 | Grad: 0.36 | 89.8k tok/s | VRAM: 21.3G
69
+ Step 7200/24000 | SmoothLoss: 2.8744 | EstNLL: 1.9613 (PPL 7.11) | LR: 2.54e-04 | Grad: 0.31 | 87.8k tok/s | VRAM: 21.3G
70
+ Step 7400/24000 | SmoothLoss: 2.8892 | EstNLL: 1.9777 (PPL 7.23) | LR: 2.52e-04 | Grad: 0.80 | 89.3k tok/s | VRAM: 21.3G
71
+ Step 7600/24000 | SmoothLoss: 2.8941 | EstNLL: 1.9832 (PPL 7.27) | LR: 2.49e-04 | Grad: 0.29 | 89.4k tok/s | VRAM: 21.3G
72
+ Step 7800/24000 | SmoothLoss: 2.8854 | EstNLL: 1.9735 (PPL 7.20) | LR: 2.46e-04 | Grad: 0.36 | 88.9k tok/s | VRAM: 21.3G
73
+ Step 8000/24000 | SmoothLoss: 2.8866 | EstNLL: 1.9748 (PPL 7.21) | LR: 2.43e-04 | Grad: 0.33 | 89.1k tok/s | VRAM: 21.3G
74
+
75
+ -------------------------------------------------------------------------------------
76
+ 🔎 210M-PRO EVAL @ Step 8,000 | Val NLL: 1.8961 | Val PPL: 6.66
77
+ [En->My Formal]: ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးကို တည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
78
+ [En->My Spoken]: တကယ်တော့လည်း ဒီလိုဖြစ်လာလိမ့်မယ်လို့ မမျှော်လင့်ထားပါဘူး၊ စိတ်မပူပါနဲ့၊ အတူတွေးကြည့်ကြမှာပဲ။
79
+ [My->En Formal]: The Central Bank has raised interest rates to control inflation and stabilize the economy.
80
+ [My->En Spoken]: Haha, I loved you and me very much. Buddha knows, sugar knows, spirits know knowing. Because those who envy are jealous of you, you have gone crazy.
81
+ 🌟 New Best 210M-Pro Checkpoint Saved! (Val NLL: 1.8961)
82
+ -------------------------------------------------------------------------------------
83
+
84
+ Step 8200/24000 | SmoothLoss: 2.8730 | EstNLL: 1.9598 (PPL 7.10) | LR: 2.40e-04 | Grad: 0.45 | 88.7k tok/s | VRAM: 21.3G
85
+ Step 8400/24000 | SmoothLoss: 2.8723 | EstNLL: 1.9590 (PPL 7.09) | LR: 2.37e-04 | Grad: 0.35 | 87.7k tok/s | VRAM: 21.3G
86
+ Step 8600/24000 | SmoothLoss: 2.8833 | EstNLL: 1.9712 (PPL 7.18) | LR: 2.34e-04 | Grad: 0.31 | 89.1k tok/s | VRAM: 21.3G
87
+ Step 8800/24000 | SmoothLoss: 2.8756 | EstNLL: 1.9626 (PPL 7.12) | LR: 2.30e-04 | Grad: 0.22 | 87.5k tok/s | VRAM: 21.3G
88
+ Step 9000/24000 | SmoothLoss: 2.9027 | EstNLL: 1.9928 (PPL 7.34) | LR: 2.27e-04 | Grad: 0.37 | 89.0k tok/s | VRAM: 21.3G
89
+ Step 9200/24000 | SmoothLoss: 2.9001 | EstNLL: 1.9899 (PPL 7.31) | LR: 2.24e-04 | Grad: 0.24 | 89.9k tok/s | VRAM: 21.3G
90
+ Step 9400/24000 | SmoothLoss: 2.8625 | EstNLL: 1.9480 (PPL 7.01) | LR: 2.20e-04 | Grad: 0.30 | 88.0k tok/s | VRAM: 21.3G
91
+ Step 9600/24000 | SmoothLoss: 2.8750 | EstNLL: 1.9620 (PPL 7.11) | LR: 2.17e-04 | Grad: 0.34 | 88.6k tok/s | VRAM: 21.3G
92
+ Step 9800/24000 | SmoothLoss: 2.8761 | EstNLL: 1.9632 (PPL 7.12) | LR: 2.14e-04 | Grad: 0.25 | 88.7k tok/s | VRAM: 21.3G
93
+ Step 10000/24000 | SmoothLoss: 2.8638 | EstNLL: 1.9496 (PPL 7.03) | LR: 2.10e-04 | Grad: 0.28 | 88.9k tok/s | VRAM: 21.3G
94
+
95
+ -------------------------------------------------------------------------------------
96
+ 🔎 210M-PRO EVAL @ Step 10,000 | Val NLL: 1.8687 | Val PPL: 6.48
97
+ [En->My Formal]: ဗဟိုဘဏ်က ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးတည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
98
+ [En->My Spoken]: အမှန်ဆို ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ထားဘဲ စိတ်မပူပါနဲ့လို့ အတူတူတွေးမိပါတယ်။
99
+ [My->En Formal]: The Central Bank has raised interest rates to control inflation and stabilize the economy.
100
+ [My->En Spoken]: My love. I knew that you and I really loved each other, Buddha also knows, Sanka also knows, Nat also knows, those who are jealous believe that they are being tricked, so you have gone mad.
101
+ 🌟 New Best 210M-Pro Checkpoint Saved! (Val NLL: 1.8687)
102
+ -------------------------------------------------------------------------------------
103
+
104
+ Step 10200/24000 | SmoothLoss: 2.8782 | EstNLL: 1.9656 (PPL 7.14) | LR: 2.07e-04 | Grad: 0.31 | 89.1k tok/s | VRAM: 21.3G
105
+ Step 10400/24000 | SmoothLoss: 2.8700 | EstNLL: 1.9564 (PPL 7.07) | LR: 2.03e-04 | Grad: 0.32 | 88.7k tok/s | VRAM: 21.3G
106
+ Step 10600/24000 | SmoothLoss: 2.8692 | EstNLL: 1.9556 (PPL 7.07) | LR: 2.00e-04 | Grad: 0.29 | 87.7k tok/s | VRAM: 21.3G
107
+ Step 10800/24000 | SmoothLoss: 2.8580 | EstNLL: 1.9431 (PPL 6.98) | LR: 1.96e-04 | Grad: 0.27 | 88.5k tok/s | VRAM: 21.3G
108
+ Step 11000/24000 | SmoothLoss: 2.8549 | EstNLL: 1.9396 (PPL 6.96) | LR: 1.92e-04 | Grad: 0.34 | 88.8k tok/s | VRAM: 21.3G
109
+ Step 11200/24000 | SmoothLoss: 2.8666 | EstNLL: 1.9526 (PPL 7.05) | LR: 1.89e-04 | Grad: 0.44 | 88.7k tok/s | VRAM: 21.3G
110
+ Step 11400/24000 | SmoothLoss: 2.8623 | EstNLL: 1.9478 (PPL 7.01) | LR: 1.85e-04 | Grad: 0.45 | 89.1k tok/s | VRAM: 21.3G
111
+ Step 11600/24000 | SmoothLoss: 2.8479 | EstNLL: 1.9318 (PPL 6.90) | LR: 1.82e-04 | Grad: 0.38 | 88.6k tok/s | VRAM: 21.3G
112
+ Step 11800/24000 | SmoothLoss: 2.8277 | EstNLL: 1.9094 (PPL 6.75) | LR: 1.78e-04 | Grad: 0.35 | 88.3k tok/s | VRAM: 21.3G
113
+ Step 12000/24000 | SmoothLoss: 2.8467 | EstNLL: 1.9305 (PPL 6.89) | LR: 1.74e-04 | Grad: 0.31 | 88.8k tok/s | VRAM: 21.3G
114
+
115
+ -------------------------------------------------------------------------------------
116
+ 🔎 210M-PRO EVAL @ Step 12,000 | Val NLL: 1.8416 | Val PPL: 6.31
117
+ [En->My Formal]: ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးကို တည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
118
+ [En->My Spoken]: အမှန်အတိုင်းပြောရရင် ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ထားဘဲ စိတ်မပူပါနဲ့လို့ ကျွန်တော်တို့ အတူတူတွေးကြည့်ပါတယ်။
119
+ [My->En Formal]: The central bank has raised interest rates to control inflation and stabilize the economy.
120
+ [My->En Spoken]: Love, I and you deeply loved each other; Buddha knows, Sakka knows, Nat knows. Those who envy me are making jokes, so I'm going crazy.
121
+ 🌟 New Best 210M-Pro Checkpoint Saved! (Val NLL: 1.8416)
122
+ -------------------------------------------------------------------------------------
123
+
124
+ Step 12200/24000 | SmoothLoss: 2.8468 | EstNLL: 1.9306 (PPL 6.89) | LR: 1.71e-04 | Grad: 0.25 | 89.1k tok/s | VRAM: 21.3G
125
+ Step 12400/24000 | SmoothLoss: 2.8404 | EstNLL: 1.9236 (PPL 6.85) | LR: 1.67e-04 | Grad: 0.38 | 88.3k tok/s | VRAM: 21.3G
126
+ Step 12600/24000 | SmoothLoss: 2.8348 | EstNLL: 1.9173 (PPL 6.80) | LR: 1.63e-04 | Grad: 0.33 | 87.5k tok/s | VRAM: 21.3G
127
+ Step 12800/24000 | SmoothLoss: 2.8338 | EstNLL: 1.9162 (PPL 6.80) | LR: 1.59e-04 | Grad: 0.23 | 88.3k tok/s | VRAM: 21.3G
128
+ Step 13000/24000 | SmoothLoss: 2.8257 | EstNLL: 1.9072 (PPL 6.73) | LR: 1.56e-04 | Grad: 0.41 | 88.8k tok/s | VRAM: 21.3G
129
+ Step 13200/24000 | SmoothLoss: 2.8447 | EstNLL: 1.9283 (PPL 6.88) | LR: 1.52e-04 | Grad: 0.40 | 89.0k tok/s | VRAM: 21.3G
130
+ Step 13400/24000 | SmoothLoss: 2.8288 | EstNLL: 1.9106 (PPL 6.76) | LR: 1.48e-04 | Grad: 0.25 | 88.0k tok/s | VRAM: 21.3G
131
+ Step 13600/24000 | SmoothLoss: 2.8357 | EstNLL: 1.9183 (PPL 6.81) | LR: 1.45e-04 | Grad: 0.27 | 89.3k tok/s | VRAM: 21.3G
132
+ Step 13800/24000 | SmoothLoss: 2.8297 | EstNLL: 1.9116 (PPL 6.76) | LR: 1.41e-04 | Grad: 0.28 | 88.7k tok/s | VRAM: 21.3G
133
+ Step 14000/24000 | SmoothLoss: 2.8323 | EstNLL: 1.9145 (PPL 6.78) | LR: 1.38e-04 | Grad: 0.26 | 89.7k tok/s | VRAM: 21.3G
134
+
135
+ -------------------------------------------------------------------------------------
136
+ 🔎 210M-PRO EVAL @ Step 14,000 | Val NLL: 1.8221 | Val PPL: 6.18
137
+ [En->My Formal]: ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးကို တည်ငြိမ်စေရန် ဗဟိုဘဏ်၏ အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
138
+ [En->My Spoken]: တကယ်ကတော့ ဒီလိုဖြစ်လာမယ်လို့ မထင်ထားပဲ မစိုးရိမ်ပါနဲ့ အတူတူ အဖြေရှာကြမယ်
139
+ [My->En Formal]: The central bank has raised interest rates to control inflation and stabilize the economy.
140
+ [My->En Spoken]: My dear, I knew that you and I loved each other very much—I knew God too, I knew God too, I knew God too, because those who envy me are making mistakes, so you've gone mad.
141
+ 🌟 New Best 210M-Pro Checkpoint Saved! (Val NLL: 1.8221)
142
+ -------------------------------------------------------------------------------------
143
+
144
+ Step 14200/24000 | SmoothLoss: 2.8288 | EstNLL: 1.9107 (PPL 6.76) | LR: 1.34e-04 | Grad: 0.31 | 88.4k tok/s | VRAM: 21.3G
145
+ Step 14400/24000 | SmoothLoss: 2.8297 | EstNLL: 1.9116 (PPL 6.76) | LR: 1.30e-04 | Grad: 0.27 | 88.6k tok/s | VRAM: 21.3G
146
+ Step 14600/24000 | SmoothLoss: 2.8253 | EstNLL: 1.9067 (PPL 6.73) | LR: 1.27e-04 | Grad: 0.25 | 88.6k tok/s | VRAM: 21.3G
147
+ Step 14800/24000 | SmoothLoss: 2.8133 | EstNLL: 1.8934 (PPL 6.64) | LR: 1.23e-04 | Grad: 0.40 | 88.4k tok/s | VRAM: 21.3G
148
+ Step 15000/24000 | SmoothLoss: 2.8132 | EstNLL: 1.8933 (PPL 6.64) | LR: 1.20e-04 | Grad: 0.25 | 87.9k tok/s | VRAM: 21.3G
149
+ Step 15200/24000 | SmoothLoss: 2.8218 | EstNLL: 1.9028 (PPL 6.70) | LR: 1.16e-04 | Grad: 0.46 | 88.6k tok/s | VRAM: 21.3G
150
+ Step 15400/24000 | SmoothLoss: 2.8295 | EstNLL: 1.9114 (PPL 6.76) | LR: 1.13e-04 | Grad: 0.32 | 89.4k tok/s | VRAM: 21.3G
151
+ Step 15600/24000 | SmoothLoss: 2.8077 | EstNLL: 1.8872 (PPL 6.60) | LR: 1.10e-04 | Grad: 0.30 | 88.1k tok/s | VRAM: 21.3G
152
+ Step 15800/24000 | SmoothLoss: 2.8321 | EstNLL: 1.9143 (PPL 6.78) | LR: 1.06e-04 | Grad: 0.31 | 89.3k tok/s | VRAM: 21.3G
153
+ Step 16000/24000 | SmoothLoss: 2.8068 | EstNLL: 1.8862 (PPL 6.59) | LR: 1.03e-04 | Grad: 0.29 | 87.4k tok/s | VRAM: 21.3G
154
+
155
+ -------------------------------------------------------------------------------------
156
+ 🔎 210M-PRO EVAL @ Step 16,000 | Val NLL: 1.7981 | Val PPL: 6.04
157
+ [En->My Formal]: ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးကို တည်ငြိမ်စေရန် ဗဟိုဘဏ်သည် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
158
+ [En->My Spoken]: အမှန်ကတော့ ဒီလိုဖြစ်လာလိမ့်မယ်လို့ မမျှော်လင့်ထားဘဲ မပူနဲ့ အတူတူတွေးကြည့်ကြမှာပဲ
159
+ [My->En Formal]: The Central Bank has raised interest rates to control inflation and stabilize the economy.
160
+ [My->En Spoken]: My love, I knew that you and I loved each other very much—buddha knows this too, sugar knows this too, celestial knows this too. Those who envy me are being deceived; because of this, you have gone mad.
161
+ 🌟 New Best 210M-Pro Checkpoint Saved! (Val NLL: 1.7981)
162
+ -------------------------------------------------------------------------------------
163
+
164
+ Step 16200/24000 | SmoothLoss: 2.8005 | EstNLL: 1.8792 (PPL 6.55) | LR: 9.97e-05 | Grad: 0.30 | 88.9k tok/s | VRAM: 21.3G
165
+ Step 16400/24000 | SmoothLoss: 2.8184 | EstNLL: 1.8991 (PPL 6.68) | LR: 9.65e-05 | Grad: 0.53 | 87.6k tok/s | VRAM: 21.3G
166
+ Step 16600/24000 | SmoothLoss: 2.7911 | EstNLL: 1.8687 (PPL 6.48) | LR: 9.33e-05 | Grad: 0.49 | 88.9k tok/s | VRAM: 21.3G
167
+ Step 16800/24000 | SmoothLoss: 2.8040 | EstNLL: 1.8830 (PPL 6.57) | LR: 9.02e-05 | Grad: 0.30 | 89.0k tok/s | VRAM: 21.3G
168
+ Step 17000/24000 | SmoothLoss: 2.8037 | EstNLL: 1.8827 (PPL 6.57) | LR: 8.72e-05 | Grad: 0.38 | 89.3k tok/s | VRAM: 21.3G
169
+ Step 17200/24000 | SmoothLoss: 2.8083 | EstNLL: 1.8878 (PPL 6.60) | LR: 8.42e-05 | Grad: 0.37 | 88.3k tok/s | VRAM: 21.3G
170
+ Step 17400/24000 | SmoothLoss: 2.7887 | EstNLL: 1.8660 (PPL 6.46) | LR: 8.13e-05 | Grad: 0.33 | 88.6k tok/s | VRAM: 21.3G
171
+ Step 17600/24000 | SmoothLoss: 2.8052 | EstNLL: 1.8844 (PPL 6.58) | LR: 7.84e-05 | Grad: 0.31 | 89.5k tok/s | VRAM: 21.3G
172
+ Step 17800/24000 | SmoothLoss: 2.7896 | EstNLL: 1.8671 (PPL 6.47) | LR: 7.56e-05 | Grad: 0.30 | 89.0k tok/s | VRAM: 21.3G
173
+ Step 18000/24000 | SmoothLoss: 2.7952 | EstNLL: 1.8733 (PPL 6.51) | LR: 7.29e-05 | Grad: 0.28 | 88.8k tok/s | VRAM: 21.3G
174
+
175
+ -------------------------------------------------------------------------------------
176
+ 🔎 210M-PRO EVAL @ Step 18,000 | Val NLL: 1.7784 | Val PPL: 5.92
177
+ [En->My Formal]: ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးတည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
178
+ [En->My Spoken]: အမှန်ဆို ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ခဲ့ပေမယ့် စိတ်မပူပါနဲ့ အတူတူတွေးကြည့်မယ်။
179
+ [My->En Formal]: The central bank has raised interest rates to control inflation and stabilize the economy.
180
+ [My->En Spoken]: Love, I loved you and me so much that Buddha knows, Saka knows, Nats know. Those who envy are being seduced; because of this, you have gone mad.
181
+ 🌟 New Best 210M-Pro Checkpoint Saved! (Val NLL: 1.7784)
182
+ -------------------------------------------------------------------------------------
183
+
184
+ Step 18200/24000 | SmoothLoss: 2.7973 | EstNLL: 1.8756 (PPL 6.52) | LR: 7.02e-05 | Grad: 0.26 | 89.1k tok/s | VRAM: 21.3G
185
+ Step 18400/24000 | SmoothLoss: 2.7875 | EstNLL: 1.8648 (PPL 6.45) | LR: 6.76e-05 | Grad: 0.27 | 88.6k tok/s | VRAM: 21.3G
186
+ Step 18600/24000 | SmoothLoss: 2.7865 | EstNLL: 1.8637 (PPL 6.45) | LR: 6.51e-05 | Grad: 0.29 | 87.8k tok/s | VRAM: 21.3G
187
+ Step 18800/24000 | SmoothLoss: 2.7915 | EstNLL: 1.8692 (PPL 6.48) | LR: 6.27e-05 | Grad: 0.39 | 89.1k tok/s | VRAM: 21.3G
188
+ Step 19000/24000 | SmoothLoss: 2.7777 | EstNLL: 1.8538 (PPL 6.38) | LR: 6.03e-05 | Grad: 0.62 | 88.1k tok/s | VRAM: 21.3G
189
+ Step 19200/24000 | SmoothLoss: 2.7879 | EstNLL: 1.8652 (PPL 6.46) | LR: 5.80e-05 | Grad: 0.33 | 89.1k tok/s | VRAM: 21.3G
190
+ Step 19400/24000 | SmoothLoss: 2.7872 | EstNLL: 1.8644 (PPL 6.45) | LR: 5.58e-05 | Grad: 0.35 | 88.3k tok/s | VRAM: 21.3G
191
+ Step 19600/24000 | SmoothLoss: 2.7800 | EstNLL: 1.8564 (PPL 6.40) | LR: 5.37e-05 | Grad: 0.34 | 88.5k tok/s | VRAM: 21.3G
192
+ Step 19800/24000 | SmoothLoss: 2.7748 | EstNLL: 1.8506 (PPL 6.36) | LR: 5.16e-05 | Grad: 0.23 | 88.8k tok/s | VRAM: 21.3G
193
+ Step 20000/24000 | SmoothLoss: 2.7757 | EstNLL: 1.8516 (PPL 6.37) | LR: 4.97e-05 | Grad: 0.22 | 88.2k tok/s | VRAM: 21.3G
194
+
195
+ -------------------------------------------------------------------------------------
196
+ 🔎 210M-PRO EVAL @ Step 20,000 | Val NLL: 1.7648 | Val PPL: 5.84
197
+ [En->My Formal]: ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးကို တည်ငြိမ်စေရန် ဗဟိုဘဏ်သည် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
198
+ [En->My Spoken]: တကယ်တန်းတော့ ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ထားဘဲ စိတ်မပူပါနဲ့ အတူတူတွေးကြည့်ကြမယ်နော်။
199
+ [My->En Formal]: The central bank has raised interest rates to control inflation and stabilize the economy.
200
+ [My->En Spoken]: Love, I knew that you and I loved each other very much, Buddha knows too, sir knows too, those who envy me are being jealous. Because of this, you have gone mad.
201
+ 🌟 New Best 210M-Pro Checkpoint Saved! (Val NLL: 1.7648)
202
+ -------------------------------------------------------------------------------------
203
+
204
+ Step 20200/24000 | SmoothLoss: 2.7861 | EstNLL: 1.8632 (PPL 6.44) | LR: 4.78e-05 | Grad: 0.25 | 89.6k tok/s | VRAM: 21.3G
205
+ Step 20400/24000 | SmoothLoss: 2.7663 | EstNLL: 1.8412 (PPL 6.30) | LR: 4.60e-05 | Grad: 0.27 | 88.4k tok/s | VRAM: 21.3G
206
+ Step 20600/24000 | SmoothLoss: 2.7747 | EstNLL: 1.8505 (PPL 6.36) | LR: 4.43e-05 | Grad: 0.29 | 89.0k tok/s | VRAM: 21.3G
207
+ Step 20800/24000 | SmoothLoss: 2.7740 | EstNLL: 1.8497 (PPL 6.36) | LR: 4.27e-05 | Grad: 0.28 | 88.5k tok/s | VRAM: 21.3G
208
+ Step 21000/24000 | SmoothLoss: 2.7830 | EstNLL: 1.8597 (PPL 6.42) | LR: 4.12e-05 | Grad: 0.37 | 88.2k tok/s | VRAM: 21.3G
209
+ Step 21200/24000 | SmoothLoss: 2.7732 | EstNLL: 1.8489 (PPL 6.35) | LR: 3.98e-05 | Grad: 0.39 | 89.3k tok/s | VRAM: 21.3G
210
+ Step 21400/24000 | SmoothLoss: 2.7722 | EstNLL: 1.8477 (PPL 6.35) | LR: 3.84e-05 | Grad: 0.36 | 89.0k tok/s | VRAM: 21.3G
211
+ Step 21600/24000 | SmoothLoss: 2.7641 | EstNLL: 1.8388 (PPL 6.29) | LR: 3.72e-05 | Grad: 0.40 | 88.8k tok/s | VRAM: 21.3G
212
+ Step 21800/24000 | SmoothLoss: 2.7755 | EstNLL: 1.8514 (PPL 6.37) | LR: 3.61e-05 | Grad: 0.50 | 88.0k tok/s | VRAM: 21.3G
213
+ Step 22000/24000 | SmoothLoss: 2.7742 | EstNLL: 1.8499 (PPL 6.36) | LR: 3.50e-05 | Grad: 0.43 | 88.7k tok/s | VRAM: 21.3G
214
+
215
+ -------------------------------------------------------------------------------------
216
+ 🔎 210M-PRO EVAL @ Step 22,000 | Val NLL: 1.7561 | Val PPL: 5.79
217
+ [En->My Formal]: ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးကို တည်ငြိမ်စေရန် ဗဟိုဘဏ်သည် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
218
+ [En->My Spoken]: အမှန်ဆို ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ခဲ့ပေမယ့် စိတ်မပူပါနဲ့ အတူတူတွေးကြည့်ကြမယ်
219
+ [My->En Formal]: The Central Bank has raised interest rates to control inflation and stabilize the economy.
220
+ [My->En Spoken]: Love, I knew that you and I loved each other very much, Buddha knows too, Sankha knows too, Nat knows too, those who envy are being jealous. Because of this, you have gone crazy.
221
+ 🌟 New Best 210M-Pro Checkpoint Saved! (Val NLL: 1.7561)
222
+ -------------------------------------------------------------------------------------
223
+
224
+ Step 22200/24000 | SmoothLoss: 2.7567 | EstNLL: 1.8306 (PPL 6.24) | LR: 3.41e-05 | Grad: 0.43 | 88.8k tok/s | VRAM: 21.3G
225
+ Step 22400/24000 | SmoothLoss: 2.7752 | EstNLL: 1.8510 (PPL 6.37) | LR: 3.32e-05 | Grad: 0.32 | 89.3k tok/s | VRAM: 21.3G
226
+ Step 22600/24000 | SmoothLoss: 2.7837 | EstNLL: 1.8606 (PPL 6.43) | LR: 3.25e-05 | Grad: 0.27 | 88.8k tok/s | VRAM: 21.3G
227
+ Step 22800/24000 | SmoothLoss: 2.7672 | EstNLL: 1.8422 (PPL 6.31) | LR: 3.18e-05 | Grad: 0.24 | 88.3k tok/s | VRAM: 21.3G
228
+ Step 23000/24000 | SmoothLoss: 2.7640 | EstNLL: 1.8386 (PPL 6.29) | LR: 3.13e-05 | Grad: 0.29 | 88.9k tok/s | VRAM: 21.3G
229
+ Step 23200/24000 | SmoothLoss: 2.7571 | EstNLL: 1.8309 (PPL 6.24) | LR: 3.08e-05 | Grad: 0.33 | 88.8k tok/s | VRAM: 21.3G
230
+ Step 23400/24000 | SmoothLoss: 2.7612 | EstNLL: 1.8356 (PPL 6.27) | LR: 3.05e-05 | Grad: 0.37 | 88.7k tok/s | VRAM: 21.3G
231
+ Step 23600/24000 | SmoothLoss: 2.7610 | EstNLL: 1.8352 (PPL 6.27) | LR: 3.02e-05 | Grad: 0.44 | 89.2k tok/s | VRAM: 21.3G
232
+ Step 23800/24000 | SmoothLoss: 2.7653 | EstNLL: 1.8401 (PPL 6.30) | LR: 3.01e-05 | Grad: 0.43 | 88.9k tok/s | VRAM: 21.3G
233
+ Step 24000/24000 | SmoothLoss: 2.7609 | EstNLL: 1.8352 (PPL 6.27) | LR: 3.00e-05 | Grad: 0.39 | 88.7k tok/s | VRAM: 21.3G
234
+
235
+ -------------------------------------------------------------------------------------
236
+ 🔎 210M-PRO EVAL @ Step 24,000 | Val NLL: 1.7492 | Val PPL: 5.75
237
+ [En->My Formal]: ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးကို တည်ငြိမ်စေရန် ဗဟိုဘဏ်သည် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
238
+ [En->My Spoken]: အမှန်ဆို ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ထားဘဲ စိတ်မပူပါနဲ့ အတူတူတွေးကြည့်ကြမယ်
239
+ [My->En Formal]: The Central Bank has raised interest rates to control inflation and stabilize the economy.
240
+ [My->En Spoken]: Love, I knew that you and I loved each other deeply, Buddha knows too, Sankha knows too, Nat knows too, those who envy me are being harmed. Because of this, you have gone crazy.
241
+ 🌟 New Best 210M-Pro Checkpoint Saved! (Val NLL: 1.7492)
242
+ -------------------------------------------------------------------------------------
243
+
244
+
245
+ >>> Starting 210M-Pro Stage 2 Cooldown (1,000 steps | label_smoothing=0.0 | LR 3e-5 -> 4e-6)...
246
+ 🔎 210M-Pro Stage 2 Step 250/1000 | Val NLL: 1.6798 (PPL 5.36)
247
+ 🔎 210M-Pro Stage 2 Step 500/1000 | Val NLL: 1.6765 (PPL 5.35)
248
+ 🔎 210M-Pro Stage 2 Step 750/1000 | Val NLL: 1.6740 (PPL 5.33)
249
+ 🔎 210M-Pro Stage 2 Step 1000/1000 | Val NLL: 1.6724 (PPL 5.33)
250
+
251
+ ==========================================================================================
252
+ 🏆 MYANMARTRANSLATOR-210M-PRO FINAL BENCHMARK | Val NLL: 1.6724 (PPL 5.33)
253
+ ==========================================================================================
254
+ [En->My Formal (Beam=4)]: ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးတည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
255
+ [En->My Spoken (Beam=4)]: ရိုးရိုးသားသားပြောရရင် ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ထားခဲ့ပေမယ့် စိတ်မပူပါနဲ့၊ ကျွန်တော်တို့ အတူတူ အဖြေရှာကြတာပေါ့။
256
+ [My->En Formal (Beam=4)]: The central bank raised interest rates to control inflation and stabilize the economy.
257
+ [My->En Spoken (Beam=4)]: My love, heaven and earth know how deeply you and I loved each other, yet because you believed the lies fed by jealous people, you've lost your mind.
258
+ [En->My News (Beam=4)]: မိုးသည်းထန်စွာ ရွာသွန်းမှုကြောင့် မြို့နယ်အချို့တွင် ရေကြီးရေလျှံမှုများ ဆိုးရွားစွာ ဖြစ်ပေါ်ခဲ့ပြီး ထောင်နှင့်ချီသော ဒေသခံများသည် ယာယီခိုလှုံရာ စခန်းများသို့ ရွှေ့ပြောင်းခဲ့ရသည်။
259
+ [En->My Health (Beam=4)]: ရာသီတုပ်ကွေးရောဂါကနေ မြန်မြန်နာလန်ထဖို့ သန့်ရှင်းတဲ့ရေများများသောက်ဖို့နဲ့ လုံလောက်တဲ့ အနားယူချိန်ရဖို့ ဆရာဝန်တွေက အကြံပြုထားပါတယ်။
260
+ [En->My Spoken (Beam=4)]: ဒီမနက် တစ်နာရီကျော်လောက် ကားပိတ်မိနေလို့ ကျွန်တော်မပါဘဲ အစည်းအဝေးကို စထားလိုက်ပါနော်။
261
+ [My->En News (Beam=4)]: In order to promote international trade, inspections of goods are being carried out more efficiently at border trade camps.
262
+ [My->En Tech (Beam=4)]: Artificial intelligence technology can help make processes more effective in education, healthcare, and economic sectors.
263
+ [My->En Spoken (Beam=4)]: I have to go tomorrow morning, so tonight I will sleep early. I'll call you in the morning.
264
+ ==========================================================================================
config.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "MyanmarTranslatorForConditionalGeneration"
4
+ ],
5
+ "model_type": "myanmartranslator",
6
+ "auto_map": {
7
+ "AutoConfig": "configuration_myanmartranslator.MyanmarTranslatorConfig",
8
+ "AutoModel": "modeling_myanmartranslator.MyanmarTranslatorForConditionalGeneration",
9
+ "AutoModelForSeq2SeqLM": "modeling_myanmartranslator.MyanmarTranslatorForConditionalGeneration"
10
+ },
11
+ "vocab_size": 65664,
12
+ "d_model": 768,
13
+ "n_heads": 12,
14
+ "enc_layers": 8,
15
+ "dec_layers": 8,
16
+ "ffn_dim": 2816,
17
+ "max_seq_len": 1024,
18
+ "is_encoder_decoder": true,
19
+ "tie_word_embeddings": true,
20
+ "bos_token_id": 1,
21
+ "eos_token_id": 2,
22
+ "pad_token_id": 3,
23
+ "unk_token_id": 0,
24
+ "2my_token_id": 65536,
25
+ "2en_token_id": 65537,
26
+ "denoise_token_id": 65538,
27
+ "mask_token_id": 65539,
28
+ "torch_dtype": "bfloat16",
29
+ "val_nll": 1.6724,
30
+ "val_ppl": 5.33
31
+ }
configuration_myanmartranslator.py ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers import PretrainedConfig
2
+
3
+ class MyanmarTranslatorConfig(PretrainedConfig):
4
+ model_type = "myanmartranslator"
5
+ def __init__(self, vocab_size=65664, d_model=512, n_heads=8, enc_layers=6, dec_layers=6, ffn_dim=1728, max_seq_len=1024, **kwargs):
6
+ super().__init__(bos_token_id=1, eos_token_id=2, pad_token_id=3, unk_token_id=0, is_encoder_decoder=True, tie_word_embeddings=True, **kwargs)
7
+ self.vocab_size = vocab_size
8
+ self.d_model = d_model
9
+ self.n_heads = n_heads
10
+ self.enc_layers = enc_layers
11
+ self.dec_layers = dec_layers
12
+ self.ffn_dim = ffn_dim
13
+ self.max_seq_len = max_seq_len
generation_config.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 1,
3
+ "eos_token_id": 2,
4
+ "pad_token_id": 3,
5
+ "decoder_start_token_id": 1,
6
+ "max_new_tokens": 192,
7
+ "num_beams": 4,
8
+ "length_penalty": 0.85,
9
+ "repetition_penalty": 1.12
10
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b5792cccff854efe02b92ee77817de84786ddcadb022a26bdfe9ee6c594a2349
3
+ size 421812880
modeling_myanmartranslator.py ADDED
@@ -0,0 +1,205 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os, math
2
+ import torch
3
+ import torch.nn as nn
4
+ import torch.nn.functional as F
5
+ import sentencepiece as spm
6
+ from transformers import PreTrainedModel
7
+ from .configuration_myanmartranslator import MyanmarTranslatorConfig
8
+
9
+ VOCAB_SIZE, UNK_ID, BOS_ID, EOS_ID, PAD_ID = 65664, 0, 1, 2, 3
10
+ TOK_2MY, TOK_2EN, TOK_DENOISE, TOK_MASK = 65536, 65537, 65538, 65539
11
+
12
+ class RMSNorm(nn.Module):
13
+ def __init__(self, dim: int, eps: float = 1e-6):
14
+ super().__init__()
15
+ self.dim = (dim,)
16
+ self.eps = eps
17
+ self.weight = nn.Parameter(torch.ones(dim))
18
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
19
+ return F.rms_norm(x, self.dim, self.weight, self.eps)
20
+
21
+ class RotaryEmbedding(nn.Module):
22
+ def __init__(self, dim: int, max_seq_len: int = 1024, theta: float = 10000.0):
23
+ super().__init__()
24
+ inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
25
+ t = torch.arange(max_seq_len, dtype=torch.float32)
26
+ freqs = torch.outer(t, inv_freq)
27
+ self.register_buffer("cos_cached", freqs.cos()[None, None, :, :], persistent=False)
28
+ self.register_buffer("sin_cached", freqs.sin()[None, None, :, :], persistent=False)
29
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
30
+ L = x.size(2)
31
+ cos = self.cos_cached[:, :, :L, :].to(dtype=x.dtype)
32
+ sin = self.sin_cached[:, :, :L, :].to(dtype=x.dtype)
33
+ x1, x2 = x[..., ::2], x[..., 1::2]
34
+ return torch.cat([x1 * cos - x2 * sin, x1 * sin + x2 * cos], dim=-1)
35
+
36
+ class SwiGLU(nn.Module):
37
+ def __init__(self, d_model: int, ffn_dim: int):
38
+ super().__init__()
39
+ self.gate_up = nn.Linear(d_model, 2 * ffn_dim, bias=False)
40
+ self.down = nn.Linear(ffn_dim, d_model, bias=False)
41
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
42
+ gate, up = self.gate_up(x).chunk(2, dim=-1)
43
+ return self.down(F.silu(gate) * up)
44
+
45
+ class SelfAttention(nn.Module):
46
+ def __init__(self, d_model: int, n_heads: int, is_causal: bool = False):
47
+ super().__init__()
48
+ self.n_heads = n_heads
49
+ self.head_dim = d_model // n_heads
50
+ self.is_causal = is_causal
51
+ self.qkv = nn.Linear(d_model, 3 * d_model, bias=False)
52
+ self.out = nn.Linear(d_model, d_model, bias=False)
53
+ self.q_norm = RMSNorm(self.head_dim)
54
+ self.k_norm = RMSNorm(self.head_dim)
55
+ def forward(self, x: torch.Tensor, rope: RotaryEmbedding, attn_mask: torch.Tensor = None) -> torch.Tensor:
56
+ B, L, D = x.shape
57
+ q, k, v = self.qkv(x).chunk(3, dim=-1)
58
+ q = self.q_norm(q.view(B, L, self.n_heads, self.head_dim).transpose(1, 2))
59
+ k = self.k_norm(k.view(B, L, self.n_heads, self.head_dim).transpose(1, 2))
60
+ v = v.view(B, L, self.n_heads, self.head_dim).transpose(1, 2)
61
+ q, k = rope(q), rope(k)
62
+ y = F.scaled_dot_product_attention(q, k, v, attn_mask=None if self.is_causal else attn_mask, is_causal=self.is_causal)
63
+ return self.out(y.transpose(1, 2).contiguous().view(B, L, D))
64
+
65
+ class CrossAttention(nn.Module):
66
+ def __init__(self, d_model: int, n_heads: int):
67
+ super().__init__()
68
+ self.n_heads = n_heads
69
+ self.head_dim = d_model // n_heads
70
+ self.q_proj = nn.Linear(d_model, d_model, bias=False)
71
+ self.kv_proj = nn.Linear(d_model, 2 * d_model, bias=False)
72
+ self.out = nn.Linear(d_model, d_model, bias=False)
73
+ self.q_norm = RMSNorm(self.head_dim)
74
+ self.k_norm = RMSNorm(self.head_dim)
75
+ def forward(self, x: torch.Tensor, enc_out: torch.Tensor, enc_mask: torch.Tensor) -> torch.Tensor:
76
+ B, T, D = x.shape
77
+ S = enc_out.size(1)
78
+ q = self.q_norm(self.q_proj(x).view(B, T, self.n_heads, self.head_dim).transpose(1, 2))
79
+ k, v = self.kv_proj(enc_out).chunk(2, dim=-1)
80
+ k = self.k_norm(k.view(B, S, self.n_heads, self.head_dim).transpose(1, 2))
81
+ v = v.view(B, S, self.n_heads, self.head_dim).transpose(1, 2)
82
+ y = F.scaled_dot_product_attention(q, k, v, attn_mask=enc_mask, is_causal=False)
83
+ return self.out(y.transpose(1, 2).contiguous().view(B, T, D))
84
+
85
+ class EncoderLayer(nn.Module):
86
+ def __init__(self, d_model: int, n_heads: int, ffn_dim: int):
87
+ super().__init__()
88
+ self.norm1 = RMSNorm(d_model)
89
+ self.self_attn = SelfAttention(d_model, n_heads, is_causal=False)
90
+ self.norm2 = RMSNorm(d_model)
91
+ self.mlp = SwiGLU(d_model, ffn_dim)
92
+ def forward(self, x: torch.Tensor, rope: RotaryEmbedding, enc_mask: torch.Tensor) -> torch.Tensor:
93
+ x = x + self.self_attn(self.norm1(x), rope, enc_mask)
94
+ x = x + self.mlp(self.norm2(x))
95
+ return x
96
+
97
+ class DecoderLayer(nn.Module):
98
+ def __init__(self, d_model: int, n_heads: int, ffn_dim: int):
99
+ super().__init__()
100
+ self.norm1 = RMSNorm(d_model)
101
+ self.self_attn = SelfAttention(d_model, n_heads, is_causal=True)
102
+ self.norm2 = RMSNorm(d_model)
103
+ self.cross_attn = CrossAttention(d_model, n_heads)
104
+ self.norm3 = RMSNorm(d_model)
105
+ self.mlp = SwiGLU(d_model, ffn_dim)
106
+ def forward(self, x: torch.Tensor, enc_out: torch.Tensor, rope: RotaryEmbedding, enc_mask: torch.Tensor) -> torch.Tensor:
107
+ x = x + self.self_attn(self.norm1(x), rope, None)
108
+ x = x + self.cross_attn(self.norm2(x), enc_out, enc_mask)
109
+ x = x + self.mlp(self.norm3(x))
110
+ return x
111
+
112
+ class MyanmarTranslatorForConditionalGeneration(PreTrainedModel):
113
+ config_class = MyanmarTranslatorConfig
114
+ _tied_weights_keys = ["lm_head.weight"]
115
+
116
+ def __init__(self, config: MyanmarTranslatorConfig):
117
+ super().__init__(config)
118
+ self.d_model = config.d_model
119
+ self.embed_scale = math.sqrt(config.d_model)
120
+ self.embed_tokens = nn.Embedding(config.vocab_size, config.d_model, padding_idx=PAD_ID)
121
+ self.rope = RotaryEmbedding(config.d_model // config.n_heads, max_seq_len=config.max_seq_len)
122
+ self.encoder = nn.ModuleList([EncoderLayer(config.d_model, config.n_heads, config.ffn_dim) for _ in range(config.enc_layers)])
123
+ self.enc_norm = RMSNorm(config.d_model)
124
+ self.decoder = nn.ModuleList([DecoderLayer(config.d_model, config.n_heads, config.ffn_dim) for _ in range(config.dec_layers)])
125
+ self.dec_norm = RMSNorm(config.d_model)
126
+ self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False)
127
+ self.lm_head.weight = self.embed_tokens.weight
128
+ self.sp = None
129
+
130
+ def get_input_embeddings(self):
131
+ return self.embed_tokens
132
+
133
+ def set_input_embeddings(self, value):
134
+ self.embed_tokens = value
135
+ self.lm_head.weight = self.embed_tokens.weight
136
+
137
+ def get_output_embeddings(self):
138
+ return self.lm_head
139
+
140
+ def encode(self, src_ids: torch.Tensor):
141
+ enc_mask = (src_ids != PAD_ID).unsqueeze(1).unsqueeze(2)
142
+ x = self.embed_tokens(src_ids) * self.embed_scale
143
+ for layer in self.encoder:
144
+ x = layer(x, self.rope, enc_mask)
145
+ return self.enc_norm(x), enc_mask
146
+
147
+ def decode(self, tgt_in: torch.Tensor, enc_out: torch.Tensor, enc_mask: torch.Tensor):
148
+ x = self.embed_tokens(tgt_in) * self.embed_scale
149
+ for layer in self.decoder:
150
+ x = layer(x, enc_out, self.rope, enc_mask)
151
+ return self.dec_norm(x)
152
+
153
+ def forward(self, input_ids: torch.Tensor, decoder_input_ids: torch.Tensor, labels: torch.Tensor = None):
154
+ enc_out, enc_mask = self.encode(input_ids)
155
+ dec_out = self.decode(decoder_input_ids, enc_out, enc_mask)
156
+ logits = self.lm_head(dec_out)
157
+ loss = F.cross_entropy(logits.view(-1, logits.size(-1)), labels.view(-1), ignore_index=-100) if labels is not None else None
158
+ return {"loss": loss, "logits": logits}
159
+
160
+ @torch.no_grad()
161
+ def translate(self, text: str, direction: str = "en2my", tokenizer_path: str = None, num_beams: int = 4, max_new_tokens: int = 192, length_penalty: float = 0.85, rep_penalty: float = 1.12) -> str:
162
+ self.eval()
163
+ if self.sp is None:
164
+ if tokenizer_path is None:
165
+ tokenizer_path = os.path.join(getattr(self.config, "_name_or_path", "."), "tokenizer.model")
166
+ if not os.path.exists(tokenizer_path):
167
+ from huggingface_hub import hf_hub_download
168
+ tokenizer_path = hf_hub_download(self.config._name_or_path, "tokenizer.model")
169
+ self.sp = spm.SentencePieceProcessor()
170
+ self.sp.load(tokenizer_path)
171
+
172
+ dev = next(self.parameters()).device
173
+ dir_tok = TOK_2MY if direction == "en2my" else TOK_2EN
174
+ s_ids = [dir_tok] + self.sp.encode_as_ids(text.strip())[:1020] + [EOS_ID]
175
+ src_t = torch.tensor([s_ids], dtype=torch.long, device=dev)
176
+ enc_out, enc_mask = self.encode(src_t)
177
+ beams = [(0.0, [BOS_ID], False)]
178
+ for _ in range(max_new_tokens):
179
+ active = [(sc, seq) for sc, seq, done in beams if not done]
180
+ if not active:
181
+ break
182
+ B_act = len(active)
183
+ tgt_t = torch.tensor([seq for _, seq in active], dtype=torch.long, device=dev)
184
+ dec_out = self.decode(tgt_t, enc_out.expand(B_act, -1, -1), enc_mask.expand(B_act, -1, -1, -1))
185
+ logits = self.lm_head(dec_out[:, -1, :]).float()
186
+ logits[:, [UNK_ID, PAD_ID, BOS_ID]] = -1e9
187
+ logits[:, 65536:] = -1e9
188
+ for b_i, (_, seq) in enumerate(active):
189
+ if rep_penalty != 1.0 and len(seq) > 1:
190
+ seen = torch.tensor(list(set(seq[1:])), dtype=torch.long, device=dev)
191
+ s_l = logits[b_i, seen]
192
+ logits[b_i, seen] = torch.where(s_l > 0, s_l / rep_penalty, s_l * rep_penalty)
193
+ log_probs = F.log_softmax(logits, dim=-1)
194
+ topk_lp, topk_ids = torch.topk(log_probs, max(1, num_beams * 2), dim=-1)
195
+ candidates = [(sc, seq, True) for sc, seq, done in beams if done]
196
+ for b_i, (sc, seq) in enumerate(active):
197
+ for k in range(topk_ids.size(1)):
198
+ tok_id = int(topk_ids[b_i, k].item())
199
+ new_sc = sc + float(topk_lp[b_i, k].item())
200
+ candidates.append((new_sc, seq if tok_id == EOS_ID else seq + [tok_id], tok_id == EOS_ID))
201
+ candidates.sort(key=lambda x: x[0] / (max(1, len(x[1]) - 1) ** length_penalty), reverse=True)
202
+ beams = candidates[:num_beams]
203
+ if all(done for _, _, done in beams):
204
+ break
205
+ return self.sp.decode_ids([x for x in beams[0][1][1:] if x < 65536])
special_tokens_map.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<s>",
3
+ "eos_token": "</s>",
4
+ "unk_token": "<unk>",
5
+ "pad_token": "<pad>",
6
+ "additional_special_tokens": [
7
+ "<2my>",
8
+ "<2en>",
9
+ "<denoise>",
10
+ "<mask>"
11
+ ]
12
+ }
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:abb0486c9edaaf32ee5b10074db0077263407d1fe4006c74ad2c6261285b6403
3
+ size 2210735
tokenizer_config.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<unk>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<s>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "</s>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<pad>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ }
35
+ },
36
+ "backend": "custom",
37
+ "bos_token": "<s>",
38
+ "eos_token": "</s>",
39
+ "model_max_length": 131072,
40
+ "pad_token": "<pad>",
41
+ "tokenizer_class": "MyanmarSentencePieceTokenizer",
42
+ "unk_token": "<unk>"
43
+ }
training_codes.txt ADDED
@@ -0,0 +1,793 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ==============================================================================
2
+ # 🏛️ CELL 11 (100% STANDALONE): Flagship MyanmarTranslator-210M-Pro
3
+ # (8L Enc + 8L Dec, d=768, 210.90M Params) + Net2Wider/Deeper + Auto-Upload
4
+ # ==============================================================================
5
+ import os
6
+ os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
7
+
8
+ import sys, gc, math, time, json, shutil, traceback
9
+ import numpy as np
10
+ import sentencepiece as spm
11
+ import torch
12
+ import torch.nn as nn
13
+ import torch.nn.functional as F
14
+ from torch.utils.checkpoint import checkpoint as grad_ckpt
15
+ from torch.utils.data import Dataset, DataLoader, Sampler
16
+ from safetensors.torch import save_file
17
+ from huggingface_hub import HfApi, create_repo
18
+
19
+ torch.backends.cuda.matmul.allow_tf32 = True
20
+ torch.backends.cudnn.allow_tf32 = True
21
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
22
+
23
+ # ── 0. Capture Exact Cell Code & Setup Incremental Logger ──
24
+ try:
25
+ CELL_SOURCE_CODE = get_ipython().user_ns["In"][-1]
26
+ except Exception:
27
+ CELL_SOURCE_CODE = "# MyanmarTranslator-210M-Pro Standalone Training Script"
28
+
29
+ PRO_REPO_ID = "freococo/MyanmarTranslator-210M-Pro"
30
+ WORK_DIR = "/workspace"
31
+ DATA_DIR = "/workspace/nmt_dataset_memmap"
32
+ TOK_DIR = "/workspace/nmt_tokenizer"
33
+ TOK_FILE = os.path.join(TOK_DIR, "tokenizer.model")
34
+ TEACHER_84M = "/workspace/nmt_84m_polished/best_model.pt"
35
+ PRO_CKPT = "/workspace/pro_210m_checkpoints"
36
+ PRO_EXPORT = "/workspace/MyanmarTranslator-210M-Pro-export"
37
+ os.makedirs(PRO_CKPT, exist_ok=True)
38
+
39
+ if not os.path.exists(os.path.join(PRO_CKPT, "latest_pro.pt")):
40
+ with open(os.path.join(PRO_CKPT, "trainingcelloutputwitheval.txt"), "w", encoding="utf-8") as lf:
41
+ lf.write("")
42
+
43
+ with open(os.path.join(PRO_CKPT, "training_codes.txt"), "w", encoding="utf-8") as f:
44
+ f.write(CELL_SOURCE_CODE)
45
+
46
+ LOG_LINES = []
47
+ def log(msg: str = ""):
48
+ print(msg, flush=True)
49
+ LOG_LINES.append(str(msg))
50
+ with open(os.path.join(PRO_CKPT, "trainingcelloutputwitheval.txt"), "a", encoding="utf-8") as lf:
51
+ lf.write(str(msg) + "\n")
52
+
53
+ gc.collect()
54
+ if torch.cuda.is_available():
55
+ torch.cuda.empty_cache()
56
+ free_vram, tot_vram = torch.cuda.mem_get_info()
57
+ _, _, free_disk = shutil.disk_usage(WORK_DIR)
58
+ log(f"🟢 Standalone Kernel Ready for 210M-Pro | GPU Free: {free_vram/1e9:.2f}/{tot_vram/1e9:.2f} GB | Disk Free: {free_disk/1e9:.1f} GB")
59
+
60
+ sp = spm.SentencePieceProcessor()
61
+ sp.load(TOK_FILE)
62
+
63
+ VOCAB_SIZE, UNK_ID, BOS_ID, EOS_ID, PAD_ID = 65664, 0, 1, 2, 3
64
+ TOK_2MY, TOK_2EN, TOK_DENOISE, TOK_MASK = 65536, 65537, 65538, 65539
65
+
66
+ # ── 1. Standalone Architecture Definitions ──
67
+ class RMSNorm(nn.Module):
68
+ def __init__(self, dim: int, eps: float = 1e-6):
69
+ super().__init__()
70
+ self.dim = (dim,)
71
+ self.eps = eps
72
+ self.weight = nn.Parameter(torch.ones(dim))
73
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
74
+ return F.rms_norm(x, self.dim, self.weight, self.eps)
75
+
76
+ class RotaryEmbedding(nn.Module):
77
+ def __init__(self, dim: int, max_seq_len: int = 1024, theta: float = 10000.0):
78
+ super().__init__()
79
+ inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
80
+ t = torch.arange(max_seq_len, dtype=torch.float32)
81
+ freqs = torch.outer(t, inv_freq)
82
+ self.register_buffer("cos_cached", freqs.cos()[None, None, :, :], persistent=False)
83
+ self.register_buffer("sin_cached", freqs.sin()[None, None, :, :], persistent=False)
84
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
85
+ L = x.size(2)
86
+ cos = self.cos_cached[:, :, :L, :].to(dtype=x.dtype)
87
+ sin = self.sin_cached[:, :, :L, :].to(dtype=x.dtype)
88
+ x1, x2 = x[..., ::2], x[..., 1::2]
89
+ return torch.cat([x1 * cos - x2 * sin, x1 * sin + x2 * cos], dim=-1)
90
+
91
+ class SwiGLU(nn.Module):
92
+ def __init__(self, d_model: int, ffn_dim: int):
93
+ super().__init__()
94
+ self.gate_up = nn.Linear(d_model, 2 * ffn_dim, bias=False)
95
+ self.down = nn.Linear(ffn_dim, d_model, bias=False)
96
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
97
+ gate, up = self.gate_up(x).chunk(2, dim=-1)
98
+ return self.down(F.silu(gate) * up)
99
+
100
+ class SelfAttention(nn.Module):
101
+ def __init__(self, d_model: int, n_heads: int, is_causal: bool = False):
102
+ super().__init__()
103
+ self.n_heads = n_heads
104
+ self.head_dim = d_model // n_heads
105
+ self.is_causal = is_causal
106
+ self.qkv = nn.Linear(d_model, 3 * d_model, bias=False)
107
+ self.out = nn.Linear(d_model, d_model, bias=False)
108
+ self.q_norm = RMSNorm(self.head_dim)
109
+ self.k_norm = RMSNorm(self.head_dim)
110
+ def forward(self, x: torch.Tensor, rope: RotaryEmbedding, attn_mask: torch.Tensor = None) -> torch.Tensor:
111
+ B, L, D = x.shape
112
+ q, k, v = self.qkv(x).chunk(3, dim=-1)
113
+ q = self.q_norm(q.view(B, L, self.n_heads, self.head_dim).transpose(1, 2))
114
+ k = self.k_norm(k.view(B, L, self.n_heads, self.head_dim).transpose(1, 2))
115
+ v = v.view(B, L, self.n_heads, self.head_dim).transpose(1, 2)
116
+ q, k = rope(q), rope(k)
117
+ y = F.scaled_dot_product_attention(q, k, v, attn_mask=None if self.is_causal else attn_mask, is_causal=self.is_causal)
118
+ return self.out(y.transpose(1, 2).contiguous().view(B, L, D))
119
+
120
+ class CrossAttention(nn.Module):
121
+ def __init__(self, d_model: int, n_heads: int):
122
+ super().__init__()
123
+ self.n_heads = n_heads
124
+ self.head_dim = d_model // n_heads
125
+ self.q_proj = nn.Linear(d_model, d_model, bias=False)
126
+ self.kv_proj = nn.Linear(d_model, 2 * d_model, bias=False)
127
+ self.out = nn.Linear(d_model, d_model, bias=False)
128
+ self.q_norm = RMSNorm(self.head_dim)
129
+ self.k_norm = RMSNorm(self.head_dim)
130
+ def forward(self, x: torch.Tensor, enc_out: torch.Tensor, enc_mask: torch.Tensor) -> torch.Tensor:
131
+ B, T, D = x.shape
132
+ S = enc_out.size(1)
133
+ q = self.q_norm(self.q_proj(x).view(B, T, self.n_heads, self.head_dim).transpose(1, 2))
134
+ k, v = self.kv_proj(enc_out).chunk(2, dim=-1)
135
+ k = self.k_norm(k.view(B, S, self.n_heads, self.head_dim).transpose(1, 2))
136
+ v = v.view(B, S, self.n_heads, self.head_dim).transpose(1, 2)
137
+ y = F.scaled_dot_product_attention(q, k, v, attn_mask=enc_mask, is_causal=False)
138
+ return self.out(y.transpose(1, 2).contiguous().view(B, T, D))
139
+
140
+ class EncoderLayer(nn.Module):
141
+ def __init__(self, d_model: int, n_heads: int, ffn_dim: int):
142
+ super().__init__()
143
+ self.norm1 = RMSNorm(d_model)
144
+ self.self_attn = SelfAttention(d_model, n_heads, is_causal=False)
145
+ self.norm2 = RMSNorm(d_model)
146
+ self.mlp = SwiGLU(d_model, ffn_dim)
147
+ def forward(self, x: torch.Tensor, rope: RotaryEmbedding, enc_mask: torch.Tensor) -> torch.Tensor:
148
+ x = x + self.self_attn(self.norm1(x), rope, enc_mask)
149
+ x = x + self.mlp(self.norm2(x))
150
+ return x
151
+
152
+ class DecoderLayer(nn.Module):
153
+ def __init__(self, d_model: int, n_heads: int, ffn_dim: int):
154
+ super().__init__()
155
+ self.norm1 = RMSNorm(d_model)
156
+ self.self_attn = SelfAttention(d_model, n_heads, is_causal=True)
157
+ self.norm2 = RMSNorm(d_model)
158
+ self.cross_attn = CrossAttention(d_model, n_heads)
159
+ self.norm3 = RMSNorm(d_model)
160
+ self.mlp = SwiGLU(d_model, ffn_dim)
161
+ def forward(self, x: torch.Tensor, enc_out: torch.Tensor, rope: RotaryEmbedding, enc_mask: torch.Tensor) -> torch.Tensor:
162
+ x = x + self.self_attn(self.norm1(x), rope, None)
163
+ x = x + self.cross_attn(self.norm2(x), enc_out, enc_mask)
164
+ x = x + self.mlp(self.norm3(x))
165
+ return x
166
+
167
+ class MyanmarTranslator84M(nn.Module):
168
+ def __init__(self, vocab_size=65664, d_model=768, n_heads=12, enc_layers=8, dec_layers=8, ffn_dim=2816, max_seq_len=1024):
169
+ super().__init__()
170
+ self.config = dict(
171
+ vocab_size=vocab_size, d_model=d_model, n_heads=n_heads,
172
+ enc_layers=enc_layers, dec_layers=dec_layers, ffn_dim=ffn_dim, max_seq_len=max_seq_len
173
+ )
174
+ self.d_model = d_model
175
+ self.embed_scale = math.sqrt(d_model)
176
+ self.embed_tokens = nn.Embedding(vocab_size, d_model, padding_idx=PAD_ID)
177
+ self.rope = RotaryEmbedding(d_model // n_heads, max_seq_len=max_seq_len)
178
+ self.encoder = nn.ModuleList([EncoderLayer(d_model, n_heads, ffn_dim) for _ in range(enc_layers)])
179
+ self.enc_norm = RMSNorm(d_model)
180
+ self.decoder = nn.ModuleList([DecoderLayer(d_model, n_heads, ffn_dim) for _ in range(dec_layers)])
181
+ self.dec_norm = RMSNorm(d_model)
182
+ self.lm_head = nn.Linear(d_model, vocab_size, bias=False)
183
+ self.lm_head.weight = self.embed_tokens.weight
184
+ self._init_weights(enc_layers + dec_layers)
185
+
186
+ def _init_weights(self, total_layers: int):
187
+ res_std = 0.02 / math.sqrt(2 * total_layers)
188
+ for name, m in self.named_modules():
189
+ if isinstance(m, nn.Linear):
190
+ nn.init.normal_(m.weight, mean=0.0, std=res_std if (name.endswith(".out") or name.endswith(".down")) else 0.02)
191
+ elif isinstance(m, nn.Embedding):
192
+ nn.init.normal_(m.weight, mean=0.0, std=0.02)
193
+ if m.padding_idx is not None:
194
+ m.weight.data[m.padding_idx].zero_()
195
+
196
+ def encode(self, src_ids: torch.Tensor, use_ckpt: bool = False):
197
+ enc_mask = (src_ids != PAD_ID).unsqueeze(1).unsqueeze(2)
198
+ x = self.embed_tokens(src_ids) * self.embed_scale
199
+ for layer in self.encoder:
200
+ x = grad_ckpt(layer, x, self.rope, enc_mask, use_reentrant=False) if (use_ckpt and self.training) else layer(x, self.rope, enc_mask)
201
+ return self.enc_norm(x), enc_mask
202
+
203
+ def decode(self, tgt_in: torch.Tensor, enc_out: torch.Tensor, enc_mask: torch.Tensor, use_ckpt: bool = False):
204
+ x = self.embed_tokens(tgt_in) * self.embed_scale
205
+ for layer in self.decoder:
206
+ x = grad_ckpt(layer, x, enc_out, self.rope, enc_mask, use_reentrant=False) if (use_ckpt and self.training) else layer(x, enc_out, self.rope, enc_mask)
207
+ return self.dec_norm(x)
208
+
209
+ def forward_and_backward(self, src_ids: torch.Tensor, tgt_in: torch.Tensor, labels: torch.Tensor, loss_scale: float = 1.0, label_smoothing: float = 0.1):
210
+ use_ckpt = max(src_ids.size(1), tgt_in.size(1)) > 192
211
+ with torch.amp.autocast("cuda", dtype=torch.bfloat16):
212
+ enc_out, enc_mask = self.encode(src_ids, use_ckpt=use_ckpt)
213
+ dec_out = self.decode(tgt_in, enc_out, enc_mask, use_ckpt=use_ckpt)
214
+ valid_mask = labels.ne(-100)
215
+ valid_h = dec_out[valid_mask]
216
+ valid_y = labels[valid_mask]
217
+ n_valid = valid_y.numel()
218
+
219
+ h_det = valid_h.detach().requires_grad_(True)
220
+ denom = float(n_valid) * float(loss_scale)
221
+ tot_loss = 0.0
222
+ chunk_sz = 2048
223
+
224
+ for i in range(0, n_valid, chunk_sz):
225
+ hc = h_det[i : i + chunk_sz]
226
+ yc = valid_y[i : i + chunk_sz]
227
+ with torch.amp.autocast("cuda", dtype=torch.bfloat16):
228
+ logits_c = self.lm_head(hc)
229
+ lc = F.cross_entropy(logits_c, yc, label_smoothing=label_smoothing, reduction="sum")
230
+ (lc / denom).backward()
231
+ tot_loss += float(lc.detach().item())
232
+ del logits_c, lc
233
+
234
+ valid_h.backward(h_det.grad)
235
+ return tot_loss / max(n_valid, 1), n_valid
236
+
237
+ @torch.no_grad()
238
+ def eval_nll(self, src_ids: torch.Tensor, tgt_in: torch.Tensor, labels: torch.Tensor):
239
+ with torch.amp.autocast("cuda", dtype=torch.bfloat16):
240
+ enc_out, enc_mask = self.encode(src_ids, use_ckpt=False)
241
+ dec_out = self.decode(tgt_in, enc_out, enc_mask, use_ckpt=False)
242
+ valid_mask = labels.ne(-100)
243
+ valid_h = dec_out[valid_mask]
244
+ valid_y = labels[valid_mask]
245
+ n_valid = valid_y.numel()
246
+ tot_nll = 0.0
247
+ for i in range(0, n_valid, 2048):
248
+ logits_c = self.lm_head(valid_h[i : i + 2048])
249
+ tot_nll += float(F.cross_entropy(logits_c, valid_y[i : i + 2048], reduction="sum").item())
250
+ return tot_nll, n_valid
251
+
252
+ @torch.no_grad()
253
+ def translate(self, text: str, direction: str = "en2my", max_new_tokens: int = 192, rep_penalty: float = 1.15) -> str:
254
+ self.eval()
255
+ dir_tok = TOK_2MY if direction == "en2my" else TOK_2EN
256
+ s_ids = [dir_tok] + sp.encode_as_ids(text.strip())[:1020] + [EOS_ID]
257
+ src_t = torch.tensor([s_ids], dtype=torch.long, device=next(self.parameters()).device)
258
+ with torch.amp.autocast("cuda", dtype=torch.bfloat16):
259
+ enc_out, enc_mask = self.encode(src_t, use_ckpt=False)
260
+ ys = [BOS_ID]
261
+ for _ in range(max_new_tokens):
262
+ tgt_t = torch.tensor([ys], dtype=torch.long, device=src_t.device)
263
+ dec_out = self.decode(tgt_t, enc_out, enc_mask, use_ckpt=False)
264
+ logits = self.lm_head(dec_out[:, -1, :]).float()[0]
265
+ logits[[UNK_ID, PAD_ID, BOS_ID]] = -1e9
266
+ logits[65536:] = -1e9
267
+ if rep_penalty != 1.0 and len(ys) > 1:
268
+ seen = torch.tensor(list(set(ys[1:])), dtype=torch.long, device=src_t.device)
269
+ s_logits = logits[seen]
270
+ logits[seen] = torch.where(s_logits > 0, s_logits / rep_penalty, s_logits * rep_penalty)
271
+ nxt = int(logits.argmax().item())
272
+ if nxt == EOS_ID:
273
+ break
274
+ ys.append(nxt)
275
+ return sp.decode_ids([x for x in ys[1:] if x < 65536])
276
+
277
+ @torch.no_grad()
278
+ def translate_beam(self, text: str, direction: str = "en2my", num_beams: int = 4, max_new_tokens: int = 192, length_penalty: float = 0.85, rep_penalty: float = 1.12) -> str:
279
+ self.eval()
280
+ if num_beams <= 1:
281
+ return self.translate(text, direction=direction, max_new_tokens=max_new_tokens, rep_penalty=rep_penalty)
282
+ dir_tok = TOK_2MY if direction == "en2my" else TOK_2EN
283
+ s_ids = [dir_tok] + sp.encode_as_ids(text.strip())[:1020] + [EOS_ID]
284
+ src_t = torch.tensor([s_ids], dtype=torch.long, device=next(self.parameters()).device)
285
+ with torch.amp.autocast("cuda", dtype=torch.bfloat16):
286
+ enc_out, enc_mask = self.encode(src_t, use_ckpt=False)
287
+ beams = [(0.0, [BOS_ID], False)]
288
+ for _ in range(max_new_tokens):
289
+ active = [(sc, seq) for sc, seq, done in beams if not done]
290
+ if not active:
291
+ break
292
+ B_act = len(active)
293
+ tgt_t = torch.tensor([seq for _, seq in active], dtype=torch.long, device=src_t.device)
294
+ dec_out = self.decode(tgt_t, enc_out.expand(B_act, -1, -1), enc_mask.expand(B_act, -1, -1, -1), use_ckpt=False)
295
+ logits = self.lm_head(dec_out[:, -1, :]).float()
296
+ logits[:, [UNK_ID, PAD_ID, BOS_ID]] = -1e9
297
+ logits[:, 65536:] = -1e9
298
+ for b_i, (_, seq) in enumerate(active):
299
+ if rep_penalty != 1.0 and len(seq) > 1:
300
+ seen = torch.tensor(list(set(seq[1:])), dtype=torch.long, device=src_t.device)
301
+ s_l = logits[b_i, seen]
302
+ logits[b_i, seen] = torch.where(s_l > 0, s_l / rep_penalty, s_l * rep_penalty)
303
+ log_probs = F.log_softmax(logits, dim=-1)
304
+ topk_lp, topk_ids = torch.topk(log_probs, num_beams * 2, dim=-1)
305
+ candidates = [(sc, seq, True) for sc, seq, done in beams if done]
306
+ for b_i, (sc, seq) in enumerate(active):
307
+ for k in range(num_beams * 2):
308
+ tok_id = int(topk_ids[b_i, k].item())
309
+ new_sc = sc + float(topk_lp[b_i, k].item())
310
+ candidates.append((new_sc, seq if tok_id == EOS_ID else seq + [tok_id], tok_id == EOS_ID))
311
+ candidates.sort(key=lambda x: x[0] / (max(1, len(x[1]) - 1) ** length_penalty), reverse=True)
312
+ beams = candidates[:num_beams]
313
+ if all(done for _, _, done in beams):
314
+ break
315
+ return sp.decode_ids([x for x in beams[0][1][1:] if x < 65536])
316
+
317
+ # ── 2. Standalone Dataset & Sampler Definitions ──
318
+ class Seq2SeqMemmapDataset(Dataset):
319
+ def __init__(self, data_dir: str, split: str = "train"):
320
+ self.src_path = os.path.join(data_dir, f"{split}_src.bin")
321
+ self.tgt_path = os.path.join(data_dir, f"{split}_tgt.bin")
322
+ self.index = np.load(os.path.join(data_dir, f"{split}_index.npy"))
323
+ self.src_mm = None
324
+ self.tgt_mm = None
325
+ def _ensure_open(self):
326
+ if self.src_mm is None:
327
+ self.src_mm = np.memmap(self.src_path, dtype=np.uint16, mode="r")
328
+ self.tgt_mm = np.memmap(self.tgt_path, dtype=np.uint16, mode="r")
329
+ def __len__(self):
330
+ return len(self.index)
331
+ def __getitem__(self, idx: int):
332
+ self._ensure_open()
333
+ s0, sl, t0, tl, kind = self.index[idx]
334
+ s_raw = self.src_mm[s0 : s0 + sl].astype(np.int64)
335
+ if kind == 2:
336
+ s_raw[s_raw == 0] = TOK_MASK
337
+ src = np.empty(sl + 1, dtype=np.int64)
338
+ src[0] = 65536 + kind
339
+ src[1:] = s_raw
340
+ tgt = self.tgt_mm[t0 : t0 + tl].astype(np.int64)
341
+ return src, tgt
342
+
343
+ class LengthBucketBatchSampler(Sampler):
344
+ def __init__(self, index_arr: np.ndarray, batch_size: int, mega_mult: int = 256, shuffle: bool = True, seed: int = 42, start_batch: int = 0):
345
+ self.lengths = (index_arr[:, 1] + index_arr[:, 3]).astype(np.int32)
346
+ self.batch_size = batch_size
347
+ self.mega_size = batch_size * mega_mult
348
+ self.shuffle = shuffle
349
+ self.seed = seed
350
+ self.start_batch = start_batch
351
+ self.num_batches = len(self.lengths) // batch_size
352
+ def __len__(self):
353
+ return max(0, self.num_batches - self.start_batch)
354
+ def __iter__(self):
355
+ rng = np.random.RandomState(self.seed)
356
+ n = len(self.lengths)
357
+ order = rng.permutation(n) if self.shuffle else np.arange(n)
358
+ batches = []
359
+ for i in range(0, n - self.batch_size + 1, self.mega_size):
360
+ chunk = order[i : i + self.mega_size]
361
+ chunk = chunk[np.argsort(self.lengths[chunk], kind="mergesort")]
362
+ for j in range(0, len(chunk) - self.batch_size + 1, self.batch_size):
363
+ batches.append(chunk[j : j + self.batch_size])
364
+ if self.shuffle:
365
+ rng.shuffle(batches)
366
+ for b in batches[self.start_batch:]:
367
+ yield b.tolist()
368
+
369
+ def collate_nmt(batch):
370
+ B = len(batch)
371
+ max_s = min(1024, ((max(len(x[0]) for x in batch) + 15) // 16) * 16)
372
+ max_t = min(1024, ((max(len(x[1]) - 1 for x in batch) + 15) // 16) * 16)
373
+ src_ids = torch.full((B, max_s), PAD_ID, dtype=torch.long)
374
+ tgt_in = torch.full((B, max_t), PAD_ID, dtype=torch.long)
375
+ labels = torch.full((B, max_t), -100, dtype=torch.long)
376
+ for i, (s, t) in enumerate(batch):
377
+ sl = min(len(s), max_s)
378
+ tl = min(len(t) - 1, max_t)
379
+ src_ids[i, :sl] = torch.from_numpy(s[:sl])
380
+ tgt_in[i, :tl] = torch.from_numpy(t[:tl])
381
+ labels[i, :tl] = torch.from_numpy(t[1 : tl + 1])
382
+ return src_ids, tgt_in, labels
383
+
384
+ AUDIT_SAMPLES = [
385
+ ("En->My Formal", "en2my", "The central bank raised interest rates to control inflation and stabilize the economy."),
386
+ ("En->My Spoken", "en2my", "Honestly, I didn't expect things to turn out like this, but don't worry—we'll figure it out together."),
387
+ ("My->En Formal", "my2en", "ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးတည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။"),
388
+ ("My->En Spoken", "my2en", "အချစ်ရေ မင်းနဲ့ ငါနဲ့ သိပ်ချစ်ခဲ့တာ ���ုရားလည်း သိတယ်၊ သကြားလည်း သိတယ်၊ နတ်လည်း သိတယ်၊ မနာလိုသူတွေ ဂျင်းထည့်တာ ယုံနေတော့ မင်း ရူးသွားပြီပဲ။"),
389
+ ]
390
+
391
+ UNSEEN_TESTS = [
392
+ ("En->My News", "en2my", "Heavy monsoon rains caused severe flooding in several townships, forcing thousands of residents to evacuate to temporary shelters."),
393
+ ("En->My Health", "en2my", "Doctors recommend drinking plenty of clean water and getting enough rest to recover quickly from seasonal flu."),
394
+ ("En->My Spoken", "en2my", "I was stuck in traffic for over an hour this morning, so please start the meeting without me."),
395
+ ("My->En News", "my2en", "နိုင်ငံတကာ ကုန်သွယ်ရေး တိုးတက်လာစေရန်အတွက် နယ်စပ်ကုန်သွယ်ရေးစခန်းများတွင် ကုန်ပစ္စည်းစစ်ဆေးမှုများကို ပိုမိုမြန်ဆန်အောင် ဆောင်ရွက်နေပါသည်။"),
396
+ ("My->En Tech", "my2en", "ဉာဏ်ရည်တုနည်းပညာသည် ပညာရေး၊ ကျန်းမာရေးနှင့် စီးပွားရေးလုပ်ငန်းများတွင် လုပ်ငန်းစဉ်များကို ပိုမိုထိရောက်စေရန် ကူညီပေးနိုင်သည်။"),
397
+ ("My->En Spoken", "my2en", "မနက်ဖြန် မနက်စောစော သွားစရာရှိလို့ ဒီညတော့ စောစောအိပ်တော့မယ်နော်၊ မနက်ကျမှ ဖုန်းဆက်လိုက်မယ်။"),
398
+ ]
399
+
400
+ SPOKEN_IDIOM_PAIRS_PRO = [
401
+ ("my2en", "အချစ်ရေ မင်းနဲ့ ငါနဲ့ သိပ်ချစ်ခဲ့တာ ဘုရားလည်း သိတယ်၊ သကြားလည်း သိတယ်၊ နတ်လည်း သိတယ်၊ မနာလိုသူတွေ ဂျင်းထည့်တာ ယုံနေတော့ မင်း ရူးသွားပြီပဲ။",
402
+ "My love, heaven and earth know how deeply you and I loved each other, yet because you believed the lies fed by jealous people, you've lost your mind."),
403
+ ("my2en", "အချစ်ရေ မင်းနဲ့ ငါနဲ့ သိပ်ချစ်ခဲ့တာ ဘုရားလည်း သိတယ်၊ သိကြားလည်း သိတယ်၊ နတ်လည်း သိတယ်၊ မနာလိုသူတွေ ဂျင်းထည့်တာ ယုံနေတော့ မင်း ယူသွားခဲ့တာပဲ။",
404
+ "My love, Buddha, Sakra, and all the spirits know how deeply you and I loved each other, but because you believed the lies stirred up by jealous people, you ended up marrying someone else."),
405
+ ("my2en", "သူများတွေ ဂျင်းထည့်တာကို လွယ်လွယ်မယုံပါနဲ့၊ ကိုယ်တိုင်သေချာစုံစမ်းပြီးမှ ဆုံးဖြတ်ပါ။",
406
+ "Don't easily believe it when people try to trick you or feed you lies; look into it carefully yourself before deciding."),
407
+ ("my2en", "ငါ မင်းအပေါ် ဘယ်လောက်သစ္စာရှိခဲ့လဲဆိုတာ ဘုရားသိ သိကြားသိပါကွာ။",
408
+ "Heaven and earth know just how loyal I have been to you."),
409
+ ("my2en", "မနာလိုတဲ့လူတွေ ဘေးကနေ သွေးထိုးပြီး ဂျင်းထည့်နေတာကို မင်း ဘာလို့ နားယောင်နေရတာလဲ။",
410
+ "Why are you falling for the lies and manipulation of jealous people trying to stir up trouble from the sidelines?"),
411
+ ("my2en", "စိတ်မပူပါနဲ့ သူငယ်ချင်းရာ၊ အခက်အခဲဆိုတာ ခဏပဲ၊ ငါတို့ အတူတူ ရင်ဆိုင်ကြတာပေါ့။",
412
+ "Don't worry, my friend—hardships are only temporary, and we'll face this together."),
413
+ ("my2en", "နေကောင်းရဲ့လား မိတ်ဆွေ၊ မတွေ့ရတာတောင် တော်တော်ကြာပြီနော်။",
414
+ "How have you been doing, my friend? It's been quite a long time since we last met."),
415
+ ("en2my", "Honestly, I didn't expect things to turn out like this, but don't worry—we'll figure it out together.",
416
+ "ရိုးရိုးသားသားပြောရရင် ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ထားခဲ့ပေမယ့် စိတ်မပူပါနဲ့၊ ကျွန်တော်တို့ အတူတူ အဖြေရှာကြတာပေါ့။"),
417
+ ("en2my", "Don't let jealous people trick you or feed you lies; trust what we've built together.",
418
+ "မနာလိုသူတွေ ဂျင်းထည့်တာကို မယုံမိပါစေနဲ့၊ ငါတို့နှစ်ယောက် အတူတူ တည်ဆောက်ခဲ့တဲ့ အရာကိုပဲ ယုံကြည်ပေးပါ။"),
419
+ ("en2my", "The central bank raised interest rates to control inflation and stabilize the economy.",
420
+ "ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးတည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။"),
421
+ ("my2en", "ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးတည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။",
422
+ "The central bank raised interest rates to control inflation and stabilize the economy."),
423
+ ("en2my", "I was stuck in traffic for over an hour this morning, so please start the meeting without me.",
424
+ "ဒီမနက် တစ်နာရီကျော်လောက် ကားပိတ်မိနေလို့ ကျွန်တော်မပါဘဲ အစည်းအဝေးကို စထားလိုက်ပါနော်။"),
425
+ ("en2my", "Please go ahead and have dinner without me; I'll be home a bit late tonight.",
426
+ "ဒီည အိမ်ပြန်နည်းနည်းနောက်ကျမှာမို့လို့ ကျွန်တော်မပါဘဲ ညစာ အရင်စားနှင့်ကြပါနော်။"),
427
+ ]
428
+
429
+ # ── 3. Initialize 210M-Pro & Run All-Layer Net2Wider+Net2Deeper Transfer ──
430
+ pro_model = MyanmarTranslator84M(
431
+ vocab_size=VOCAB_SIZE, d_model=768, n_heads=12, enc_layers=8, dec_layers=8, ffn_dim=2816, max_seq_len=1024
432
+ ).to(device)
433
+
434
+ n_pro_params = sum(p.numel() for p in pro_model.parameters())
435
+ log(f"🏛️ MyanmarTranslator-210M-Pro Initialized: {n_pro_params:,} parameters ({n_pro_params/1e6:.2f}M) | 8L Encoder + 8L Decoder (d=768)")
436
+
437
+ def transfer_84m_to_210m(student: nn.Module, teacher_ckpt_path: str):
438
+ if not os.path.exists(teacher_ckpt_path):
439
+ log("⚠️ Teacher checkpoint not found, using standard initialization.")
440
+ return
441
+ t_sd = torch.load(teacher_ckpt_path, map_location=device)["model"]
442
+ scale_d = math.sqrt(512.0 / 768.0)
443
+
444
+ with torch.no_grad():
445
+ student.embed_tokens.weight.data.normal_(0.0, 0.004)
446
+ student.embed_tokens.weight.data[:, :512].copy_(t_sd["embed_tokens.weight"] * scale_d)
447
+ student.embed_tokens.weight.data[PAD_ID].zero_()
448
+
449
+ def copy_rmsnorm(s_norm, prefix):
450
+ s_norm.weight.data.fill_(1.0)
451
+ s_norm.weight.data[:512].copy_(t_sd[f"{prefix}.weight"] * scale_d)
452
+
453
+ def copy_self_attn(s_attn, prefix):
454
+ s_attn.qkv.weight.data.normal_(0.0, 0.008)
455
+ s_attn.out.weight.data.normal_(0.0, 0.002)
456
+ tq, tk, tv = t_sd[f"{prefix}.qkv.weight"].chunk(3, dim=0)
457
+ sq, sk, sv = s_attn.qkv.weight.data.chunk(3, dim=0)
458
+ sq[:512, :512].copy_(tq)
459
+ sk[:512, :512].copy_(tk)
460
+ sv[:512, :512].copy_(tv)
461
+ s_attn.out.weight.data[:512, :512].copy_(t_sd[f"{prefix}.out.weight"])
462
+ s_attn.q_norm.weight.data.copy_(t_sd[f"{prefix}.q_norm.weight"])
463
+ s_attn.k_norm.weight.data.copy_(t_sd[f"{prefix}.k_norm.weight"])
464
+
465
+ def copy_cross_attn(s_attn, prefix):
466
+ s_attn.q_proj.weight.data.normal_(0.0, 0.008)
467
+ s_attn.kv_proj.weight.data.normal_(0.0, 0.008)
468
+ s_attn.out.weight.data.normal_(0.0, 0.002)
469
+ s_attn.q_proj.weight.data[:512, :512].copy_(t_sd[f"{prefix}.q_proj.weight"])
470
+ tk, tv = t_sd[f"{prefix}.kv_proj.weight"].chunk(2, dim=0)
471
+ sk, sv = s_attn.kv_proj.weight.data.chunk(2, dim=0)
472
+ sk[:512, :512].copy_(tk)
473
+ sv[:512, :512].copy_(tv)
474
+ s_attn.out.weight.data[:512, :512].copy_(t_sd[f"{prefix}.out.weight"])
475
+ s_attn.q_norm.weight.data.copy_(t_sd[f"{prefix}.q_norm.weight"])
476
+ s_attn.k_norm.weight.data.copy_(t_sd[f"{prefix}.k_norm.weight"])
477
+
478
+ def copy_swiglu(s_mlp, prefix):
479
+ s_mlp.gate_up.weight.data.normal_(0.0, 0.008)
480
+ s_mlp.down.weight.data.normal_(0.0, 0.002)
481
+ tg, tu = t_sd[f"{prefix}.gate_up.weight"].chunk(2, dim=0)
482
+ sg, su = s_mlp.gate_up.weight.data.chunk(2, dim=0)
483
+ sg[:1728, :512].copy_(tg)
484
+ su[:1728, :512].copy_(tu)
485
+ s_mlp.down.weight.data[:512, :1728].copy_(t_sd[f"{prefix}.down.weight"])
486
+
487
+ for i in range(6):
488
+ copy_rmsnorm(student.encoder[i].norm1, f"encoder.{i}.norm1")
489
+ copy_self_attn(student.encoder[i].self_attn, f"encoder.{i}.self_attn")
490
+ copy_rmsnorm(student.encoder[i].norm2, f"encoder.{i}.norm2")
491
+ copy_swiglu(student.encoder[i].mlp, f"encoder.{i}.mlp")
492
+ for i in range(6, 8):
493
+ student.encoder[i].self_attn.out.weight.data.normal_(0.0, 0.001)
494
+ student.encoder[i].mlp.down.weight.data.normal_(0.0, 0.001)
495
+ copy_rmsnorm(student.enc_norm, "enc_norm")
496
+
497
+ for i in range(6):
498
+ copy_rmsnorm(student.decoder[i].norm1, f"decoder.{i}.norm1")
499
+ copy_self_attn(student.decoder[i].self_attn, f"decoder.{i}.self_attn")
500
+ copy_rmsnorm(student.decoder[i].norm2, f"decoder.{i}.norm2")
501
+ copy_cross_attn(student.decoder[i].cross_attn, f"decoder.{i}.cross_attn")
502
+ copy_rmsnorm(student.decoder[i].norm3, f"decoder.{i}.norm3")
503
+ copy_swiglu(student.decoder[i].mlp, f"decoder.{i}.mlp")
504
+ for i in range(6, 8):
505
+ student.decoder[i].self_attn.out.weight.data.normal_(0.0, 0.001)
506
+ student.decoder[i].cross_attn.out.weight.data.normal_(0.0, 0.001)
507
+ student.decoder[i].mlp.down.weight.data.normal_(0.0, 0.001)
508
+ copy_rmsnorm(student.dec_norm, "dec_norm")
509
+
510
+ del t_sd
511
+ gc.collect()
512
+ torch.cuda.empty_cache()
513
+ log("✅ Full All-Layer Net2Wider + Net2Deeper Transfer (84M -> 210M) Complete!")
514
+
515
+ transfer_84m_to_210m(pro_model, TEACHER_84M)
516
+
517
+ # ── 4. DataLoader, Optimizer & Schedule (24,000 Expansion Steps + 1,000 Cooldown Steps) ──
518
+ MICRO_BS_PRO = 128
519
+ GRAD_ACCUM_PRO = 2 # Global BS = 256 sequences/step (~80.6k tokens/step)
520
+ MAX_TOK_CHUNK_PRO = 16384 # Caps peak VRAM at ~17-19 GB on RTX 5090 (zero OOM risk!)
521
+ train_ds_pro = Seq2SeqMemmapDataset(DATA_DIR, "train")
522
+ val_ds_pro = Seq2SeqMemmapDataset(DATA_DIR, "val")
523
+
524
+ STAGE1_STEPS_PRO = 24000 # ~2.0B expansion tokens starting from Val PPL 6.73 (= 58,000 effective steps!)
525
+ WARMUP_STEPS_PRO = 1000
526
+ PEAK_LR_PRO = 3.0e-4
527
+ MIN_LR_PRO = 3.0e-5
528
+ EVAL_EVERY_PRO = 2000
529
+ LOG_EVERY_PRO = 200
530
+
531
+ decay_pro = [p for n, p in pro_model.named_parameters() if p.requires_grad and p.ndim >= 2]
532
+ nodecay_pro = [p for n, p in pro_model.named_parameters() if p.requires_grad and p.ndim < 2]
533
+ pro_opt = torch.optim.AdamW(
534
+ [{"params": decay_pro, "weight_decay": 0.01}, {"params": nodecay_pro, "weight_decay": 0.0}],
535
+ lr=PEAK_LR_PRO, betas=(0.9, 0.98), eps=1e-8, fused=True
536
+ )
537
+
538
+ def get_pro_lr(step: int) -> float:
539
+ if step < WARMUP_STEPS_PRO:
540
+ return PEAK_LR_PRO * (step + 1) / WARMUP_STEPS_PRO
541
+ prog = min(1.0, (step - WARMUP_STEPS_PRO) / max(1, STAGE1_STEPS_PRO - WARMUP_STEPS_PRO))
542
+ return MIN_LR_PRO + 0.5 * (PEAK_LR_PRO - MIN_LR_PRO) * (1.0 + math.cos(math.pi * prog))
543
+
544
+ start_step_pro = 0
545
+ best_pro_nll = float("inf")
546
+ latest_pro_ckpt = os.path.join(PRO_CKPT, "latest_pro.pt")
547
+ if os.path.exists(latest_pro_ckpt):
548
+ ckpt_p = torch.load(latest_pro_ckpt, map_location=device)
549
+ pro_model.load_state_dict(ckpt_p["model"])
550
+ pro_opt.load_state_dict(ckpt_p["optimizer"])
551
+ start_step_pro = int(ckpt_p["step"])
552
+ best_pro_nll = float(ckpt_p.get("best_val_nll", float("inf")))
553
+ log(f"🔄 Resumed 210M-Pro from step {start_step_pro:,} (best_val_nll={best_pro_nll:.4f})")
554
+
555
+ train_sampler_pro = LengthBucketBatchSampler(
556
+ train_ds_pro.index, batch_size=MICRO_BS_PRO, shuffle=True, seed=2026, start_batch=start_step_pro * GRAD_ACCUM_PRO
557
+ )
558
+ val_sampler_pro = LengthBucketBatchSampler(
559
+ val_ds_pro.index, batch_size=MICRO_BS_PRO, shuffle=True, seed=12345, start_batch=0
560
+ )
561
+
562
+ train_loader_pro = DataLoader(train_ds_pro, batch_sampler=train_sampler_pro, collate_fn=collate_nmt, num_workers=6, pin_memory=True, persistent_workers=True)
563
+ val_loader_pro = DataLoader(val_ds_pro, batch_sampler=val_sampler_pro, collate_fn=collate_nmt, num_workers=2, pin_memory=True)
564
+
565
+ @torch.no_grad()
566
+ def evaluate_pro(max_batches: int = 80):
567
+ pro_model.eval()
568
+ tot_nll, tot_tok = 0.0, 0
569
+ for i, (src_ids, tgt_in, labels) in enumerate(val_loader_pro):
570
+ if i >= max_batches:
571
+ break
572
+ B, S = src_ids.shape
573
+ T = tgt_in.size(1)
574
+ sub_bs = max(4, min(B, MAX_TOK_CHUNK_PRO // max(S, T, 1)))
575
+ for s_i in range(0, B, sub_bs):
576
+ sc = src_ids[s_i : s_i + sub_bs].to(device, non_blocking=True)
577
+ tc = tgt_in[s_i : s_i + sub_bs].to(device, non_blocking=True)
578
+ yc = labels[s_i : s_i + sub_bs].to(device, non_blocking=True)
579
+ nll_s, n_v = pro_model.eval_nll(sc, tc, yc)
580
+ tot_nll += nll_s
581
+ tot_tok += n_v
582
+ avg_nll = tot_nll / max(tot_tok, 1)
583
+ return avg_nll, math.exp(min(avg_nll, 20.0))
584
+
585
+ if start_step_pro == 0:
586
+ init_pro_nll, init_pro_ppl = evaluate_pro()
587
+ best_pro_nll = init_pro_nll
588
+ log(f"🔎 Step 0 (Post-Net2Wider/Deeper Warm-Start) | Val NLL: {init_pro_nll:.4f} | Val PPL: {init_pro_ppl:.2f}")
589
+
590
+ encoded_idioms_pro = []
591
+ for d_code, s_txt, t_txt in SPOKEN_IDIOM_PAIRS_PRO:
592
+ dir_tok = TOK_2MY if d_code == "en2my" else TOK_2EN
593
+ s_arr = np.asarray([dir_tok] + sp.encode_as_ids(s_txt.strip()) + [EOS_ID], dtype=np.int64)
594
+ t_arr = np.asarray([BOS_ID] + sp.encode_as_ids(t_txt.strip()) + [EOS_ID], dtype=np.int64)
595
+ encoded_idioms_pro.append((s_arr, t_arr))
596
+
597
+ # ── 5. Stage 1 Expansion Training Loop (24,000 Steps) ──
598
+ log(f"\n>>> Starting Flagship MyanmarTranslator-210M-Pro Stage 1 (Step {start_step_pro:,} -> {STAGE1_STEPS_PRO:,})...")
599
+ pro_model.train()
600
+ pro_opt.zero_grad(set_to_none=True)
601
+
602
+ step = start_step_pro
603
+ micro_idx = start_step_pro * GRAD_ACCUM_PRO
604
+ run_loss = run_toks = 0.0
605
+ t_start = time.time()
606
+
607
+ for src_ids, tgt_in, labels in train_loader_pro:
608
+ micro_idx += 1
609
+ B, S = src_ids.shape
610
+ T = tgt_in.size(1)
611
+ max_len = max(S, T)
612
+ sub_bs = max(4, min(B, MAX_TOK_CHUNK_PRO // max(max_len, 1)))
613
+ n_subs = math.ceil(B / sub_bs)
614
+
615
+ for s_i in range(0, B, sub_bs):
616
+ s_chunk = src_ids[s_i : s_i + sub_bs].to(device, non_blocking=True)
617
+ t_chunk = tgt_in[s_i : s_i + sub_bs].to(device, non_blocking=True)
618
+ y_chunk = labels[s_i : s_i + sub_bs].to(device, non_blocking=True)
619
+
620
+ loss_val, n_val = pro_model.forward_and_backward(
621
+ s_chunk, t_chunk, y_chunk, loss_scale=(GRAD_ACCUM_PRO * n_subs), label_smoothing=0.1
622
+ )
623
+ run_loss += loss_val / n_subs
624
+ run_toks += n_val + int((s_chunk != PAD_ID).sum().item())
625
+
626
+ if micro_idx % GRAD_ACCUM_PRO == 0:
627
+ lr = get_pro_lr(step)
628
+ for pg in pro_opt.param_groups:
629
+ pg["lr"] = lr
630
+ grad_norm = torch.nn.utils.clip_grad_norm_(pro_model.parameters(), 1.0)
631
+ pro_opt.step()
632
+ pro_opt.zero_grad(set_to_none=True)
633
+ step += 1
634
+
635
+ if step % LOG_EVERY_PRO == 0:
636
+ elapsed = time.time() - t_start
637
+ avg_l = run_loss / (LOG_EVERY_PRO * GRAD_ACCUM_PRO)
638
+ est_nll = max(0.0, (avg_l - 0.1 * math.log(VOCAB_SIZE)) / 0.9)
639
+ tok_s = run_toks / max(elapsed, 1e-3)
640
+ peak_gb = torch.cuda.max_memory_allocated() / 1e9
641
+ log(f"Step {step:6d}/{STAGE1_STEPS_PRO} | SmoothLoss: {avg_l:.4f} | EstNLL: {est_nll:.4f} (PPL {math.exp(min(est_nll,15)):6.2f}) | LR: {lr:.2e} | Grad: {grad_norm:.2f} | {tok_s/1e3:.1f}k tok/s | VRAM: {peak_gb:.1f}G")
642
+ run_loss = run_toks = 0.0
643
+ t_start = time.time()
644
+
645
+ if step % EVAL_EVERY_PRO == 0 or step == STAGE1_STEPS_PRO:
646
+ val_nll, val_ppl = evaluate_pro()
647
+ log("\n" + "-"*85)
648
+ log(f"🔎 210M-PRO EVAL @ Step {step:,} | Val NLL: {val_nll:.4f} | Val PPL: {val_ppl:.2f}")
649
+ for tag, d_code, prompt_txt in AUDIT_SAMPLES:
650
+ pred_txt = pro_model.translate(prompt_txt, d_code)
651
+ log(f" [{tag}]: {pred_txt}")
652
+
653
+ torch.save({"step": step, "model": pro_model.state_dict(), "optimizer": pro_opt.state_dict(), "best_val_nll": best_pro_nll, "config": pro_model.config}, latest_pro_ckpt)
654
+
655
+ if val_nll < best_pro_nll:
656
+ best_pro_nll = val_nll
657
+ torch.save({"step": step, "model": pro_model.state_dict(), "val_nll": val_nll, "config": pro_model.config}, os.path.join(PRO_CKPT, "best_pro.pt"))
658
+ log(f" 🌟 New Best 210M-Pro Checkpoint Saved! (Val NLL: {best_pro_nll:.4f})")
659
+ log("-"*85 + "\n")
660
+ pro_model.train()
661
+ t_start = time.time()
662
+
663
+ if step >= STAGE1_STEPS_PRO:
664
+ break
665
+
666
+ # ── 6. Stage 2 Unsmoothed Cooldown & Idiom Polish (1,000 Steps, label_smoothing=0.0) ──
667
+ log(f"\n>>> Starting 210M-Pro Stage 2 Cooldown (1,000 steps | label_smoothing=0.0 | LR 3e-5 -> 4e-6)...")
668
+ best_pro_s1 = torch.load(os.path.join(PRO_CKPT, "best_pro.pt"), map_location=device)
669
+ pro_model.load_state_dict(best_pro_s1["model"])
670
+
671
+ pro_s2_opt = torch.optim.AdamW(
672
+ [{"params": decay_pro, "weight_decay": 0.01}, {"params": nodecay_pro, "weight_decay": 0.0}],
673
+ lr=3e-5, betas=(0.9, 0.98), eps=1e-8, fused=True
674
+ )
675
+ clean_mask_pro = (train_ds_pro.index[:, 4] < 2) & (train_ds_pro.index[:, 1] >= 16) & (train_ds_pro.index[:, 1] <= 256)
676
+ clean_idx_pro = np.flatnonzero(clean_mask_pro)
677
+ rng_pro = np.random.RandomState(2026)
678
+
679
+ pro_model.train()
680
+ for s2_step in range(1, 1001):
681
+ prog = s2_step / 1000.0
682
+ lr = 4e-6 + 0.5 * (3e-5 - 4e-6) * (1.0 + math.cos(math.pi * prog))
683
+ for pg in pro_s2_opt.param_groups:
684
+ pg["lr"] = lr
685
+
686
+ picked = rng_pro.choice(clean_idx_pro, size=88, replace=False)
687
+ batch_items = [train_ds_pro[int(ix)] for ix in picked]
688
+ for _ in range(8):
689
+ batch_items.append(encoded_idioms_pro[int(rng_pro.randint(len(encoded_idioms_pro)))])
690
+
691
+ src_ids, tgt_in, labels = collate_nmt(batch_items)
692
+ pro_s2_opt.zero_grad(set_to_none=True)
693
+ pro_model.forward_and_backward(
694
+ src_ids.to(device, non_blocking=True),
695
+ tgt_in.to(device, non_blocking=True),
696
+ labels.to(device, non_blocking=True),
697
+ loss_scale=1.0, label_smoothing=0.0
698
+ )
699
+ torch.nn.utils.clip_grad_norm_(pro_model.parameters(), 1.0)
700
+ pro_s2_opt.step()
701
+
702
+ if s2_step % 250 == 0:
703
+ val_nll, val_ppl = evaluate_pro()
704
+ log(f"🔎 210M-Pro Stage 2 Step {s2_step}/1000 | Val NLL: {val_nll:.4f} (PPL {val_ppl:.2f})")
705
+ if val_nll <= best_pro_nll + 0.01:
706
+ best_pro_nll = min(best_pro_nll, val_nll)
707
+ torch.save({"step": STAGE1_STEPS_PRO + s2_step, "model": pro_model.state_dict(), "val_nll": val_nll, "config": pro_model.config},
708
+ os.path.join(PRO_CKPT, "best_pro.pt"))
709
+ pro_model.train()
710
+
711
+ # ── 7. Final Benchmark & Private HF Upload: freococo/MyanmarTranslator-210M-Pro ──
712
+ best_pro_final = torch.load(os.path.join(PRO_CKPT, "best_pro.pt"), map_location=device)
713
+ pro_model.load_state_dict(best_pro_final["model"])
714
+ pro_model.eval()
715
+ final_pro_nll = float(best_pro_final["val_nll"])
716
+ final_pro_ppl = math.exp(final_pro_nll)
717
+
718
+ log("\n" + "="*90)
719
+ log(f"🏆 MYANMARTRANSLATOR-210M-PRO FINAL BENCHMARK | Val NLL: {final_pro_nll:.4f} (PPL {final_pro_ppl:.2f})")
720
+ log("="*90)
721
+ for tag, d_code, prompt_txt in AUDIT_SAMPLES + UNSEEN_TESTS:
722
+ pred_txt = pro_model.translate_beam(prompt_txt, d_code, num_beams=4)
723
+ log(f" [{tag} (Beam=4)]: {pred_txt}")
724
+ log("="*90)
725
+
726
+ CKPT_SUB_PRO = os.path.join(PRO_EXPORT, "checkpoint-step-25000")
727
+ shutil.rmtree(PRO_EXPORT, ignore_errors=True)
728
+ os.makedirs(CKPT_SUB_PRO, exist_ok=True)
729
+
730
+ sd_fp32 = {k: v.detach().cpu().float().contiguous() for k, v in pro_model.state_dict().items() if k != "lm_head.weight"}
731
+ sd_bf16 = {k: v.to(torch.bfloat16).contiguous() for k, v in sd_fp32.items()}
732
+ save_file(sd_bf16, os.path.join(PRO_EXPORT, "model.safetensors"))
733
+ save_file(sd_fp32, os.path.join(CKPT_SUB_PRO, "model.safetensors"))
734
+
735
+ if os.path.exists(latest_pro_ckpt):
736
+ os.remove(latest_pro_ckpt)
737
+ torch.save(best_pro_final, os.path.join(CKPT_SUB_PRO, "training_state.pt"))
738
+
739
+ BASE_EXPORT = "/workspace/MyanmarTranslator-84M-Base-export"
740
+ for folder, dt in [(PRO_EXPORT, "bfloat16"), (CKPT_SUB_PRO, "float32")]:
741
+ for fn in ["configuration_myanmartranslator.py", "modeling_myanmartranslator.py", "generation_config.json",
742
+ "tokenizer.model", "tokenizer_config.json", "special_tokens_map.json"]:
743
+ shutil.copyfile(os.path.join(BASE_EXPORT, fn), os.path.join(folder, fn))
744
+ with open(os.path.join(BASE_EXPORT, "config.json"), "r") as f:
745
+ cfg_d = json.load(f)
746
+ cfg_d.update(pro_model.config)
747
+ cfg_d["torch_dtype"] = dt
748
+ cfg_d["val_nll"] = round(final_pro_nll, 4)
749
+ cfg_d["val_ppl"] = round(final_pro_ppl, 2)
750
+ with open(os.path.join(folder, "config.json"), "w", encoding="utf-8") as f:
751
+ json.dump(cfg_d, f, indent=2)
752
+
753
+ full_output_log = "\n".join(LOG_LINES) + "\n"
754
+ for folder in (PRO_EXPORT, CKPT_SUB_PRO):
755
+ with open(os.path.join(folder, "training_codes.txt"), "w", encoding="utf-8") as f:
756
+ f.write(CELL_SOURCE_CODE)
757
+ with open(os.path.join(folder, "trainingcelloutputwitheval.txt"), "w", encoding="utf-8") as f:
758
+ f.write(full_output_log)
759
+
760
+ with open(os.path.join(PRO_EXPORT, "README.md"), "w", encoding="utf-8") as f:
761
+ f.write(f"""---
762
+ language:
763
+ - my
764
+ - en
765
+ license: apache-2.0
766
+ library_name: transformers
767
+ tags:
768
+ - Translation
769
+ - Safetensors
770
+ - Burmese
771
+ - English
772
+ - MyanmarLLM
773
+ - flagship
774
+ - custom_code
775
+ ---
776
+
777
+ # MyanmarTranslator-210M-Pro
778
+
779
+ Flagship `210.90M`-parameter **8-Layer Bidirectional Encoder + 8-Layer Cross-Attentive Decoder** (`d_model=768`, `n_heads=12`, `ffn_dim=2816` SwiGLU) for **English ↔ Burmese (`my`)**, warm-started via All-Layer Net2Wider+Net2Deeper expansion from `MyanmarTranslator-84M-Base`.
780
+
781
+ - **Validation NLL:** `{final_pro_nll:.4f}` (**`Val PPL: {final_pro_ppl:.2f}`**)
782
+ - **Reproducibility Artifacts:** `training_codes.txt` & `trainingcelloutputwitheval.txt`
783
+ """)
784
+
785
+ api = HfApi()
786
+ create_repo(PRO_REPO_ID, repo_type="model", private=True, exist_ok=True)
787
+ api.upload_folder(
788
+ folder_path=PRO_EXPORT,
789
+ repo_id=PRO_REPO_ID,
790
+ repo_type="model",
791
+ commit_message=f"Release Flagship MyanmarTranslator-210M-Pro (Val PPL {final_pro_ppl:.2f} + code & logs)",
792
+ )
793
+ log(f"\n🔒🚀 SUCCESS! Flagship MyanmarTranslator-210M-Pro Live at: https://huggingface.co/{PRO_REPO_ID}")
trainingcelloutputwitheval.txt ADDED
@@ -0,0 +1,264 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 🟢 Standalone Kernel Ready for 210M-Pro | GPU Free: 33.14/33.67 GB | Disk Free: 46.2 GB
2
+ 🏛️ MyanmarTranslator-210M-Pro Initialized: 210,897,408 parameters (210.90M) | 8L Encoder + 8L Decoder (d=768)
3
+ ✅ Full All-Layer Net2Wider + Net2Deeper Transfer (84M -> 210M) Complete!
4
+ 🔎 Step 0 (Post-Net2Wider/Deeper Warm-Start) | Val NLL: 1.9073 | Val PPL: 6.73
5
+
6
+ >>> Starting Flagship MyanmarTranslator-210M-Pro Stage 1 (Step 0 -> 24,000)...
7
+ Step 200/24000 | SmoothLoss: 2.8552 | EstNLL: 1.9399 (PPL 6.96) | LR: 6.00e-05 | Grad: 0.35 | 87.8k tok/s | VRAM: 21.3G
8
+ Step 400/24000 | SmoothLoss: 2.8429 | EstNLL: 1.9263 (PPL 6.86) | LR: 1.20e-04 | Grad: 0.36 | 87.8k tok/s | VRAM: 21.3G
9
+ Step 600/24000 | SmoothLoss: 2.8464 | EstNLL: 1.9302 (PPL 6.89) | LR: 1.80e-04 | Grad: 0.33 | 88.0k tok/s | VRAM: 21.3G
10
+ Step 800/24000 | SmoothLoss: 2.8775 | EstNLL: 1.9648 (PPL 7.13) | LR: 2.40e-04 | Grad: 0.37 | 88.8k tok/s | VRAM: 21.3G
11
+ Step 1000/24000 | SmoothLoss: 2.8745 | EstNLL: 1.9614 (PPL 7.11) | LR: 3.00e-04 | Grad: 0.42 | 89.1k tok/s | VRAM: 21.3G
12
+ Step 1200/24000 | SmoothLoss: 2.8935 | EstNLL: 1.9825 (PPL 7.26) | LR: 3.00e-04 | Grad: 0.45 | 89.0k tok/s | VRAM: 21.3G
13
+ Step 1400/24000 | SmoothLoss: 2.8996 | EstNLL: 1.9892 (PPL 7.31) | LR: 3.00e-04 | Grad: 0.26 | 88.9k tok/s | VRAM: 21.3G
14
+ Step 1600/24000 | SmoothLoss: 2.9059 | EstNLL: 1.9963 (PPL 7.36) | LR: 3.00e-04 | Grad: 0.29 | 88.3k tok/s | VRAM: 21.3G
15
+ Step 1800/24000 | SmoothLoss: 2.9176 | EstNLL: 2.0093 (PPL 7.46) | LR: 2.99e-04 | Grad: 0.37 | 88.9k tok/s | VRAM: 21.3G
16
+ Step 2000/24000 | SmoothLoss: 2.9026 | EstNLL: 1.9926 (PPL 7.33) | LR: 2.99e-04 | Grad: 0.31 | 88.3k tok/s | VRAM: 21.3G
17
+
18
+ -------------------------------------------------------------------------------------
19
+ 🔎 210M-PRO EVAL @ Step 2,000 | Val NLL: 1.9182 | Val PPL: 6.81
20
+ [En->My Formal]: ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးတည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
21
+ [En->My Spoken]: အမှန်တော့ ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ထားဘဲ စိတ်မပူပါနဲ့ အတူတူတွေးကြည့်ကြတာပေါ့။
22
+ [My->En Formal]: The central bank raised interest rates to control inflation and stabilize the economy.
23
+ [My->En Spoken]: Love, you and I loved each other so much that they knew everything about it—buddha knows it, sugar knows it, and spirits know it. Those who are jealous of me believe in being deceived, so you've gone mad.
24
+ -------------------------------------------------------------------------------------
25
+
26
+ Step 2200/24000 | SmoothLoss: 2.9118 | EstNLL: 2.0029 (PPL 7.41) | LR: 2.98e-04 | Grad: 0.28 | 88.1k tok/s | VRAM: 21.3G
27
+ Step 2400/24000 | SmoothLoss: 2.9170 | EstNLL: 2.0087 (PPL 7.45) | LR: 2.98e-04 | Grad: 0.24 | 88.9k tok/s | VRAM: 21.3G
28
+ Step 2600/24000 | SmoothLoss: 2.9014 | EstNLL: 1.9913 (PPL 7.32) | LR: 2.97e-04 | Grad: 0.37 | 87.5k tok/s | VRAM: 21.3G
29
+ Step 2800/24000 | SmoothLoss: 2.9057 | EstNLL: 1.9960 (PPL 7.36) | LR: 2.96e-04 | Grad: 0.57 | 88.1k tok/s | VRAM: 21.3G
30
+ Step 3000/24000 | SmoothLoss: 2.9090 | EstNLL: 1.9998 (PPL 7.39) | LR: 2.95e-04 | Grad: 0.36 | 88.2k tok/s | VRAM: 21.3G
31
+ Step 3200/24000 | SmoothLoss: 2.9009 | EstNLL: 1.9907 (PPL 7.32) | LR: 2.94e-04 | Grad: 0.30 | 88.6k tok/s | VRAM: 21.3G
32
+ Step 3400/24000 | SmoothLoss: 2.9048 | EstNLL: 1.9951 (PPL 7.35) | LR: 2.93e-04 | Grad: 0.31 | 89.2k tok/s | VRAM: 21.3G
33
+ Step 3600/24000 | SmoothLoss: 2.9222 | EstNLL: 2.0144 (PPL 7.50) | LR: 2.92e-04 | Grad: 0.34 | 88.8k tok/s | VRAM: 21.3G
34
+ Step 3800/24000 | SmoothLoss: 2.9093 | EstNLL: 2.0001 (PPL 7.39) | LR: 2.90e-04 | Grad: 0.49 | 89.1k tok/s | VRAM: 21.3G
35
+ Step 4000/24000 | SmoothLoss: 2.9209 | EstNLL: 2.0130 (PPL 7.49) | LR: 2.89e-04 | Grad: 0.44 | 88.7k tok/s | VRAM: 21.3G
36
+
37
+ -------------------------------------------------------------------------------------
38
+ 🔎 210M-PRO EVAL @ Step 4,000 | Val NLL: 1.9249 | Val PPL: 6.85
39
+ [En->My Formal]: ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေး တည်ငြိမ်စေရန် အတိုးနှုန်းများ မြှင့်တင်ခဲ့သည်။
40
+ [En->My Spoken]: အမှန်ဆိုရရင် ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ထားပါဘူး၊ စိတ်မပူပါနဲ့၊ အတူတူစဉ်းစားကြည့်ကြမှာပဲ
41
+ [My->En Formal]: The central bank has raised interest rates to control inflation and stabilize the economy.
42
+ [My->En Spoken]: Love, you and I have loved each other so much that Buddha knows it, sugar knows it, celestial beings also know it. Those who envy are just jealous of me, so you've gone mad.
43
+ -------------------------------------------------------------------------------------
44
+
45
+ Step 4200/24000 | SmoothLoss: 2.9093 | EstNLL: 2.0001 (PPL 7.39) | LR: 2.87e-04 | Grad: 0.23 | 88.4k tok/s | VRAM: 21.3G
46
+ Step 4400/24000 | SmoothLoss: 2.9011 | EstNLL: 1.9909 (PPL 7.32) | LR: 2.86e-04 | Grad: 0.24 | 88.5k tok/s | VRAM: 21.3G
47
+ Step 4600/24000 | SmoothLoss: 2.9157 | EstNLL: 2.0072 (PPL 7.44) | LR: 2.84e-04 | Grad: 0.31 | 88.8k tok/s | VRAM: 21.3G
48
+ Step 4800/24000 | SmoothLoss: 2.9097 | EstNLL: 2.0005 (PPL 7.39) | LR: 2.82e-04 | Grad: 0.31 | 88.7k tok/s | VRAM: 21.3G
49
+ Step 5000/24000 | SmoothLoss: 2.9163 | EstNLL: 2.0078 (PPL 7.45) | LR: 2.80e-04 | Grad: 0.26 | 88.5k tok/s | VRAM: 21.3G
50
+ Step 5200/24000 | SmoothLoss: 2.8903 | EstNLL: 1.9789 (PPL 7.23) | LR: 2.78e-04 | Grad: 0.50 | 88.2k tok/s | VRAM: 21.3G
51
+ Step 5400/24000 | SmoothLoss: 2.9017 | EstNLL: 1.9916 (PPL 7.33) | LR: 2.76e-04 | Grad: 0.31 | 88.6k tok/s | VRAM: 21.3G
52
+ Step 5600/24000 | SmoothLoss: 2.9062 | EstNLL: 1.9966 (PPL 7.36) | LR: 2.74e-04 | Grad: 0.27 | 88.6k tok/s | VRAM: 21.3G
53
+ Step 5800/24000 | SmoothLoss: 2.8924 | EstNLL: 1.9813 (PPL 7.25) | LR: 2.72e-04 | Grad: 0.28 | 89.0k tok/s | VRAM: 21.3G
54
+ Step 6000/24000 | SmoothLoss: 2.8909 | EstNLL: 1.9797 (PPL 7.24) | LR: 2.70e-04 | Grad: 0.42 | 88.3k tok/s | VRAM: 21.3G
55
+
56
+ -------------------------------------------------------------------------------------
57
+ 🔎 210M-PRO EVAL @ Step 6,000 | Val NLL: 1.9110 | Val PPL: 6.76
58
+ [En->My Formal]: ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးကို တည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
59
+ [En->My Spoken]: တကယ်တော့ ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ထားဘဲ စိတ်မပူပါနဲ့ အတူတူတွေးကြည့်မှာပါ။
60
+ [My->En Formal]: The central bank has raised interest rates to control inflation and stabilize the economy.
61
+ [My->En Spoken]: In love, I knew that you and I loved each other very much, Buddha knows, sugar knows, and celestial beings know. Those who envy me are teasing you, so you've gone mad.
62
+ -------------------------------------------------------------------------------------
63
+
64
+ Step 6200/24000 | SmoothLoss: 2.8958 | EstNLL: 1.9851 (PPL 7.28) | LR: 2.67e-04 | Grad: 0.26 | 88.1k tok/s | VRAM: 21.3G
65
+ Step 6400/24000 | SmoothLoss: 2.9049 | EstNLL: 1.9951 (PPL 7.35) | LR: 2.65e-04 | Grad: 0.30 | 88.6k tok/s | VRAM: 21.3G
66
+ Step 6600/24000 | SmoothLoss: 2.8825 | EstNLL: 1.9703 (PPL 7.17) | LR: 2.62e-04 | Grad: 0.30 | 88.0k tok/s | VRAM: 21.3G
67
+ Step 6800/24000 | SmoothLoss: 2.8893 | EstNLL: 1.9778 (PPL 7.23) | LR: 2.60e-04 | Grad: 0.27 | 88.0k tok/s | VRAM: 21.3G
68
+ Step 7000/24000 | SmoothLoss: 2.9112 | EstNLL: 2.0022 (PPL 7.41) | LR: 2.57e-04 | Grad: 0.36 | 89.8k tok/s | VRAM: 21.3G
69
+ Step 7200/24000 | SmoothLoss: 2.8744 | EstNLL: 1.9613 (PPL 7.11) | LR: 2.54e-04 | Grad: 0.31 | 87.8k tok/s | VRAM: 21.3G
70
+ Step 7400/24000 | SmoothLoss: 2.8892 | EstNLL: 1.9777 (PPL 7.23) | LR: 2.52e-04 | Grad: 0.80 | 89.3k tok/s | VRAM: 21.3G
71
+ Step 7600/24000 | SmoothLoss: 2.8941 | EstNLL: 1.9832 (PPL 7.27) | LR: 2.49e-04 | Grad: 0.29 | 89.4k tok/s | VRAM: 21.3G
72
+ Step 7800/24000 | SmoothLoss: 2.8854 | EstNLL: 1.9735 (PPL 7.20) | LR: 2.46e-04 | Grad: 0.36 | 88.9k tok/s | VRAM: 21.3G
73
+ Step 8000/24000 | SmoothLoss: 2.8866 | EstNLL: 1.9748 (PPL 7.21) | LR: 2.43e-04 | Grad: 0.33 | 89.1k tok/s | VRAM: 21.3G
74
+
75
+ -------------------------------------------------------------------------------------
76
+ 🔎 210M-PRO EVAL @ Step 8,000 | Val NLL: 1.8961 | Val PPL: 6.66
77
+ [En->My Formal]: ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးကို တည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
78
+ [En->My Spoken]: တကယ်တော့လည်း ဒီလိုဖြစ်လာလိမ့်မယ်လို့ မမျှော်လင့်ထားပါဘူး၊ စိတ်မပူပါနဲ့၊ အတူတွေးကြည့်ကြမှာပဲ။
79
+ [My->En Formal]: The Central Bank has raised interest rates to control inflation and stabilize the economy.
80
+ [My->En Spoken]: Haha, I loved you and me very much. Buddha knows, sugar knows, spirits know knowing. Because those who envy are jealous of you, you have gone crazy.
81
+ 🌟 New Best 210M-Pro Checkpoint Saved! (Val NLL: 1.8961)
82
+ -------------------------------------------------------------------------------------
83
+
84
+ Step 8200/24000 | SmoothLoss: 2.8730 | EstNLL: 1.9598 (PPL 7.10) | LR: 2.40e-04 | Grad: 0.45 | 88.7k tok/s | VRAM: 21.3G
85
+ Step 8400/24000 | SmoothLoss: 2.8723 | EstNLL: 1.9590 (PPL 7.09) | LR: 2.37e-04 | Grad: 0.35 | 87.7k tok/s | VRAM: 21.3G
86
+ Step 8600/24000 | SmoothLoss: 2.8833 | EstNLL: 1.9712 (PPL 7.18) | LR: 2.34e-04 | Grad: 0.31 | 89.1k tok/s | VRAM: 21.3G
87
+ Step 8800/24000 | SmoothLoss: 2.8756 | EstNLL: 1.9626 (PPL 7.12) | LR: 2.30e-04 | Grad: 0.22 | 87.5k tok/s | VRAM: 21.3G
88
+ Step 9000/24000 | SmoothLoss: 2.9027 | EstNLL: 1.9928 (PPL 7.34) | LR: 2.27e-04 | Grad: 0.37 | 89.0k tok/s | VRAM: 21.3G
89
+ Step 9200/24000 | SmoothLoss: 2.9001 | EstNLL: 1.9899 (PPL 7.31) | LR: 2.24e-04 | Grad: 0.24 | 89.9k tok/s | VRAM: 21.3G
90
+ Step 9400/24000 | SmoothLoss: 2.8625 | EstNLL: 1.9480 (PPL 7.01) | LR: 2.20e-04 | Grad: 0.30 | 88.0k tok/s | VRAM: 21.3G
91
+ Step 9600/24000 | SmoothLoss: 2.8750 | EstNLL: 1.9620 (PPL 7.11) | LR: 2.17e-04 | Grad: 0.34 | 88.6k tok/s | VRAM: 21.3G
92
+ Step 9800/24000 | SmoothLoss: 2.8761 | EstNLL: 1.9632 (PPL 7.12) | LR: 2.14e-04 | Grad: 0.25 | 88.7k tok/s | VRAM: 21.3G
93
+ Step 10000/24000 | SmoothLoss: 2.8638 | EstNLL: 1.9496 (PPL 7.03) | LR: 2.10e-04 | Grad: 0.28 | 88.9k tok/s | VRAM: 21.3G
94
+
95
+ -------------------------------------------------------------------------------------
96
+ 🔎 210M-PRO EVAL @ Step 10,000 | Val NLL: 1.8687 | Val PPL: 6.48
97
+ [En->My Formal]: ဗဟိုဘဏ်က ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးတည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
98
+ [En->My Spoken]: အမှန်ဆို ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ထားဘဲ စိတ်မပူပါနဲ့လို့ အတူတူတွေးမိပါတယ်။
99
+ [My->En Formal]: The Central Bank has raised interest rates to control inflation and stabilize the economy.
100
+ [My->En Spoken]: My love. I knew that you and I really loved each other, Buddha also knows, Sanka also knows, Nat also knows, those who are jealous believe that they are being tricked, so you have gone mad.
101
+ 🌟 New Best 210M-Pro Checkpoint Saved! (Val NLL: 1.8687)
102
+ -------------------------------------------------------------------------------------
103
+
104
+ Step 10200/24000 | SmoothLoss: 2.8782 | EstNLL: 1.9656 (PPL 7.14) | LR: 2.07e-04 | Grad: 0.31 | 89.1k tok/s | VRAM: 21.3G
105
+ Step 10400/24000 | SmoothLoss: 2.8700 | EstNLL: 1.9564 (PPL 7.07) | LR: 2.03e-04 | Grad: 0.32 | 88.7k tok/s | VRAM: 21.3G
106
+ Step 10600/24000 | SmoothLoss: 2.8692 | EstNLL: 1.9556 (PPL 7.07) | LR: 2.00e-04 | Grad: 0.29 | 87.7k tok/s | VRAM: 21.3G
107
+ Step 10800/24000 | SmoothLoss: 2.8580 | EstNLL: 1.9431 (PPL 6.98) | LR: 1.96e-04 | Grad: 0.27 | 88.5k tok/s | VRAM: 21.3G
108
+ Step 11000/24000 | SmoothLoss: 2.8549 | EstNLL: 1.9396 (PPL 6.96) | LR: 1.92e-04 | Grad: 0.34 | 88.8k tok/s | VRAM: 21.3G
109
+ Step 11200/24000 | SmoothLoss: 2.8666 | EstNLL: 1.9526 (PPL 7.05) | LR: 1.89e-04 | Grad: 0.44 | 88.7k tok/s | VRAM: 21.3G
110
+ Step 11400/24000 | SmoothLoss: 2.8623 | EstNLL: 1.9478 (PPL 7.01) | LR: 1.85e-04 | Grad: 0.45 | 89.1k tok/s | VRAM: 21.3G
111
+ Step 11600/24000 | SmoothLoss: 2.8479 | EstNLL: 1.9318 (PPL 6.90) | LR: 1.82e-04 | Grad: 0.38 | 88.6k tok/s | VRAM: 21.3G
112
+ Step 11800/24000 | SmoothLoss: 2.8277 | EstNLL: 1.9094 (PPL 6.75) | LR: 1.78e-04 | Grad: 0.35 | 88.3k tok/s | VRAM: 21.3G
113
+ Step 12000/24000 | SmoothLoss: 2.8467 | EstNLL: 1.9305 (PPL 6.89) | LR: 1.74e-04 | Grad: 0.31 | 88.8k tok/s | VRAM: 21.3G
114
+
115
+ -------------------------------------------------------------------------------------
116
+ 🔎 210M-PRO EVAL @ Step 12,000 | Val NLL: 1.8416 | Val PPL: 6.31
117
+ [En->My Formal]: ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးကို တည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
118
+ [En->My Spoken]: အမှန်အတိုင်းပြောရရင် ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ထားဘဲ စိတ်မပူပါနဲ့လို့ ကျွန်တော်တို့ အတူတူတွေးကြည့်ပါတယ်။
119
+ [My->En Formal]: The central bank has raised interest rates to control inflation and stabilize the economy.
120
+ [My->En Spoken]: Love, I and you deeply loved each other; Buddha knows, Sakka knows, Nat knows. Those who envy me are making jokes, so I'm going crazy.
121
+ 🌟 New Best 210M-Pro Checkpoint Saved! (Val NLL: 1.8416)
122
+ -------------------------------------------------------------------------------------
123
+
124
+ Step 12200/24000 | SmoothLoss: 2.8468 | EstNLL: 1.9306 (PPL 6.89) | LR: 1.71e-04 | Grad: 0.25 | 89.1k tok/s | VRAM: 21.3G
125
+ Step 12400/24000 | SmoothLoss: 2.8404 | EstNLL: 1.9236 (PPL 6.85) | LR: 1.67e-04 | Grad: 0.38 | 88.3k tok/s | VRAM: 21.3G
126
+ Step 12600/24000 | SmoothLoss: 2.8348 | EstNLL: 1.9173 (PPL 6.80) | LR: 1.63e-04 | Grad: 0.33 | 87.5k tok/s | VRAM: 21.3G
127
+ Step 12800/24000 | SmoothLoss: 2.8338 | EstNLL: 1.9162 (PPL 6.80) | LR: 1.59e-04 | Grad: 0.23 | 88.3k tok/s | VRAM: 21.3G
128
+ Step 13000/24000 | SmoothLoss: 2.8257 | EstNLL: 1.9072 (PPL 6.73) | LR: 1.56e-04 | Grad: 0.41 | 88.8k tok/s | VRAM: 21.3G
129
+ Step 13200/24000 | SmoothLoss: 2.8447 | EstNLL: 1.9283 (PPL 6.88) | LR: 1.52e-04 | Grad: 0.40 | 89.0k tok/s | VRAM: 21.3G
130
+ Step 13400/24000 | SmoothLoss: 2.8288 | EstNLL: 1.9106 (PPL 6.76) | LR: 1.48e-04 | Grad: 0.25 | 88.0k tok/s | VRAM: 21.3G
131
+ Step 13600/24000 | SmoothLoss: 2.8357 | EstNLL: 1.9183 (PPL 6.81) | LR: 1.45e-04 | Grad: 0.27 | 89.3k tok/s | VRAM: 21.3G
132
+ Step 13800/24000 | SmoothLoss: 2.8297 | EstNLL: 1.9116 (PPL 6.76) | LR: 1.41e-04 | Grad: 0.28 | 88.7k tok/s | VRAM: 21.3G
133
+ Step 14000/24000 | SmoothLoss: 2.8323 | EstNLL: 1.9145 (PPL 6.78) | LR: 1.38e-04 | Grad: 0.26 | 89.7k tok/s | VRAM: 21.3G
134
+
135
+ -------------------------------------------------------------------------------------
136
+ 🔎 210M-PRO EVAL @ Step 14,000 | Val NLL: 1.8221 | Val PPL: 6.18
137
+ [En->My Formal]: ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးကို တည်ငြိမ်စေရန် ဗဟိုဘဏ်၏ အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
138
+ [En->My Spoken]: တကယ်ကတော့ ဒီလိုဖြစ်လာမယ်လို့ မထင်ထားပဲ မစိုးရိမ်ပါနဲ့ အတူတူ အဖြေရှာကြမယ်
139
+ [My->En Formal]: The central bank has raised interest rates to control inflation and stabilize the economy.
140
+ [My->En Spoken]: My dear, I knew that you and I loved each other very much—I knew God too, I knew God too, I knew God too, because those who envy me are making mistakes, so you've gone mad.
141
+ 🌟 New Best 210M-Pro Checkpoint Saved! (Val NLL: 1.8221)
142
+ -------------------------------------------------------------------------------------
143
+
144
+ Step 14200/24000 | SmoothLoss: 2.8288 | EstNLL: 1.9107 (PPL 6.76) | LR: 1.34e-04 | Grad: 0.31 | 88.4k tok/s | VRAM: 21.3G
145
+ Step 14400/24000 | SmoothLoss: 2.8297 | EstNLL: 1.9116 (PPL 6.76) | LR: 1.30e-04 | Grad: 0.27 | 88.6k tok/s | VRAM: 21.3G
146
+ Step 14600/24000 | SmoothLoss: 2.8253 | EstNLL: 1.9067 (PPL 6.73) | LR: 1.27e-04 | Grad: 0.25 | 88.6k tok/s | VRAM: 21.3G
147
+ Step 14800/24000 | SmoothLoss: 2.8133 | EstNLL: 1.8934 (PPL 6.64) | LR: 1.23e-04 | Grad: 0.40 | 88.4k tok/s | VRAM: 21.3G
148
+ Step 15000/24000 | SmoothLoss: 2.8132 | EstNLL: 1.8933 (PPL 6.64) | LR: 1.20e-04 | Grad: 0.25 | 87.9k tok/s | VRAM: 21.3G
149
+ Step 15200/24000 | SmoothLoss: 2.8218 | EstNLL: 1.9028 (PPL 6.70) | LR: 1.16e-04 | Grad: 0.46 | 88.6k tok/s | VRAM: 21.3G
150
+ Step 15400/24000 | SmoothLoss: 2.8295 | EstNLL: 1.9114 (PPL 6.76) | LR: 1.13e-04 | Grad: 0.32 | 89.4k tok/s | VRAM: 21.3G
151
+ Step 15600/24000 | SmoothLoss: 2.8077 | EstNLL: 1.8872 (PPL 6.60) | LR: 1.10e-04 | Grad: 0.30 | 88.1k tok/s | VRAM: 21.3G
152
+ Step 15800/24000 | SmoothLoss: 2.8321 | EstNLL: 1.9143 (PPL 6.78) | LR: 1.06e-04 | Grad: 0.31 | 89.3k tok/s | VRAM: 21.3G
153
+ Step 16000/24000 | SmoothLoss: 2.8068 | EstNLL: 1.8862 (PPL 6.59) | LR: 1.03e-04 | Grad: 0.29 | 87.4k tok/s | VRAM: 21.3G
154
+
155
+ -------------------------------------------------------------------------------------
156
+ 🔎 210M-PRO EVAL @ Step 16,000 | Val NLL: 1.7981 | Val PPL: 6.04
157
+ [En->My Formal]: ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးကို တည်ငြိမ်စေရန် ဗဟိုဘဏ်သည် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
158
+ [En->My Spoken]: အမှန်ကတော့ ဒီလိုဖြစ်လာလိမ့်မယ်လို့ မမျှော်လင့်ထားဘဲ မပူနဲ့ အတူတူတွေးကြည့်ကြမှာပဲ
159
+ [My->En Formal]: The Central Bank has raised interest rates to control inflation and stabilize the economy.
160
+ [My->En Spoken]: My love, I knew that you and I loved each other very much—buddha knows this too, sugar knows this too, celestial knows this too. Those who envy me are being deceived; because of this, you have gone mad.
161
+ 🌟 New Best 210M-Pro Checkpoint Saved! (Val NLL: 1.7981)
162
+ -------------------------------------------------------------------------------------
163
+
164
+ Step 16200/24000 | SmoothLoss: 2.8005 | EstNLL: 1.8792 (PPL 6.55) | LR: 9.97e-05 | Grad: 0.30 | 88.9k tok/s | VRAM: 21.3G
165
+ Step 16400/24000 | SmoothLoss: 2.8184 | EstNLL: 1.8991 (PPL 6.68) | LR: 9.65e-05 | Grad: 0.53 | 87.6k tok/s | VRAM: 21.3G
166
+ Step 16600/24000 | SmoothLoss: 2.7911 | EstNLL: 1.8687 (PPL 6.48) | LR: 9.33e-05 | Grad: 0.49 | 88.9k tok/s | VRAM: 21.3G
167
+ Step 16800/24000 | SmoothLoss: 2.8040 | EstNLL: 1.8830 (PPL 6.57) | LR: 9.02e-05 | Grad: 0.30 | 89.0k tok/s | VRAM: 21.3G
168
+ Step 17000/24000 | SmoothLoss: 2.8037 | EstNLL: 1.8827 (PPL 6.57) | LR: 8.72e-05 | Grad: 0.38 | 89.3k tok/s | VRAM: 21.3G
169
+ Step 17200/24000 | SmoothLoss: 2.8083 | EstNLL: 1.8878 (PPL 6.60) | LR: 8.42e-05 | Grad: 0.37 | 88.3k tok/s | VRAM: 21.3G
170
+ Step 17400/24000 | SmoothLoss: 2.7887 | EstNLL: 1.8660 (PPL 6.46) | LR: 8.13e-05 | Grad: 0.33 | 88.6k tok/s | VRAM: 21.3G
171
+ Step 17600/24000 | SmoothLoss: 2.8052 | EstNLL: 1.8844 (PPL 6.58) | LR: 7.84e-05 | Grad: 0.31 | 89.5k tok/s | VRAM: 21.3G
172
+ Step 17800/24000 | SmoothLoss: 2.7896 | EstNLL: 1.8671 (PPL 6.47) | LR: 7.56e-05 | Grad: 0.30 | 89.0k tok/s | VRAM: 21.3G
173
+ Step 18000/24000 | SmoothLoss: 2.7952 | EstNLL: 1.8733 (PPL 6.51) | LR: 7.29e-05 | Grad: 0.28 | 88.8k tok/s | VRAM: 21.3G
174
+
175
+ -------------------------------------------------------------------------------------
176
+ 🔎 210M-PRO EVAL @ Step 18,000 | Val NLL: 1.7784 | Val PPL: 5.92
177
+ [En->My Formal]: ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးတည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
178
+ [En->My Spoken]: အမှန်ဆို ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ခဲ့ပေမယ့် စိတ်မပူပါနဲ့ အတူတူတွေးကြည့်မယ်။
179
+ [My->En Formal]: The central bank has raised interest rates to control inflation and stabilize the economy.
180
+ [My->En Spoken]: Love, I loved you and me so much that Buddha knows, Saka knows, Nats know. Those who envy are being seduced; because of this, you have gone mad.
181
+ 🌟 New Best 210M-Pro Checkpoint Saved! (Val NLL: 1.7784)
182
+ -------------------------------------------------------------------------------------
183
+
184
+ Step 18200/24000 | SmoothLoss: 2.7973 | EstNLL: 1.8756 (PPL 6.52) | LR: 7.02e-05 | Grad: 0.26 | 89.1k tok/s | VRAM: 21.3G
185
+ Step 18400/24000 | SmoothLoss: 2.7875 | EstNLL: 1.8648 (PPL 6.45) | LR: 6.76e-05 | Grad: 0.27 | 88.6k tok/s | VRAM: 21.3G
186
+ Step 18600/24000 | SmoothLoss: 2.7865 | EstNLL: 1.8637 (PPL 6.45) | LR: 6.51e-05 | Grad: 0.29 | 87.8k tok/s | VRAM: 21.3G
187
+ Step 18800/24000 | SmoothLoss: 2.7915 | EstNLL: 1.8692 (PPL 6.48) | LR: 6.27e-05 | Grad: 0.39 | 89.1k tok/s | VRAM: 21.3G
188
+ Step 19000/24000 | SmoothLoss: 2.7777 | EstNLL: 1.8538 (PPL 6.38) | LR: 6.03e-05 | Grad: 0.62 | 88.1k tok/s | VRAM: 21.3G
189
+ Step 19200/24000 | SmoothLoss: 2.7879 | EstNLL: 1.8652 (PPL 6.46) | LR: 5.80e-05 | Grad: 0.33 | 89.1k tok/s | VRAM: 21.3G
190
+ Step 19400/24000 | SmoothLoss: 2.7872 | EstNLL: 1.8644 (PPL 6.45) | LR: 5.58e-05 | Grad: 0.35 | 88.3k tok/s | VRAM: 21.3G
191
+ Step 19600/24000 | SmoothLoss: 2.7800 | EstNLL: 1.8564 (PPL 6.40) | LR: 5.37e-05 | Grad: 0.34 | 88.5k tok/s | VRAM: 21.3G
192
+ Step 19800/24000 | SmoothLoss: 2.7748 | EstNLL: 1.8506 (PPL 6.36) | LR: 5.16e-05 | Grad: 0.23 | 88.8k tok/s | VRAM: 21.3G
193
+ Step 20000/24000 | SmoothLoss: 2.7757 | EstNLL: 1.8516 (PPL 6.37) | LR: 4.97e-05 | Grad: 0.22 | 88.2k tok/s | VRAM: 21.3G
194
+
195
+ -------------------------------------------------------------------------------------
196
+ 🔎 210M-PRO EVAL @ Step 20,000 | Val NLL: 1.7648 | Val PPL: 5.84
197
+ [En->My Formal]: ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးကို တည်ငြိမ်စေရန် ဗဟိုဘဏ်သည် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
198
+ [En->My Spoken]: တကယ်တန်းတော့ ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ထားဘဲ စိတ်မပူပါနဲ့ အတူတူတွေးကြည့်ကြမယ်နော်။
199
+ [My->En Formal]: The central bank has raised interest rates to control inflation and stabilize the economy.
200
+ [My->En Spoken]: Love, I knew that you and I loved each other very much, Buddha knows too, sir knows too, those who envy me are being jealous. Because of this, you have gone mad.
201
+ 🌟 New Best 210M-Pro Checkpoint Saved! (Val NLL: 1.7648)
202
+ -------------------------------------------------------------------------------------
203
+
204
+ Step 20200/24000 | SmoothLoss: 2.7861 | EstNLL: 1.8632 (PPL 6.44) | LR: 4.78e-05 | Grad: 0.25 | 89.6k tok/s | VRAM: 21.3G
205
+ Step 20400/24000 | SmoothLoss: 2.7663 | EstNLL: 1.8412 (PPL 6.30) | LR: 4.60e-05 | Grad: 0.27 | 88.4k tok/s | VRAM: 21.3G
206
+ Step 20600/24000 | SmoothLoss: 2.7747 | EstNLL: 1.8505 (PPL 6.36) | LR: 4.43e-05 | Grad: 0.29 | 89.0k tok/s | VRAM: 21.3G
207
+ Step 20800/24000 | SmoothLoss: 2.7740 | EstNLL: 1.8497 (PPL 6.36) | LR: 4.27e-05 | Grad: 0.28 | 88.5k tok/s | VRAM: 21.3G
208
+ Step 21000/24000 | SmoothLoss: 2.7830 | EstNLL: 1.8597 (PPL 6.42) | LR: 4.12e-05 | Grad: 0.37 | 88.2k tok/s | VRAM: 21.3G
209
+ Step 21200/24000 | SmoothLoss: 2.7732 | EstNLL: 1.8489 (PPL 6.35) | LR: 3.98e-05 | Grad: 0.39 | 89.3k tok/s | VRAM: 21.3G
210
+ Step 21400/24000 | SmoothLoss: 2.7722 | EstNLL: 1.8477 (PPL 6.35) | LR: 3.84e-05 | Grad: 0.36 | 89.0k tok/s | VRAM: 21.3G
211
+ Step 21600/24000 | SmoothLoss: 2.7641 | EstNLL: 1.8388 (PPL 6.29) | LR: 3.72e-05 | Grad: 0.40 | 88.8k tok/s | VRAM: 21.3G
212
+ Step 21800/24000 | SmoothLoss: 2.7755 | EstNLL: 1.8514 (PPL 6.37) | LR: 3.61e-05 | Grad: 0.50 | 88.0k tok/s | VRAM: 21.3G
213
+ Step 22000/24000 | SmoothLoss: 2.7742 | EstNLL: 1.8499 (PPL 6.36) | LR: 3.50e-05 | Grad: 0.43 | 88.7k tok/s | VRAM: 21.3G
214
+
215
+ -------------------------------------------------------------------------------------
216
+ 🔎 210M-PRO EVAL @ Step 22,000 | Val NLL: 1.7561 | Val PPL: 5.79
217
+ [En->My Formal]: ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးကို တည်ငြိမ်စေရန် ဗဟိုဘဏ်သည် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
218
+ [En->My Spoken]: အမှန်ဆို ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ခဲ့ပေမယ့် စိတ်မပူပါနဲ့ အတူတူတွေးကြည့်ကြမယ်
219
+ [My->En Formal]: The Central Bank has raised interest rates to control inflation and stabilize the economy.
220
+ [My->En Spoken]: Love, I knew that you and I loved each other very much, Buddha knows too, Sankha knows too, Nat knows too, those who envy are being jealous. Because of this, you have gone crazy.
221
+ 🌟 New Best 210M-Pro Checkpoint Saved! (Val NLL: 1.7561)
222
+ -------------------------------------------------------------------------------------
223
+
224
+ Step 22200/24000 | SmoothLoss: 2.7567 | EstNLL: 1.8306 (PPL 6.24) | LR: 3.41e-05 | Grad: 0.43 | 88.8k tok/s | VRAM: 21.3G
225
+ Step 22400/24000 | SmoothLoss: 2.7752 | EstNLL: 1.8510 (PPL 6.37) | LR: 3.32e-05 | Grad: 0.32 | 89.3k tok/s | VRAM: 21.3G
226
+ Step 22600/24000 | SmoothLoss: 2.7837 | EstNLL: 1.8606 (PPL 6.43) | LR: 3.25e-05 | Grad: 0.27 | 88.8k tok/s | VRAM: 21.3G
227
+ Step 22800/24000 | SmoothLoss: 2.7672 | EstNLL: 1.8422 (PPL 6.31) | LR: 3.18e-05 | Grad: 0.24 | 88.3k tok/s | VRAM: 21.3G
228
+ Step 23000/24000 | SmoothLoss: 2.7640 | EstNLL: 1.8386 (PPL 6.29) | LR: 3.13e-05 | Grad: 0.29 | 88.9k tok/s | VRAM: 21.3G
229
+ Step 23200/24000 | SmoothLoss: 2.7571 | EstNLL: 1.8309 (PPL 6.24) | LR: 3.08e-05 | Grad: 0.33 | 88.8k tok/s | VRAM: 21.3G
230
+ Step 23400/24000 | SmoothLoss: 2.7612 | EstNLL: 1.8356 (PPL 6.27) | LR: 3.05e-05 | Grad: 0.37 | 88.7k tok/s | VRAM: 21.3G
231
+ Step 23600/24000 | SmoothLoss: 2.7610 | EstNLL: 1.8352 (PPL 6.27) | LR: 3.02e-05 | Grad: 0.44 | 89.2k tok/s | VRAM: 21.3G
232
+ Step 23800/24000 | SmoothLoss: 2.7653 | EstNLL: 1.8401 (PPL 6.30) | LR: 3.01e-05 | Grad: 0.43 | 88.9k tok/s | VRAM: 21.3G
233
+ Step 24000/24000 | SmoothLoss: 2.7609 | EstNLL: 1.8352 (PPL 6.27) | LR: 3.00e-05 | Grad: 0.39 | 88.7k tok/s | VRAM: 21.3G
234
+
235
+ -------------------------------------------------------------------------------------
236
+ 🔎 210M-PRO EVAL @ Step 24,000 | Val NLL: 1.7492 | Val PPL: 5.75
237
+ [En->My Formal]: ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးကို တည်ငြိမ်စေရန် ဗဟိုဘဏ်သည် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
238
+ [En->My Spoken]: အမှန်ဆို ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ထားဘဲ စိတ်မပူပါနဲ့ အတူတူတွေးကြည့်ကြမယ်
239
+ [My->En Formal]: The Central Bank has raised interest rates to control inflation and stabilize the economy.
240
+ [My->En Spoken]: Love, I knew that you and I loved each other deeply, Buddha knows too, Sankha knows too, Nat knows too, those who envy me are being harmed. Because of this, you have gone crazy.
241
+ 🌟 New Best 210M-Pro Checkpoint Saved! (Val NLL: 1.7492)
242
+ -------------------------------------------------------------------------------------
243
+
244
+
245
+ >>> Starting 210M-Pro Stage 2 Cooldown (1,000 steps | label_smoothing=0.0 | LR 3e-5 -> 4e-6)...
246
+ 🔎 210M-Pro Stage 2 Step 250/1000 | Val NLL: 1.6798 (PPL 5.36)
247
+ 🔎 210M-Pro Stage 2 Step 500/1000 | Val NLL: 1.6765 (PPL 5.35)
248
+ 🔎 210M-Pro Stage 2 Step 750/1000 | Val NLL: 1.6740 (PPL 5.33)
249
+ 🔎 210M-Pro Stage 2 Step 1000/1000 | Val NLL: 1.6724 (PPL 5.33)
250
+
251
+ ==========================================================================================
252
+ 🏆 MYANMARTRANSLATOR-210M-PRO FINAL BENCHMARK | Val NLL: 1.6724 (PPL 5.33)
253
+ ==========================================================================================
254
+ [En->My Formal (Beam=4)]: ဗဟိုဘဏ်သည် ငွေကြေးဖောင်းပွမှုကို ထိန်းချုပ်ရန်နှင့် စီးပွားရေးတည်ငြိမ်စေရန် အတိုးနှုန်းများကို မြှင့်တင်ခဲ့သည်။
255
+ [En->My Spoken (Beam=4)]: ရိုးရိုးသားသားပြောရရင် ဒီလိုဖြစ်လာမယ်လို့ မမျှော်လင့်ထားခဲ့ပေမယ့် စိတ်မပူပါနဲ့၊ ကျွန်တော်တို့ အတူတူ အဖြေရှာကြတာပေါ့။
256
+ [My->En Formal (Beam=4)]: The central bank raised interest rates to control inflation and stabilize the economy.
257
+ [My->En Spoken (Beam=4)]: My love, heaven and earth know how deeply you and I loved each other, yet because you believed the lies fed by jealous people, you've lost your mind.
258
+ [En->My News (Beam=4)]: မိုးသည်းထန်စွာ ရွာသွန်းမှုကြောင့် မြို့နယ်အချို့တွင် ရေကြီးရေလျှံမှုများ ဆိုးရွားစွာ ဖြစ်ပေါ်ခဲ့ပြီး ထောင်နှင့်ချီသော ဒေသခံများသည် ယာယီခိုလှုံရာ စခန်းများသို့ ရွှေ့ပြောင်းခဲ့ရသည်။
259
+ [En->My Health (Beam=4)]: ရာသီတုပ်ကွေးရောဂါကနေ မြန်မြန်နာလန်ထဖို့ သန့်ရှင်းတဲ့ရေများများသောက်ဖို့နဲ့ လုံလောက်တဲ့ အနားယူချိန်ရဖို့ ဆရာဝန်တွေက အကြံပြုထားပါတယ်။
260
+ [En->My Spoken (Beam=4)]: ဒီမနက် တစ်နာရီကျော်လောက် ကားပိတ်မိနေလို့ ကျွန်တော်မပါဘဲ အစည်းအဝေးကို စထားလိုက်ပါနော်။
261
+ [My->En News (Beam=4)]: In order to promote international trade, inspections of goods are being carried out more efficiently at border trade camps.
262
+ [My->En Tech (Beam=4)]: Artificial intelligence technology can help make processes more effective in education, healthcare, and economic sectors.
263
+ [My->En Spoken (Beam=4)]: I have to go tomorrow morning, so tonight I will sleep early. I'll call you in the morning.
264
+ ==========================================================================================