freococo commited on
Commit
c429db6
·
verified ·
1 Parent(s): a0c6fec

Update checkpoint-stage1-step-58000/training_codes.txt

Browse files
checkpoint-stage1-step-58000/training_codes.txt CHANGED
@@ -1,3 +1,727 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import os
2
  os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
3
 
 
1
+ # === PHASE 1, CELL 1: Stratified Corpus Sampling Across All 33 Subsets ===
2
+ import os
3
+ import random
4
+ import time
5
+ from collections import defaultdict
6
+ from concurrent.futures import ThreadPoolExecutor, as_completed
7
+
8
+ import pyarrow.parquet as pq
9
+ from google.colab import userdata
10
+ from huggingface_hub import HfApi, HfFileSystem
11
+ from huggingface_hub.hf_api import RepoFile
12
+
13
+ REPO_ID = "freococo/myanmar-english-corpus"
14
+ HF_TOKEN = userdata.get("HF_TOKEN")
15
+ assert HF_TOKEN, "HF_TOKEN not found in Colab Secrets!"
16
+
17
+ TOK_TEXT_FILE = "/content/mytranslation_tok_train.txt"
18
+ api = HfApi(token=HF_TOKEN)
19
+ fs = HfFileSystem(token=HF_TOKEN)
20
+
21
+ print(f"🔍 Listing all Parquet files in {REPO_ID}...")
22
+ tree = list(api.list_repo_tree(repo_id=REPO_ID, repo_type="dataset", path_in_repo="data", recursive=True))
23
+
24
+ subset_files = defaultdict(list)
25
+ for item in tree:
26
+ if isinstance(item, RepoFile) and item.path.endswith(".parquet"):
27
+ parts = item.path.split("/")
28
+ if len(parts) == 4:
29
+ group, subset = parts[1], parts[2]
30
+ subset_files[(group, subset)].append(item.path)
31
+
32
+ print(f"📦 Found {len(subset_files)} subsets across {sum(len(v) for v in subset_files.values())} Parquet files.")
33
+
34
+ # Sampling budget (rows per subset) — 100% of small/specialized sets, balanced slices of huge sets
35
+ TARGET_ROWS_BY_GROUP = {
36
+ "non_myanmartranslator": 160_000, # Takes 100% of all <160k sets (musannaf, quran, proverbs, idioms, pali, taco, etc.)
37
+ "MyanmarTranslator_works": 120_000, # Balanced slice per MT domain (literature, spoken, written, news, wiki, etc.)
38
+ }
39
+ # Give finetranslations & myanmar-xnli slightly more weight as core large parallel sets
40
+ CUSTOM_QUOTA = {
41
+ "finetranslations": 300_000,
42
+ "myanmar-xnli": 180_000,
43
+ "myanmar-literature-corpus": 150_000,
44
+ "myanmar-wikipedia-corpus": 150_000,
45
+ "myanmar_spoken_corpus": 150_000,
46
+ "myanmar-written-corpus": 150_000,
47
+ }
48
+
49
+ rng = random.Random(2026)
50
+
51
+ def sample_subset(group, subset, file_paths):
52
+ quota = CUSTOM_QUOTA.get(subset, TARGET_ROWS_BY_GROUP[group])
53
+ file_paths = sorted(file_paths)
54
+ # Pick up to 6 evenly spaced files if folder has many shards
55
+ if len(file_paths) > 6:
56
+ step = len(file_paths) / 6.0
57
+ picked_files = [file_paths[int(i * step)] for i in range(6)]
58
+ else:
59
+ picked_files = file_paths
60
+
61
+ per_file_quota = max(1, (quota + len(picked_files) - 1) // len(picked_files))
62
+ lines = []
63
+
64
+ for fp_path in picked_files:
65
+ with fs.open(f"datasets/{REPO_ID}/{fp_path}", "rb") as f:
66
+ pf = pq.ParquetFile(f)
67
+ n_rg = pf.num_row_groups
68
+ rg_indices = list(range(n_rg))
69
+ if n_rg > 4:
70
+ # Sample evenly across row groups
71
+ step_rg = n_rg / 4.0
72
+ rg_indices = sorted({rg_indices[int(i * step_rg)] for i in range(4)})
73
+
74
+ collected_my, collected_en = [], []
75
+ per_rg_cap = max(500, (per_file_quota + len(rg_indices) - 1) // len(rg_indices))
76
+
77
+ for rg_i in rg_indices:
78
+ tbl = pf.read_row_group(rg_i, columns=["myanmar_text_original", "english_text_translation"])
79
+ n_r = tbl.num_rows
80
+ if n_r > per_rg_cap:
81
+ stride = max(1, n_r // per_rg_cap)
82
+ idx_slice = list(range(0, n_r, stride))[:per_rg_cap]
83
+ tbl = tbl.take(idx_slice)
84
+ d = tbl.to_pydict()
85
+ collected_my.extend(d["myanmar_text_original"])
86
+ collected_en.extend(d["english_text_translation"])
87
+
88
+ for m, e in zip(collected_my, collected_en):
89
+ if isinstance(m, str):
90
+ for para in m.splitlines():
91
+ p = para.strip()
92
+ if p:
93
+ lines.append(p)
94
+ if isinstance(e, str):
95
+ for para in e.splitlines():
96
+ p = para.strip()
97
+ if p:
98
+ lines.append(p)
99
+ if len(lines) >= quota * 2:
100
+ break
101
+ if len(lines) >= quota * 2:
102
+ break
103
+
104
+ return group, subset, lines
105
+
106
+ t0 = time.time()
107
+ all_lines = []
108
+ print("\n⏳ Streaming stratified samples across all 33 subsets...")
109
+
110
+ with ThreadPoolExecutor(max_workers=10) as ex:
111
+ futures = [
112
+ ex.submit(sample_subset, grp, sub, paths)
113
+ for (grp, sub), paths in sorted(subset_files.items())
114
+ ]
115
+ for fut in as_completed(futures):
116
+ grp, sub, lines = fut.result()
117
+ all_lines.extend(lines)
118
+ print(f" ✅ {grp}/{sub:<44} -> {len(lines):>8,} lines")
119
+
120
+ rng.shuffle(all_lines)
121
+ with open(TOK_TEXT_FILE, "w", encoding="utf-8") as f:
122
+ for ln in all_lines:
123
+ f.write(ln + "\n")
124
+
125
+ mb = os.path.getsize(TOK_TEXT_FILE) / 1e6
126
+ print(f"\n🎉 Saved stratified tokenizer corpus: {TOK_TEXT_FILE}")
127
+ print(f" • Total lines: {len(all_lines):,} | Size: {mb:.1f} MB | Elapsed: {time.time() - t0:.1f}s")
128
+
129
+ # === PHASE 1, CELL 2 (FIXED PRE-FLIGHT): Train 49k & 65k Unigram+ByteFallback & benchmark ===
130
+ import os
131
+ import time
132
+ import pyarrow.parquet as pq
133
+ import sentencepiece as spm
134
+ from google.colab import userdata
135
+ from huggingface_hub import HfApi, HfFileSystem, hf_hub_download
136
+ from huggingface_hub.hf_api import RepoFile
137
+
138
+ HF_TOKEN = userdata.get("HF_TOKEN")
139
+ TOK_TEXT_FILE = "/content/mytranslation_tok_train.txt"
140
+ TOK_OUT_DIR = "/content/mytranslation_tokenizers"
141
+ os.makedirs(TOK_OUT_DIR, exist_ok=True)
142
+
143
+ CONTROL_SYMBOLS = ["<2my>", "<2en>", "<formal>", "<spoken>", "<bt>", "<denoise>", "<mask>"]
144
+
145
+ # 0. Pre-flight check on first 5,000 lines to verify '\n' and control tokens roundtrip 100%
146
+ mini_txt = "/content/_mini_preflight.txt"
147
+ with open(TOK_TEXT_FILE, "r", encoding="utf-8") as fin, open(mini_txt, "w", encoding="utf-8") as fout:
148
+ for i, line in enumerate(fin):
149
+ fout.write(line)
150
+ if i >= 5000:
151
+ break
152
+ fout.write("Hello world.\nမင်္ဂလာပါ ခင်ဗျာ။\n12345\n")
153
+
154
+ spm.SentencePieceTrainer.train(
155
+ input=mini_txt,
156
+ model_prefix="/content/_mini_tok",
157
+ vocab_size=1000,
158
+ model_type="unigram",
159
+ character_coverage=0.9999,
160
+ byte_fallback=True,
161
+ split_digits=True,
162
+ normalization_rule_name="identity",
163
+ remove_extra_whitespaces=False,
164
+ allow_whitespace_only_pieces=True,
165
+ unk_id=0, bos_id=1, eos_id=2, pad_id=3,
166
+ user_defined_symbols=CONTROL_SYMBOLS,
167
+ )
168
+ sp_pre = spm.SentencePieceProcessor()
169
+ sp_pre.load("/content/_mini_tok.model")
170
+ for test_s in ["Hello\n\nWorld", "မင်္ဂလာပါ\n\nနေကောင်းလား", "<2my> <formal> Hello\n123"]:
171
+ dec_s = sp_pre.decode_ids(sp_pre.encode_as_ids(test_s))
172
+ assert dec_s == test_s, f"Pre-flight newline roundtrip failed: {repr(test_s)} -> {repr(dec_s)}"
173
+ print("✅ Pre-flight check PASSED: '\\n' and control tokens roundtrip 100.0% identically!")
174
+
175
+ # 1. Train Candidate A (49,152) and Candidate B (65,536)
176
+ trained_models = {}
177
+ for label, v_size in [("MyTranslation-49k", 49152), ("MyTranslation-65k", 65536)]:
178
+ prefix = os.path.join(TOK_OUT_DIR, f"tok_{v_size}")
179
+ print(f"\n🏋️ Training {label} (vocab_size={v_size:,}) on {TOK_TEXT_FILE} ...")
180
+ t0 = time.time()
181
+ spm.SentencePieceTrainer.train(
182
+ input=TOK_TEXT_FILE,
183
+ model_prefix=prefix,
184
+ vocab_size=v_size,
185
+ model_type="unigram",
186
+ character_coverage=0.9999,
187
+ byte_fallback=True,
188
+ split_digits=True,
189
+ normalization_rule_name="identity",
190
+ remove_extra_whitespaces=False,
191
+ allow_whitespace_only_pieces=True,
192
+ max_sentencepiece_length=16,
193
+ input_sentence_size=2_500_000,
194
+ shuffle_input_sentence=True,
195
+ num_threads=max(4, os.cpu_count() or 4),
196
+ unk_id=0, bos_id=1, eos_id=2, pad_id=3,
197
+ user_defined_symbols=CONTROL_SYMBOLS,
198
+ )
199
+ sp_m = spm.SentencePieceProcessor()
200
+ sp_m.load(prefix + ".model")
201
+ trained_models[label] = (sp_m, prefix + ".model")
202
+ print(f" ✅ {label} trained in {time.time() - t0:.1f}s | actual vocab={sp_m.get_piece_size():,}")
203
+
204
+ # Load old 210M-Pro tokenizer for baseline comparison
205
+ old_tok_path = hf_hub_download(repo_id="freococo/MyanmarTranslator-210M-Pro", filename="tokenizer.model", token=HF_TOKEN)
206
+ sp_old = spm.SentencePieceProcessor()
207
+ sp_old.load(old_tok_path)
208
+
209
+ all_toks = {
210
+ "Old-210M-65k": sp_old,
211
+ "MyTrans-49k": trained_models["MyTranslation-49k"][0],
212
+ "MyTrans-65k": trained_models["MyTranslation-65k"][0],
213
+ }
214
+
215
+ # 2. Side-by-Side Benchmark across the exact 8 subsets
216
+ CORPUS_REPO = "freococo/myanmar-english-corpus"
217
+ api = HfApi(token=HF_TOKEN)
218
+ fs = HfFileSystem(token=HF_TOKEN)
219
+
220
+ TARGETS = [
221
+ "data/MyanmarTranslator_works/myanmar-literature-corpus",
222
+ "data/non_myanmartranslator/finetranslations",
223
+ "data/non_myanmartranslator/alpaca_myanmar_burmese_taco",
224
+ "data/non_myanmartranslator/musannaf_ibn_abi_shaybah",
225
+ "data/non_myanmartranslator/arabic_names_with_myanmar_and_english_transliteration",
226
+ "data/non_myanmartranslator/myanmar-english-pali-dictionary",
227
+ "data/non_myanmartranslator/quran_written_spoken_myanmar_english",
228
+ "data/non_myanmartranslator/myanmar_proverbs_lexicon",
229
+ ]
230
+
231
+ print("\n" + "=" * 116)
232
+ print(f"{'SUBSET':<38} {'SIDE':<4} | {'OLD CH/T':>8} {'OLD BYT%':>8} {'OLD RT%':>7} | {'49K CH/T':>8} {'49K BYT%':>8} {'49K RT%':>7} | {'65K CH/T':>8} {'65K BYT%':>8} {'65K RT%':>7}")
233
+ print("=" * 116)
234
+
235
+ for folder_path in TARGETS:
236
+ tree = list(api.list_repo_tree(repo_id=CORPUS_REPO, repo_type="dataset", path_in_repo=folder_path))
237
+ first_pq = next(item.path for item in tree if isinstance(item, RepoFile) and item.path.endswith(".parquet"))
238
+ with fs.open(f"datasets/{CORPUS_REPO}/{first_pq}", "rb") as f:
239
+ tbl = pq.ParquetFile(f).read_row_group(0).slice(0, 500).to_pydict()
240
+
241
+ subset = folder_path.split("/")[-1][:37]
242
+ for side, col in [("MY", "myanmar_text_original"), ("EN", "english_text_translation")]:
243
+ texts = [t for t in tbl[col] if isinstance(t, str) and t.strip()]
244
+ row_metrics = {}
245
+ for mname, sp_inst in all_toks.items():
246
+ tot_ch, tot_tok, tot_byte, rt_ok = 0, 0, 0, 0
247
+ for t in texts:
248
+ ids = sp_inst.encode_as_ids(t)
249
+ dec = sp_inst.decode_ids(ids)
250
+ tot_ch += len(t)
251
+ tot_tok += len(ids)
252
+ tot_byte += sum(1 for i in ids if sp_inst.is_byte(i))
253
+ if dec == t:
254
+ rt_ok += 1
255
+ row_metrics[mname] = (
256
+ tot_ch / max(1, tot_tok),
257
+ 100.0 * tot_byte / max(1, tot_tok),
258
+ 100.0 * rt_ok / max(1, len(texts)),
259
+ )
260
+
261
+ o = row_metrics["Old-210M-65k"]
262
+ m49 = row_metrics["MyTrans-49k"]
263
+ m65 = row_metrics["MyTrans-65k"]
264
+ print(
265
+ f"{subset:<38} {side:<4} | "
266
+ f"{o[0]:>8.2f} {o[1]:>7.2f}% {o[2]:>6.1f}% | "
267
+ f"{m49[0]:>8.2f} {m49[1]:>7.2f}% {m49[2]:>6.1f}% | "
268
+ f"{m65[0]:>8.2f} {m65[1]:>7.2f}% {m65[2]:>6.1f}%"
269
+ )
270
+
271
+ print("\n📌 Control Token IDs check (MyTrans-65k):")
272
+ for sym in ["<unk>", "<s>", "</s>", "<pad>"] + CONTROL_SYMBOLS:
273
+ print(f" {sym:<10} -> {trained_models['MyTranslation-65k'][0].piece_to_id(sym)}")
274
+
275
+ # === PHASE 1 BACKUP: Convert tok_train.txt -> Parquet & upload package to freococo/mytrans_backup ===
276
+ import os
277
+ import shutil
278
+ import time
279
+
280
+ import pyarrow as pa
281
+ import pyarrow.parquet as pq
282
+ from google.colab import userdata
283
+ from huggingface_hub import HfApi, create_repo
284
+
285
+ HF_TOKEN = userdata.get("HF_TOKEN")
286
+ assert HF_TOKEN, "HF_TOKEN not found in Colab Secrets!"
287
+
288
+ BACKUP_REPO = "freococo/mytrans_backup"
289
+ TXT_PATH = "/content/mytranslation_tok_train.txt"
290
+ TOK_DIR = "/content/mytranslation_tokenizers"
291
+ PKG_DIR = "/content/mytrans_backup_pkg"
292
+
293
+ assert os.path.exists(TXT_PATH), f"Missing {TXT_PATH}"
294
+ os.makedirs(os.path.join(PKG_DIR, "tokenizers", "49k"), exist_ok=True)
295
+ os.makedirs(os.path.join(PKG_DIR, "tokenizers", "65k"), exist_ok=True)
296
+
297
+ # 1. Stream-convert /content/mytranslation_tok_train.txt -> compressed Parquet (low RAM)
298
+ pq_out = os.path.join(PKG_DIR, "tok_train_corpus.parquet")
299
+ schema = pa.schema([("text", pa.string())])
300
+ writer = pq.ParquetWriter(pq_out, schema, compression="zstd")
301
+
302
+ print("📦 Converting mytranslation_tok_train.txt -> tok_train_corpus.parquet (zstd) ...")
303
+ t0 = time.time()
304
+ buf = []
305
+ total_lines = 0
306
+ CHUNK = 500_000
307
+
308
+ with open(TXT_PATH, "r", encoding="utf-8") as f:
309
+ for line in f:
310
+ s = line.rstrip("\r\n")
311
+ if not s:
312
+ continue
313
+ buf.append(s)
314
+ total_lines += 1
315
+ if len(buf) >= CHUNK:
316
+ writer.write_table(pa.Table.from_pydict({"text": buf}, schema=schema))
317
+ buf.clear()
318
+ print(f" Written {total_lines:,} lines...")
319
+
320
+ if buf:
321
+ writer.write_table(pa.Table.from_pydict({"text": buf}, schema=schema))
322
+ buf.clear()
323
+
324
+ writer.close()
325
+ pq_mb = os.path.getsize(pq_out) / 1e6
326
+ print(f"✅ Parquet built: {total_lines:,} lines | {pq_mb:.1f} MB in {time.time() - t0:.1f}s")
327
+
328
+ # 2. Copy both 49k and 65k tokenizers into package folder
329
+ for sz, tag in [(49152, "49k"), (65536, "65k")]:
330
+ for ext in (".model", ".vocab"):
331
+ src = os.path.join(TOK_DIR, f"tok_{sz}{ext}")
332
+ dst = os.path.join(PKG_DIR, "tokenizers", tag, f"tokenizer{ext}")
333
+ assert os.path.exists(src), f"Missing {src}"
334
+ shutil.copyfile(src, dst)
335
+
336
+ readme_txt = f"""---
337
+ pretty_name: MyTranslation Project Backup
338
+ language:
339
+ - my
340
+ - en
341
+ ---
342
+
343
+ # `freococo/mytrans_backup`
344
+
345
+ Private project backup for **MyTranslation** (tokenizer training corpus, trained SentencePiece models, and tokenized memmaps).
346
+
347
+ ## Contents
348
+ - `tok_train_corpus.parquet` — {total_lines:,} stratified lines across all 33 subsets of `freococo/myanmar-english-corpus` ({pq_mb:.1f} MB, zstd).
349
+ - `tokenizers/49k/tokenizer.model` & `.vocab` — `49,152` Unigram + byte-fallback (`identity` norm, newline-preserving).
350
+ - `tokenizers/65k/tokenizer.model` & `.vocab` — `65,536` Unigram + byte-fallback (`identity` norm, newline-preserving, **winner on compression**).
351
+
352
+ ## Special & Control Token IDs
353
+ - `0`: `<unk>` | `1`: `<s>` | `2`: `</s>` | `3`: `<pad>`
354
+ - `4`: `<2my>` | `5`: `<2en>` | `6`: `<formal>` | `7`: `<spoken>` | `8`: `<bt>` | `9`: `<denoise>` | `10`: `<mask>`
355
+ """
356
+ with open(os.path.join(PKG_DIR, "README.md"), "w", encoding="utf-8") as f:
357
+ f.write(readme_txt)
358
+
359
+ # 3. Create private repo & upload entire package
360
+ api = HfApi(token=HF_TOKEN)
361
+ create_repo(BACKUP_REPO, repo_type="dataset", private=True, exist_ok=True)
362
+ print(f"\n☁️ Uploading package to private repo https://huggingface.co/datasets/{BACKUP_REPO} ...")
363
+ t1 = time.time()
364
+ api.upload_folder(
365
+ folder_path=PKG_DIR,
366
+ repo_id=BACKUP_REPO,
367
+ repo_type="dataset",
368
+ commit_message="Backup stratified tokenizer corpus Parquet + 49k & 65k MyTranslation tokenizers",
369
+ )
370
+ print(f"🎉 Backup complete in {time.time() - t1:.1f}s! https://huggingface.co/datasets/{BACKUP_REPO}")
371
+
372
+ # === PHASE 1C, CELL 1: Tokenize all 48.35M+ rows with MyTrans-65k into canonical uint16 memmaps ===
373
+ import gc
374
+ import json
375
+ import os
376
+ import shutil
377
+ import time
378
+ from collections import defaultdict
379
+ from concurrent.futures import ThreadPoolExecutor
380
+
381
+ import numpy as np
382
+ import pyarrow.parquet as pq
383
+ import sentencepiece as spm
384
+ from google.colab import userdata
385
+ from huggingface_hub import HfApi, hf_hub_download
386
+ from huggingface_hub.hf_api import RepoFile
387
+
388
+ HF_TOKEN = userdata.get("HF_TOKEN")
389
+ assert HF_TOKEN, "HF_TOKEN not found in Colab Secrets!"
390
+
391
+ CORPUS_REPO = "freococo/myanmar-english-corpus"
392
+ BACKUP_REPO = "freococo/mytrans_backup"
393
+
394
+ # Clean up Phase 1 raw txt/pkg to maximize disk space
395
+ for old_p in ["/content/mytranslation_tok_train.txt", "/content/mytrans_backup_pkg", "/content/_mini_preflight.txt"]:
396
+ if os.path.isdir(old_p):
397
+ shutil.rmtree(old_p, ignore_errors=True)
398
+ elif os.path.exists(old_p):
399
+ os.remove(old_p)
400
+
401
+ MEMMAP_DIR = "/content/mytrans_memmap_65k"
402
+ DL_CACHE_DIR = "/content/_tmp_pq_cache"
403
+ os.makedirs(MEMMAP_DIR, exist_ok=True)
404
+ os.makedirs(DL_CACHE_DIR, exist_ok=True)
405
+
406
+ # Load MyTrans-65k (local if present, else pull from freococo/mytrans_backup)
407
+ LOCAL_TOK = "/content/mytranslation_tokenizers/tok_65536.model"
408
+ if os.path.exists(LOCAL_TOK):
409
+ TOK_PATH = LOCAL_TOK
410
+ else:
411
+ TOK_PATH = hf_hub_download(repo_id=BACKUP_REPO, repo_type="dataset", filename="tokenizers/65k/tokenizer.model", token=HF_TOKEN)
412
+
413
+ shutil.copyfile(TOK_PATH, os.path.join(MEMMAP_DIR, "tokenizer.model"))
414
+ sp = spm.SentencePieceProcessor()
415
+ sp.load(TOK_PATH)
416
+ assert sp.get_piece_size() == 65536, f"Expected 65536 vocab, got {sp.get_piece_size()}"
417
+
418
+ N_THREADS = max(4, os.cpu_count() or 8)
419
+ try:
420
+ _ = sp.encode(["test"], out_type=int, num_threads=N_THREADS)
421
+ USE_SP_THREADS = True
422
+ print(f"⚡ SentencePiece native C++ batch encode enabled (num_threads={N_THREADS})")
423
+ except TypeError:
424
+ USE_SP_THREADS = False
425
+ print(f"⚡ SentencePiece fallback encode (using {N_THREADS} threads via pool)")
426
+
427
+ def encode_batch(texts: list[str]) -> list[list[int]]:
428
+ if USE_SP_THREADS:
429
+ return sp.encode(texts, out_type=int, num_threads=N_THREADS)
430
+ return [sp.encode_as_ids(t) for t in texts]
431
+
432
+ # 1. Discover all 247 Parquet files across all 33 subsets
433
+ api = HfApi(token=HF_TOKEN)
434
+ tree = list(api.list_repo_tree(repo_id=CORPUS_REPO, repo_type="dataset", path_in_repo="data", recursive=True))
435
+
436
+ subset_files = defaultdict(list)
437
+ for item in tree:
438
+ if isinstance(item, RepoFile) and item.path.endswith(".parquet"):
439
+ parts = item.path.split("/")
440
+ if len(parts) == 4:
441
+ subset_files[(parts[1], parts[2])].append(item.path)
442
+
443
+ all_subsets = sorted(subset_files.keys())
444
+ subset_to_id = {f"{g}/{s}": idx for idx, (g, s) in enumerate(all_subsets)}
445
+ print(f"📦 Found {len(all_subsets)} subsets across {sum(len(v) for v in subset_files.values())} Parquet files")
446
+
447
+ # Style & BT flags bitmask
448
+ FLAG_BT = 1 # bit 0: MyanmarTranslator_works synthetic EN side
449
+ FLAG_FORMAL = 2 # bit 1: Formal/written style
450
+ FLAG_SPOKEN = 4 # bit 2: Spoken/conversational style
451
+
452
+ FORMAL_SUBSETS = {
453
+ "myanmar-written-corpus",
454
+ "jw_myanmar_bible_dataset",
455
+ "tipitaka_myanmar_translation_books",
456
+ "myanmar-wikipedia-corpus",
457
+ "myanmar-wikipedia-dataset",
458
+ "combined_4_repos",
459
+ "musannaf_ibn_abi_shaybah",
460
+ "imam_albani_weaknfab_series_dataset",
461
+ "Quran_English_Myanmar_Parrelel_Corpus",
462
+ }
463
+ SPOKEN_SUBSETS = {
464
+ "myanmar_spoken_corpus",
465
+ "mig-english-myanmar-translation",
466
+ "MT-myanmar-english",
467
+ "english-myanmar",
468
+ }
469
+
470
+ MAX_SEQ_TOKENS = 2040 # Leaves room for prefix control tokens + EOS inside max_seq_len=2048
471
+
472
+ INDEX_DTYPE = np.dtype([
473
+ ("my_off", np.uint64),
474
+ ("my_len", np.uint16),
475
+ ("en_off", np.uint64),
476
+ ("en_len", np.uint16),
477
+ ("subset_id", np.uint8),
478
+ ("flags", np.uint8),
479
+ ])
480
+
481
+ # Build flat ordered task list: (group, subset, subset_id, file_idx_in_subset, remote_path)
482
+ tasks = []
483
+ for g, s in all_subsets:
484
+ sid = subset_to_id[f"{g}/{s}"]
485
+ for f_i, rp in enumerate(sorted(subset_files[(g, s)])):
486
+ tasks.append((g, s, sid, f_i, rp))
487
+
488
+ def download_one(remote_path: str) -> str:
489
+ return hf_hub_download(
490
+ repo_id=CORPUS_REPO,
491
+ repo_type="dataset",
492
+ filename=remote_path,
493
+ local_dir=DL_CACHE_DIR,
494
+ token=HF_TOKEN,
495
+ )
496
+
497
+ # Open binary writers
498
+ train_my_f = open(os.path.join(MEMMAP_DIR, "train_my.bin"), "wb")
499
+ train_en_f = open(os.path.join(MEMMAP_DIR, "train_en.bin"), "wb")
500
+ val_my_f = open(os.path.join(MEMMAP_DIR, "val_my.bin"), "wb")
501
+ val_en_f = open(os.path.join(MEMMAP_DIR, "val_en.bin"), "wb")
502
+
503
+ train_my_off = 0
504
+ train_en_off = 0
505
+ val_my_off = 0
506
+ val_en_off = 0
507
+
508
+ train_idx_chunks = []
509
+ val_idx_chunks = []
510
+ subset_stats = defaultdict(lambda: {"train_rows": 0, "val_rows": 0, "my_toks": 0, "en_toks": 0})
511
+
512
+ t_global = time.time()
513
+ total_rows_seen = 0
514
+ total_skipped = 0
515
+
516
+ # Prefetch pipeline (1 ahead)
517
+ with ThreadPoolExecutor(max_workers=2) as dl_pool:
518
+ next_fut = dl_pool.submit(download_one, tasks[0][4])
519
+
520
+ for t_idx, (g, s, sid, f_i, rp) in enumerate(tasks):
521
+ local_pq = next_fut.result()
522
+ if t_idx + 1 < len(tasks):
523
+ next_fut = dl_pool.submit(download_one, tasks[t_idx + 1][4])
524
+
525
+ t_file = time.time()
526
+ pf = pq.ParquetFile(local_pq)
527
+ file_total_rows = pf.metadata.num_rows
528
+
529
+ # Deterministic stratified val selection ONLY from the first file of each subset
530
+ val_set_local = set()
531
+ if f_i == 0 and file_total_rows >= 20:
532
+ val_target = min(1000, max(20, file_total_rows // 50))
533
+ stride = max(1, file_total_rows // val_target)
534
+ val_set_local = set(range(0, file_total_rows, stride)[:val_target])
535
+
536
+ base_flag = FLAG_BT if g == "MyanmarTranslator_works" else 0
537
+ if s in FORMAL_SUBSETS:
538
+ base_flag |= FLAG_FORMAL
539
+ elif s in SPOKEN_SUBSETS:
540
+ base_flag |= FLAG_SPOKEN
541
+
542
+ local_row_cursor = 0
543
+ t_idx_buf = []
544
+ v_idx_buf = []
545
+
546
+ for batch in pf.iter_batches(batch_size=40_000, columns=["myanmar_text_original", "english_text_translation"]):
547
+ my_raw = batch.column(0).to_pylist()
548
+ en_raw = batch.column(1).to_pylist()
549
+
550
+ valid_pairs = []
551
+ valid_pos = []
552
+ for k, (m, e) in enumerate(zip(my_raw, en_raw)):
553
+ r_pos = local_row_cursor + k
554
+ if isinstance(m, str) and isinstance(e, str):
555
+ ms, es = m.strip(), e.strip()
556
+ if ms and es:
557
+ valid_pairs.append((ms, es))
558
+ valid_pos.append(r_pos)
559
+ else:
560
+ total_skipped += 1
561
+ else:
562
+ total_skipped += 1
563
+
564
+ local_row_cursor += len(my_raw)
565
+ if not valid_pairs:
566
+ continue
567
+
568
+ my_tok_lists = encode_batch([p[0] for p in valid_pairs])
569
+ en_tok_lists = encode_batch([p[1] for p in valid_pairs])
570
+
571
+ t_my_arrays, t_en_arrays = [], []
572
+ v_my_arrays, v_en_arrays = [], []
573
+
574
+ for r_pos, m_ids, e_ids in zip(valid_pos, my_tok_lists, en_tok_lists):
575
+ if not m_ids or not e_ids:
576
+ total_skipped += 1
577
+ continue
578
+ if len(m_ids) > MAX_SEQ_TOKENS:
579
+ m_ids = m_ids[:MAX_SEQ_TOKENS]
580
+ if len(e_ids) > MAX_SEQ_TOKENS:
581
+ e_ids = e_ids[:MAX_SEQ_TOKENS]
582
+
583
+ m_arr = np.asarray(m_ids, dtype=np.uint16)
584
+ e_arr = np.asarray(e_ids, dtype=np.uint16)
585
+ ml = len(m_arr)
586
+ el = len(e_arr)
587
+
588
+ row_flag = base_flag
589
+ if s == "quran_written_spoken_myanmar_english":
590
+ row_flag |= (FLAG_FORMAL if (r_pos % 2 == 0) else FLAG_SPOKEN)
591
+ elif s == "myanmar_quran_parallel_human_vs_ai_native":
592
+ row_flag |= (FLAG_FORMAL if (r_pos % 3 in (0, 1)) else FLAG_SPOKEN)
593
+
594
+ key = f"{g}/{s}"
595
+ subset_stats[key]["my_toks"] += ml
596
+ subset_stats[key]["en_toks"] += el
597
+
598
+ if r_pos in val_set_local:
599
+ v_idx_buf.append((val_my_off, ml, val_en_off, el, sid, row_flag))
600
+ v_my_arrays.append(m_arr)
601
+ v_en_arrays.append(e_arr)
602
+ val_my_off += ml
603
+ val_en_off += el
604
+ subset_stats[key]["val_rows"] += 1
605
+ else:
606
+ t_idx_buf.append((train_my_off, ml, train_en_off, el, sid, row_flag))
607
+ t_my_arrays.append(m_arr)
608
+ t_en_arrays.append(e_arr)
609
+ train_my_off += ml
610
+ train_en_off += el
611
+ subset_stats[key]["train_rows"] += 1
612
+
613
+ if t_my_arrays:
614
+ train_my_f.write(np.concatenate(t_my_arrays).tobytes())
615
+ train_en_f.write(np.concatenate(t_en_arrays).tobytes())
616
+ if v_my_arrays:
617
+ val_my_f.write(np.concatenate(v_my_arrays).tobytes())
618
+ val_en_f.write(np.concatenate(v_en_arrays).tobytes())
619
+
620
+ total_rows_seen += len(my_raw)
621
+
622
+ if t_idx_buf:
623
+ train_idx_chunks.append(np.asarray(t_idx_buf, dtype=INDEX_DTYPE))
624
+ if v_idx_buf:
625
+ val_idx_chunks.append(np.asarray(v_idx_buf, dtype=INDEX_DTYPE))
626
+
627
+ # Remove temporary downloaded Parquet file immediately to keep disk low
628
+ if os.path.exists(local_pq):
629
+ os.remove(local_pq)
630
+
631
+ if (t_idx + 1) % 10 == 0 or (t_idx + 1) == len(tasks):
632
+ elapsed = time.time() - t_global
633
+ rate = total_rows_seen / max(1e-3, elapsed)
634
+ tot_tok_b = (train_my_off + train_en_off + val_my_off + val_en_off) / 1e9
635
+ print(
636
+ f" [{t_idx + 1:3d}/{len(tasks)}] {g}/{s:<38} | "
637
+ f"Rows: {total_rows_seen:>10,} ({rate:,.0f} rows/s) | "
638
+ f"Tokens: {tot_tok_b:.2f}B | Elapsed: {elapsed/60:.1f}m"
639
+ )
640
+
641
+ train_my_f.close()
642
+ train_en_f.close()
643
+ val_my_f.close()
644
+ val_en_f.close()
645
+ shutil.rmtree(DL_CACHE_DIR, ignore_errors=True)
646
+
647
+ train_index = np.concatenate(train_idx_chunks) if train_idx_chunks else np.empty(0, dtype=INDEX_DTYPE)
648
+ val_index = np.concatenate(val_idx_chunks) if val_idx_chunks else np.empty(0, dtype=INDEX_DTYPE)
649
+ del train_idx_chunks, val_idx_chunks
650
+ gc.collect()
651
+
652
+ np.save(os.path.join(MEMMAP_DIR, "train_index.npy"), train_index)
653
+ np.save(os.path.join(MEMMAP_DIR, "val_index.npy"), val_index)
654
+
655
+ meta = {
656
+ "vocab_size": 65536,
657
+ "max_seq_tokens": MAX_SEQ_TOKENS,
658
+ "special_tokens": {
659
+ "unk": 0, "bos": 1, "eos": 2, "pad": 3,
660
+ "2my": 4, "2en": 5, "formal": 6, "spoken": 7, "bt": 8, "denoise": 9, "mask": 10,
661
+ },
662
+ "flags": {"FLAG_BT": FLAG_BT, "FLAG_FORMAL": FLAG_FORMAL, "FLAG_SPOKEN": FLAG_SPOKEN},
663
+ "subset_to_id": subset_to_id,
664
+ "id_to_subset": {str(v): k for k, v in subset_to_id.items()},
665
+ "totals": {
666
+ "train_rows": int(len(train_index)),
667
+ "val_rows": int(len(val_index)),
668
+ "skipped_rows": int(total_skipped),
669
+ "train_my_tokens": int(train_my_off),
670
+ "train_en_tokens": int(train_en_off),
671
+ "val_my_tokens": int(val_my_off),
672
+ "val_en_tokens": int(val_en_off),
673
+ },
674
+ "per_subset": dict(subset_stats),
675
+ }
676
+ with open(os.path.join(MEMMAP_DIR, "meta.json"), "w", encoding="utf-8") as f:
677
+ json.dump(meta, f, indent=2)
678
+
679
+ # Verify memmap roundtrip integrity on random train & val entries
680
+ tr_my_mm = np.memmap(os.path.join(MEMMAP_DIR, "train_my.bin"), dtype=np.uint16, mode="r")
681
+ tr_en_mm = np.memmap(os.path.join(MEMMAP_DIR, "train_en.bin"), dtype=np.uint16, mode="r")
682
+ for probe_i in [0, len(train_index) // 2, len(train_index) - 1]:
683
+ rec = train_index[probe_i]
684
+ m_dec = sp.decode_ids(tr_my_mm[int(rec["my_off"]) : int(rec["my_off"]) + int(rec["my_len"])].tolist())
685
+ e_dec = sp.decode_ids(tr_en_mm[int(rec["en_off"]) : int(rec["en_off"]) + int(rec["en_len"])].tolist())
686
+ assert m_dec and e_dec, f"Empty decode at {probe_i}"
687
+
688
+ disk_gb = sum(os.path.getsize(os.path.join(MEMMAP_DIR, fn)) for fn in os.listdir(MEMMAP_DIR)) / 1e9
689
+ print("\n" + "=" * 95)
690
+ print(f"🎉 TOKENIZATION COMPLETE in {(time.time() - t_global)/60:.1f} min! Saved to: {MEMMAP_DIR}")
691
+ print(f" • Train rows : {len(train_index):,} | Val rows: {len(val_index):,} | Skipped: {total_skipped:,}")
692
+ print(f" • Train tokens: MY={train_my_off/1e9:.3f}B + EN={train_en_off/1e9:.3f}B = {(train_my_off+train_en_off)/1e9:.3f}B tokens")
693
+ print(f" • Total Memmap Disk Size: {disk_gb:.2f} GB")
694
+ print("=" * 95)
695
+
696
+ # === PHASE 1C, CELL 2: Backup tokenized 48.35M memmap to freococo/mytrans_backup/tokenized_65k/ ===
697
+ import os
698
+ import time
699
+ from google.colab import userdata
700
+ from huggingface_hub import HfApi
701
+
702
+ HF_TOKEN = userdata.get("HF_TOKEN")
703
+ BACKUP_REPO = "freococo/mytrans_backup"
704
+ MEMMAP_DIR = "/content/mytrans_memmap_65k"
705
+
706
+ assert os.path.exists(os.path.join(MEMMAP_DIR, "train_index.npy")), "Run Cell 1 first!"
707
+
708
+ api = HfApi(token=HF_TOKEN)
709
+ print("📦 Files to upload:")
710
+ for fn in sorted(os.listdir(MEMMAP_DIR)):
711
+ fp = os.path.join(MEMMAP_DIR, fn)
712
+ print(f" • {fn:<20} {os.path.getsize(fp)/1e9:.3f} GB")
713
+
714
+ print(f"\n☁️ Uploading {MEMMAP_DIR} -> {BACKUP_REPO}/tokenized_65k/ ...")
715
+ t0 = time.time()
716
+ api.upload_folder(
717
+ folder_path=MEMMAP_DIR,
718
+ path_in_repo="tokenized_65k",
719
+ repo_id=BACKUP_REPO,
720
+ repo_type="dataset",
721
+ commit_message="Backup canonical uint16 tokenized 48.35M+ memmap (MyTrans-65k)",
722
+ )
723
+ print(f"🎉 Uploaded tokenized_65k in {(time.time() - t0)/60:.1f} min! https://huggingface.co/datasets/{BACKUP_REPO}")
724
+
725
  import os
726
  os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
727