ViuAI commited on
Commit
fabe182
·
verified ·
1 Parent(s): 1d15ee9

Delete train_viutranslate.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. train_viutranslate.py +0 -402
train_viutranslate.py DELETED
@@ -1,402 +0,0 @@
1
- # ==============================================================================
2
- # 🚀 ViuTranslate — Dedicated Neural Translation Master Training Engine
3
- # ==============================================================================
4
- # Model Architecture: ViuAI Sarus-500M
5
- # Target Repository: ViuAI/ViuTranslate
6
- # Dataset Repository: ViuAI/ViuTranslate-Data
7
- # Hardware: Auto-Tuned (Kaggle T4 x 2, P100, A100, RTX 3090/4090)
8
- # ==============================================================================
9
-
10
- import os
11
- import sys
12
- import math
13
- import time
14
- import shutil
15
- import argparse
16
- import numpy as np
17
- import torch
18
- import torch.nn as nn
19
- import torch.nn.functional as F
20
- from torch.utils.data import Dataset, DataLoader, Sampler
21
- from huggingface_hub import HfApi, hf_hub_download
22
-
23
- # Fix stdout encoding for Windows & Cloud
24
- if hasattr(sys.stdout, "reconfigure"):
25
- sys.stdout.reconfigure(encoding="utf-8", errors="replace")
26
- if hasattr(sys.stderr, "reconfigure"):
27
- sys.stderr.reconfigure(encoding="utf-8", errors="replace")
28
-
29
- cur_dir = os.path.dirname(os.path.abspath(__file__)) if "__file__" in locals() else os.getcwd()
30
- if cur_dir not in sys.path:
31
- sys.path.insert(0, cur_dir)
32
-
33
- from model import ViuAI
34
- from config import ViuAIConfig
35
-
36
- PAD_TOKEN_ID = 64000
37
- EOT_ID = 64002
38
-
39
- DOMAIN_NAMES = {
40
- 0: "en_to_hi_direct",
41
- 1: "en_to_hi_command",
42
- 2: "hi_to_en_direct",
43
- 3: "hi_to_en_command"
44
- }
45
-
46
- # ------------------------------------------------------------------------------
47
- # 1. Hardware Profiler
48
- # ------------------------------------------------------------------------------
49
- def auto_profile_hardware():
50
- if not torch.cuda.is_available():
51
- return {
52
- "tier": "CPU", "device_name": "CPU", "vram_gb": 0.0,
53
- "micro_batch": 1, "grad_accum": 64, "dtype": torch.float32,
54
- "desc": "CPU fallback mode"
55
- }
56
-
57
- props = torch.cuda.get_device_properties(0)
58
- device_name = props.name
59
- vram_gb = props.total_memory / (1024 ** 3)
60
- major, minor = props.major, props.minor
61
- bf16_supported = torch.cuda.is_bf16_supported()
62
- dtype = torch.bfloat16 if bf16_supported else torch.float16
63
-
64
- if vram_gb >= 30:
65
- micro_batch, grad_accum = 32, 2
66
- desc = "NVIDIA RTX 5090 / High-Tier 32GB Blackwell Beast"
67
- elif vram_gb >= 20:
68
- micro_batch, grad_accum = 12, 6
69
- desc = "Pro-Tier GPU (24GB)"
70
- elif vram_gb >= 12:
71
- micro_batch, grad_accum = 8, 8
72
- desc = "Standard Cloud GPU / 16GB (Kaggle T4 / P100)"
73
- else:
74
- micro_batch, grad_accum = 4, 16
75
- desc = "Budget GPU (< 12GB)"
76
-
77
- return {
78
- "tier": "GPU",
79
- "device_name": device_name,
80
- "vram_gb": vram_gb,
81
- "compute_cap": f"{major}.{minor}",
82
- "micro_batch": micro_batch,
83
- "grad_accum": grad_accum,
84
- "effective_batch": micro_batch * grad_accum,
85
- "dtype": dtype,
86
- "desc": desc
87
- }
88
-
89
- # ------------------------------------------------------------------------------
90
- # 2. Dataset & Length Grouping
91
- # ------------------------------------------------------------------------------
92
- class TranslationDataset(Dataset):
93
- def __init__(self, ids_path: str, labels_path: str, offsets_path: str, domains_path: str = None):
94
- self.tokens_mmap = np.load(ids_path, mmap_mode="r")
95
- self.labels_mmap = np.load(labels_path, mmap_mode="r")
96
- self.offsets = np.load(offsets_path)
97
- self.domains = np.load(domains_path) if (domains_path and os.path.exists(domains_path)) else None
98
- self.num_samples = len(self.offsets) - 1
99
-
100
- def __len__(self):
101
- return self.num_samples
102
-
103
- def __getitem__(self, idx):
104
- start_idx = int(self.offsets[idx])
105
- end_idx = int(self.offsets[idx + 1])
106
- tokens = torch.from_numpy(self.tokens_mmap[start_idx:end_idx].astype(np.int64))
107
- labels = torch.from_numpy(self.labels_mmap[start_idx:end_idx].astype(np.int64))
108
- domain_id = int(self.domains[idx]) if self.domains is not None else 0
109
- return tokens, labels, domain_id
110
-
111
- class LengthGroupedBatchSampler(Sampler):
112
- def __init__(self, dataset, batch_size: int, mega_batch_mult: int = 40, shuffle: bool = True):
113
- self.dataset = dataset
114
- self.batch_size = batch_size
115
- self.mega_batch_mult = mega_batch_mult
116
- self.shuffle = shuffle
117
- self.lengths = dataset.offsets[1:] - dataset.offsets[:-1]
118
-
119
- def __iter__(self):
120
- indices = np.random.permutation(len(self.dataset)) if self.shuffle else np.arange(len(self.dataset))
121
- mega_batch_size = self.batch_size * self.mega_batch_mult
122
- for i in range(0, len(indices), mega_batch_size):
123
- mega_batch = indices[i:i + mega_batch_size]
124
- mega_batch = mega_batch[np.argsort(self.lengths[mega_batch])]
125
- for j in range(0, len(mega_batch), self.batch_size):
126
- yield mega_batch[j:j + self.batch_size].tolist()
127
-
128
- def __len__(self):
129
- return math.ceil(len(self.dataset) / self.batch_size)
130
-
131
- def collate_fn(batch):
132
- tokens_list, labels_list, domain_list = zip(*batch)
133
- max_len = min(512, max(len(t) for t in tokens_list))
134
-
135
- padded_tokens = torch.full((len(batch), max_len), PAD_TOKEN_ID, dtype=torch.long)
136
- padded_labels = torch.full((len(batch), max_len), -100, dtype=torch.long)
137
-
138
- for i, (tok, lab) in enumerate(zip(tokens_list, labels_list)):
139
- l = min(len(tok), max_len)
140
- padded_tokens[i, :l] = tok[:l]
141
- padded_labels[i, :l] = lab[:l]
142
-
143
- return padded_tokens, padded_labels, torch.tensor(domain_list, dtype=torch.long)
144
-
145
- # ------------------------------------------------------------------------------
146
- # 3. Learning Rate Scheduler (Cosine with Warmup)
147
- # ------------------------------------------------------------------------------
148
- def get_lr(step, warmup_steps, total_steps, max_lr, min_lr):
149
- if step < warmup_steps:
150
- return max_lr * (step + 1) / max(1, warmup_steps)
151
- if step > total_steps:
152
- return min_lr
153
- decay_ratio = (step - warmup_steps) / max(1, (total_steps - warmup_steps))
154
- coeff = 0.5 * (1.0 + math.cos(math.pi * decay_ratio))
155
- return min_lr + coeff * (max_lr - min_lr)
156
-
157
- # ------------------------------------------------------------------------------
158
- # 4. Live Evaluation Previews
159
- # ------------------------------------------------------------------------------
160
- @torch.no_grad()
161
- def run_live_eval_previews(model, tokenizer, device):
162
- if tokenizer is None:
163
- return
164
- test_cases = [
165
- ("Direct EN -> HI", "<|user|>\nThe sun rises in the east and sets in the west.<|endofturn|>\n<|assistant|>\n"),
166
- ("Direct HI -> EN", "<|user|>\nसूरज पूर्व में उगता है और पश्चिम में डूबता है।<|endofturn|>\n<|assistant|>\n"),
167
- ("Command EN -> HI", "<|user|>\nTranslate to Hindi: 'Consistency and discipline are the keys to long term success.'<|endofturn|>\n<|assistant|>\n"),
168
- ("Command HI -> EN", "<|user|>\nTranslate to English: 'सफलता का कोई शॉर्टकट नहीं होता, निरंतर प्रयास ही कुंजी है।'<|endofturn|>\n<|assistant|>\n")
169
- ]
170
- print("\n 💬 --- [LIVE TRANSLATION PREVIEWS] ---")
171
- model.eval()
172
- for label, prompt in test_cases:
173
- ids = torch.tensor([tokenizer.encode(prompt).ids], dtype=torch.long, device=device)
174
- out = model.generate(ids, max_new_tokens=45, temperature=0.2, eos_token_id=EOT_ID)
175
- gen = tokenizer.decode(out[0][ids.shape[1]:].tolist()).replace("<|endofturn|>", "").strip()
176
- print(f" • [{label:18s}]: \"{gen}\"")
177
- model.train()
178
-
179
- # ------------------------------------------------------------------------------
180
- # 5. Cloud Auto-Download Helper
181
- # ------------------------------------------------------------------------------
182
- def ensure_dataset_and_base_ckpt(data_dir: str, base_ckpt_path: str, token: str = None):
183
- os.makedirs(data_dir, exist_ok=True)
184
- shards = [
185
- "train_tokens.npy", "train_labels.npy", "train_offsets.npy", "train_domains.npy",
186
- "val_tokens.npy", "val_labels.npy", "val_offsets.npy", "val_domains.npy",
187
- "metadata.json"
188
- ]
189
- missing = [s for s in shards if not os.path.exists(os.path.join(data_dir, s))]
190
- if missing:
191
- print(f"\n🌐 Downloading ViuTranslate-Data shards from Hugging Face Hub (ViuAI/ViuTranslate-Data)...")
192
- for s in shards:
193
- target = os.path.join(data_dir, s)
194
- if not os.path.exists(target):
195
- print(f" • Fetching {s}...")
196
- dl = hf_hub_download(repo_id="ViuAI/ViuTranslate-Data", filename=s, repo_type="dataset", token=token)
197
- shutil.copy(dl, target)
198
- print(" ✅ All dataset shards downloaded.")
199
-
200
- if not os.path.exists(base_ckpt_path):
201
- print(f"\n🌐 Base checkpoint not found. Downloading base weights from ViuAI/ViuAI-500M...")
202
- os.makedirs(os.path.dirname(base_ckpt_path), exist_ok=True)
203
- dl_b = hf_hub_download(repo_id="ViuAI/ViuAI-500M", filename="checkpoints/ckpt_latest.pt", token=token)
204
- shutil.copy(dl_b, base_ckpt_path)
205
- print(" ✅ Base checkpoint ready.")
206
-
207
- # ------------------------------------------------------------------------------
208
- # 6. Main Training Function
209
- # ------------------------------------------------------------------------------
210
- def main():
211
- hw = auto_profile_hardware()
212
-
213
- parser = argparse.ArgumentParser(description="ViuTranslate Dedicated Training Engine")
214
- parser.add_argument("--data_dir", type=str, default="data/tokenized", help="Tokenized dataset directory")
215
- parser.add_argument("--base_ckpt", type=str, default="checkpoints/ckpt_latest.pt", help="Base pretrained weights")
216
- parser.add_argument("--output_dir", type=str, default="checkpoints", help="Output directory for trained model")
217
- parser.add_argument("--batch_size", type=int, default=None, help="Micro batch size")
218
- parser.add_argument("--grad_accum", type=int, default=None, help="Gradient accumulation steps")
219
- parser.add_argument("--epochs", type=int, default=3, help="Training epochs (Default: 3)")
220
- parser.add_argument("--max_lr", type=float, default=3.2e-5, help="Peak learning rate")
221
- parser.add_argument("--min_lr", type=float, default=2.0e-6, help="Min learning rate")
222
- parser.add_argument("--neftune_alpha", type=float, default=5.0, help="NEFTune noise scale")
223
- parser.add_argument("--eval_interval", type=int, default=250, help="Evaluation interval")
224
- parser.add_argument("--push_to_hf", action="store_true", default=False, help="Upload directly to ViuAI/ViuTranslate")
225
- parser.add_argument("--hf_token", type=str, default=None, help="Hugging Face API token")
226
- args = parser.parse_args()
227
-
228
- micro_b = args.batch_size or hw["micro_batch"]
229
- grad_acc = args.grad_accum or hw["grad_accum"]
230
- eff_batch = micro_b * grad_acc
231
-
232
- device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
233
- print("=" * 80)
234
- print("🚀 ViuTranslate-500M — Dedicated Neural Translation Training Engine")
235
- print(f" • Device: {hw['device_name']} ({hw['vram_gb']:.2f} GB VRAM)")
236
- print(f" • Batch Config: Micro-Batch {micro_b} × Accum {grad_acc} = Effective Batch {eff_batch}")
237
- print(f" • Target Epochs: {args.epochs}")
238
- print(f" • Precision: {hw['dtype']}")
239
- print(f" • Target HF Repo: ViuAI/ViuTranslate")
240
- print("=" * 80)
241
-
242
- # Cloud Sync
243
- ensure_dataset_and_base_ckpt(args.data_dir, args.base_ckpt, args.hf_token)
244
-
245
- # Load Tokenizer
246
- tokenizer = None
247
- tok_candidates = ["tokenizer.json", "tokenizer/tokenizer.json", os.path.join(cur_dir, "tokenizer.json")]
248
- for tc in tok_candidates:
249
- if os.path.exists(tc):
250
- try:
251
- from tokenizers import Tokenizer
252
- tokenizer = Tokenizer.from_file(tc)
253
- print(f"✅ Tokenizer loaded successfully ({tokenizer.get_vocab_size():,} vocab)")
254
- break
255
- except Exception:
256
- pass
257
-
258
- # Datasets & Loaders
259
- train_ds = TranslationDataset(
260
- ids_path=os.path.join(args.data_dir, "train_tokens.npy"),
261
- labels_path=os.path.join(args.data_dir, "train_labels.npy"),
262
- offsets_path=os.path.join(args.data_dir, "train_offsets.npy"),
263
- domains_path=os.path.join(args.data_dir, "train_domains.npy")
264
- )
265
- val_ds = TranslationDataset(
266
- ids_path=os.path.join(args.data_dir, "val_tokens.npy"),
267
- labels_path=os.path.join(args.data_dir, "val_labels.npy"),
268
- offsets_path=os.path.join(args.data_dir, "val_offsets.npy"),
269
- domains_path=os.path.join(args.data_dir, "val_domains.npy")
270
- )
271
-
272
- train_sampler = LengthGroupedBatchSampler(train_ds, batch_size=micro_b, shuffle=True)
273
- train_loader = DataLoader(train_ds, batch_sampler=train_sampler, collate_fn=collate_fn, num_workers=2, pin_memory=True)
274
- val_loader = DataLoader(val_ds, batch_size=micro_b * 2, shuffle=False, collate_fn=collate_fn, num_workers=2)
275
-
276
- # Initialize Model
277
- cfg = ViuAIConfig.sft(vocab_size=64003, context_length=2048, neftune_alpha=args.neftune_alpha)
278
- model = ViuAI(cfg).to(device)
279
-
280
- print(f"\n📦 Loading base pretrained weights from {args.base_ckpt}...")
281
- base_state = torch.load(args.base_ckpt, map_location=device, weights_only=False)
282
- weights = base_state.get("model_state_dict", base_state)
283
- model.load_state_dict(weights, strict=False)
284
- print("✅ Pretrained weights loaded.")
285
-
286
- # Optimizer
287
- fused_available = 'fused' in torch.optim.AdamW.__init__.__code__.co_varnames and torch.cuda.is_available()
288
- optimizer = torch.optim.AdamW(model.parameters(), lr=args.max_lr, weight_decay=0.01, betas=(0.9, 0.95), fused=fused_available)
289
-
290
- total_steps = (len(train_loader) // grad_acc) * args.epochs
291
- warmup_steps = int(total_steps * 0.04)
292
- print(f"📊 Total Optimization Steps: {total_steps:,} | Warmup Steps: {warmup_steps:,}")
293
-
294
- autocast_ctx = torch.amp.autocast(device_type="cuda", dtype=hw["dtype"]) if torch.cuda.is_available() else contextlib.nullcontext()
295
-
296
- # Validation Function
297
- @torch.no_grad()
298
- def evaluate():
299
- model.eval()
300
- total_loss, total_tokens = 0.0, 0
301
- domain_losses = {k: 0.0 for k in DOMAIN_NAMES.keys()}
302
- domain_counts = {k: 0 for k in DOMAIN_NAMES.keys()}
303
-
304
- for inputs, labels, doms in val_loader:
305
- inputs, labels = inputs.to(device), labels.to(device)
306
- with autocast_ctx:
307
- logits, loss = model(inputs, targets=labels, pad_id=PAD_TOKEN_ID, shift_labels=True)
308
- tok_count = (labels != -100).sum().item()
309
- total_loss += loss.item() * tok_count
310
- total_tokens += tok_count
311
-
312
- # Domain-level tracking
313
- for d in doms.unique():
314
- d_val = d.item()
315
- mask = (doms == d)
316
- if mask.sum() > 0:
317
- with autocast_ctx:
318
- _, d_l = model(inputs[mask], targets=labels[mask], pad_id=PAD_TOKEN_ID, shift_labels=True)
319
- domain_losses[d_val] += d_l.item()
320
- domain_counts[d_val] += 1
321
-
322
- avg_loss = total_loss / max(1, total_tokens)
323
- ppl = math.exp(min(20.0, avg_loss))
324
- d_summary = {DOMAIN_NAMES[k]: (domain_losses[k] / max(1, domain_counts[k])) for k in DOMAIN_NAMES.keys()}
325
- model.train()
326
- return avg_loss, ppl, d_summary
327
-
328
- # Training Loop
329
- os.makedirs(args.output_dir, exist_ok=True)
330
- save_path = os.path.join(args.output_dir, "viutranslate_final.pt")
331
- best_val_loss = float("inf")
332
- global_step = 0
333
- start_time = time.time()
334
- total_tokens_trained = 0
335
-
336
- print("\n⚡ Starting Training...")
337
- for epoch in range(1, args.epochs + 1):
338
- accum_loss = 0.0
339
- model.train()
340
- for micro_idx, (inputs, labels, _) in enumerate(train_loader):
341
- inputs, labels = inputs.to(device, non_blocking=True), labels.to(device, non_blocking=True)
342
- active_tokens = (labels != -100).sum().item()
343
- total_tokens_trained += active_tokens
344
-
345
- with autocast_ctx:
346
- logits, loss = model(inputs, targets=labels, pad_id=PAD_TOKEN_ID, shift_labels=True)
347
- loss_scaled = loss / grad_acc
348
-
349
- loss_scaled.backward()
350
- accum_loss += loss.item()
351
-
352
- if (micro_idx + 1) % grad_acc == 0:
353
- torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
354
- lr = get_lr(global_step, warmup_steps, total_steps, args.max_lr, args.min_lr)
355
- for param_group in optimizer.param_groups:
356
- param_group["lr"] = lr
357
- optimizer.step()
358
- optimizer.zero_grad(set_to_none=True)
359
- global_step += 1
360
-
361
- step_loss = accum_loss / grad_acc
362
- accum_loss = 0.0
363
-
364
- if global_step % 10 == 0 or global_step == 1:
365
- elapsed = time.time() - start_time
366
- tok_s = total_tokens_trained / max(1.0, elapsed)
367
- vram = torch.cuda.memory_allocated() / (1024**3) if torch.cuda.is_available() else 0.0
368
- print(f"Step {global_step:4d}/{total_steps} | Epoch {epoch} | Loss: {step_loss:.4f} | LR: {lr:.2e} | Speed: {tok_s:,.0f} tok/s | VRAM: {vram:.1f}GB")
369
-
370
- if global_step % args.eval_interval == 0:
371
- v_loss, v_ppl, d_losses = evaluate()
372
- print(f"\n🌟 [Eval @ Step {global_step}] Val Loss: {v_loss:.4f} | Perplexity: {v_ppl:.2f}")
373
- print(" 📊 Direction Losses: " + " | ".join([f"{k}: {v:.3f}" for k, v in d_losses.items()]))
374
- run_live_eval_previews(model, tokenizer, device)
375
-
376
- if v_loss < best_val_loss:
377
- best_val_loss = v_loss
378
- torch.save({"model_state_dict": model.state_dict(), "global_step": global_step, "val_loss": best_val_loss}, save_path)
379
- print(f" 🏆 Saved New Best Checkpoint -> {save_path}\n")
380
-
381
- # Final Save
382
- torch.save({"model_state_dict": model.state_dict(), "global_step": global_step, "best_val_loss": best_val_loss}, save_path)
383
- print(f"\n🎉 ViuTranslate Training Complete! Final checkpoint: {save_path}")
384
-
385
- # Direct Push to ViuAI/ViuTranslate
386
- if args.push_to_hf:
387
- print("\n🚀 Pushing model weights to Hugging Face Model Repository (ViuAI/ViuTranslate)...")
388
- token = args.hf_token or os.environ.get("HF_TOKEN")
389
- if token:
390
- api = HfApi(token=token)
391
- api.upload_file(
392
- path_or_fileobj=save_path,
393
- path_in_repo="viutranslate_final.pt",
394
- repo_id="ViuAI/ViuTranslate",
395
- repo_type="model"
396
- )
397
- print("✅ Successfully uploaded viutranslate_final.pt to ViuAI/ViuTranslate!")
398
- else:
399
- print("⚠️ Skipping upload: No HF_TOKEN provided.")
400
-
401
- if __name__ == "__main__":
402
- main()