ViuAI commited on
Commit
e100ee7
·
verified ·
1 Parent(s): b410a71

Sync model/scripts/train.py with 3-stage sequence length curriculum (512->1024->2048)

Browse files
Files changed (1) hide show
  1. model/scripts/train.py +734 -191
model/scripts/train.py CHANGED
@@ -25,9 +25,10 @@ import torch.nn.functional as F
25
  from torch.utils.data import Dataset
26
 
27
  sys.path.insert(0, str(Path(__file__).parent))
28
- # repo root: data.scripts.filter_by_safety import ke liye (Kaggle pe CWD alag hota hai)
29
- sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
30
- from viu1_moe import ViuArgs, Viu1MoE, load_args as load_model_args
 
31
 
32
  try:
33
  import yaml
@@ -47,16 +48,19 @@ except ImportError:
47
 
48
  # ---------------- data ----------------
49
  def get_lang_tag(name_or_text: str) -> str:
50
- n = str(name_or_text).lower()
51
- if "hinglish" in n:
52
- return "<|hinglish|>"
53
- if "hindi" in n:
54
- return "<|hindi|>"
55
- if "english" in n:
56
  return "<|english|>"
57
- # simple heuristic for raw text: Devanagari -> hindi, else hinglish
 
 
 
 
58
  if any('\u0900' <= ch <= '\u097F' for ch in n):
59
  return "<|hindi|>"
 
 
60
  return "<|hinglish|>"
61
 
62
 
@@ -83,9 +87,19 @@ class TxtPackDataset(Dataset):
83
  ids.append(eos)
84
  if len(ids) > warn_tokens:
85
  print(f"[warn] {len(ids)/1e6:.1f}M tokens RAM me — bade corpus ke liye memmap/HF-streaming use karo.")
86
- ids = torch.tensor(ids, dtype=torch.long)
87
- n = (len(ids) // seq_len) * seq_len
88
- self.blocks = ids[:n].view(-1, seq_len) if n > 0 else torch.zeros((1, seq_len), dtype=torch.long)
 
 
 
 
 
 
 
 
 
 
89
 
90
  def __len__(self):
91
  return len(self.blocks)
@@ -104,7 +118,18 @@ def collect_txt(data_dir):
104
  return files
105
 
106
 
107
- # ---------------- sched ----------------
 
 
 
 
 
 
 
 
 
 
 
108
  def fmt_hms(sec):
109
  sec = max(int(sec), 0)
110
  h, sec = divmod(sec, 3600)
@@ -119,6 +144,152 @@ def cosine_lr(step, max_steps, warmup, lr, min_lr):
119
  return min_lr + 0.5 * (lr - min_lr) * (1 + math.cos(math.pi * min(p, 1.0)))
120
 
121
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
122
  def pick_device(pref=None):
123
  """Auto GPU: sabse zyada free VRAM wala cuda, warna cpu. --device se override."""
124
  if pref:
@@ -175,11 +346,11 @@ def probe_micro_batch(model, seq_len, vocab_size, device, amp_dtype):
175
  + fragmentation ka explicit reserve chahiye, warna wrap ke baad training step-1 pe OOM."""
176
  model.train()
177
  best = 1
178
- # Reserve: AdamW states + DDP buckets + headroom. 242M fp32 ~0.97GB/param-tensor =>
179
- # ~2GB AdamW + ~1GB buckets + ~1GB frag => try 4GB, warna 2GB, warna 1GB.
180
  adamw_buf = None
181
  if device.type == "cuda":
182
- for n_elems in (1_000_000_000, 500_000_000, 250_000_000): # ~4.0 / 2.0 / 1.0 GB
183
  try:
184
  adamw_buf = torch.empty((n_elems,), dtype=torch.float32, device=device)
185
  break
@@ -213,8 +384,77 @@ def probe_micro_batch(model, seq_len, vocab_size, device, amp_dtype):
213
  return best
214
 
215
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
216
  def save_slim_ckpt(path, model, cfg=None):
217
- """Inference & HF deployment ke liye slim checkpoint (~480MB BF16, optimizer states ke bina)."""
218
  path = Path(path)
219
  path.parent.mkdir(parents=True, exist_ok=True)
220
  state = {k: v.to(dtype=torch.bfloat16).cpu() for k, v in model.state_dict().items()}
@@ -393,7 +633,8 @@ def evaluate(model, ds, device, max_batches=20, use_amp=False, amp_dtype=torch.f
393
  # val PPL ke liye pure LM loss chahiye — MoE-aux + mt-loss hatao
394
  base = loss.item()
395
  try:
396
- base = base - float(aux.get("moe_aux", 0.0)) - 0.1 * float(aux.get("mt_loss", 0.0))
 
397
  except Exception:
398
  pass
399
  tot += base
@@ -426,8 +667,8 @@ def main():
426
  # Fragmentation kam karo (T4 pe large-alloc OOM ke baad reserved-but-unused dikhta tha)
427
  os.environ.setdefault("PYTORCH_ALLOC_CONF", "expandable_segments:True")
428
  ap = argparse.ArgumentParser()
429
- ap.add_argument("--config", default="../configs/train_config.yaml")
430
- ap.add_argument("--model_config", default="../configs/viu1_moe_config.yaml")
431
  ap.add_argument("--smoke", action="store_true", help="CPU pe tiny model + 3 steps test")
432
  ap.add_argument("--steps", "--max_steps", dest="steps", type=int, default=None, help="Max training steps override")
433
  ap.add_argument("--resume", default=None, help="Checkpoint path to resume training from")
@@ -438,7 +679,15 @@ def main():
438
  ap.add_argument("--hf_split", default="train", help="Dataset split (default: train)")
439
  ap.add_argument("--batch_size", type=int, default=None, help="Micro-batch size override")
440
  ap.add_argument("--grad_accum", type=int, default=None, help="Gradient accumulation steps override")
 
 
 
441
  ap.add_argument("--no_auto_tune", action="store_true", help="Disable auto-tune and enforce exact batch_size and grad_accum")
 
 
 
 
 
442
  args = ap.parse_args()
443
 
444
  base = Path(__file__).parent
@@ -473,7 +722,8 @@ def main():
473
  if args.smoke:
474
  print("[mode] SMOKE — tiny model, CPU, 3 steps")
475
  margs = ViuArgs(dim=64, n_layers=2, n_heads=4, n_kv_heads=2, vocab_size=512,
476
- max_seq_len=64, sliding_window=32, moe_every=2, moe_num_experts=4,
 
477
  multi_token=False, switch_gate=False)
478
  seq_len, batch, max_steps = 32, 2, 3
479
  data_dir, tok_path = base / "../../data/raw", None
@@ -484,21 +734,49 @@ def main():
484
  device = torch.device("cpu")
485
  use_bf16 = False
486
  use_fp16 = False
 
 
 
487
  else:
488
  margs = load_model_args(args.model_config if Path(args.model_config).exists()
489
  else str(base / args.model_config))
490
- seq_len = C("seq_len", 1024)
491
- batch = args.batch_size or C("batch_size", 2)
492
- max_steps = args.steps or C("max_steps", 50000)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
493
  data_dir = resolve(C("data_dir", "../../data/raw"))
494
  tok_path = resolve(C("tokenizer_path", "../../tokenizer/outputs/tokenizer.json"))
495
  lr, min_lr, warmup, wd = C("lr", 3e-4), C("min_lr", 3e-5), C("warmup_steps", 2500), C("weight_decay", 0.1)
496
- accum, clip = (args.grad_accum or C("grad_accum", 8)), C("grad_clip", 1.0)
497
  eval_every, save_every, log_every = C("eval_every", 1000), C("save_every", 5000), C("log_every", 50)
498
  sample_every = C("sample_every", 1000)
499
  ckpt_dir = resolve(C("ckpt_dir", "../checkpoints"))
500
  log_dir = resolve(C("log_dir", "../../logs"))
501
- hf_repo = C("hf_repo", "ViuAI/ViuMini-MoE-242M")
502
  hf_token = os.environ.get("HF_TOKEN")
503
  device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
504
  if args.device:
@@ -559,7 +837,7 @@ def main():
559
  import shutil
560
  try:
561
  real_tok = hf_hub_download(
562
- repo_id=hf_repo or "ViuAI/ViuMini-MoE-242M",
563
  filename="tokenizer/outputs/tokenizer.json",
564
  token=hf_token
565
  )
@@ -604,7 +882,9 @@ def main():
604
  print(f"[hf] streaming dataset {hf_dataset} (native PyArrow streaming)")
605
  # Non-5-col schema folders (AUDIT A3) — inko stream me mat lo, warna
606
  # shuffle/interleave par CastError (wikipedia: title/text/url vs text/lang/...).
607
- # CastError = training data me corrupted/NULL rows; exclude karna safe hai.
 
 
608
  # ZERO EXCLUSIONS: Stream 100% of all files in repository
609
  NON_UNIFIED_PREFIXES = ()
610
  # Corpus-slice rotation (Option A: full-data coverage round-by-round).
@@ -660,124 +940,129 @@ def main():
660
 
661
  def _stream_parquet_rows(files, repo_id):
662
  for fp in files:
663
- try:
664
- p = f"datasets/{repo_id}/{fp}"
665
- with _fs.open(p, "rb") as f:
666
- pf = _pq.ParquetFile(f)
667
- _names = set(pf.schema.names)
668
- if "target" in _names and "toxic" in _names:
669
- # Civil comments: label score is in column, not text!
670
- # Threshold at 0.5 to set safety_tag; keep text intact.
671
- # Also yield non-toxic rows so valid discussion isn't lost.
672
- cols = [c for c in ["comment_text", "text", "target", "toxic", "severe_toxicity", "obscene", "threat", "insult", "identity_attack"] if c in _names]
673
- text_col = "comment_text" if "comment_text" in _names else "text"
674
- for batch in pf.iter_batches(batch_size=2048, columns=cols):
675
- d = batch.to_pydict()
676
- txts = d.get(text_col, [])
677
- n = len(txts)
678
- for i in range(n):
679
- tw = str(txts[i]).strip()
680
- if len(tw) < 5:
681
- continue
682
- mx = 0.0
683
- for score_col in ("target", "toxic", "severe_toxicity", "insult"):
684
- if score_col in d and d[score_col][i] is not None:
685
  try:
686
- v = float(d[score_col][i])
687
- except (ValueError, TypeError):
688
  v = 0.0
689
  if v > mx:
690
  mx = v
691
- tag = "toxic" if mx >= 0.5 else "safe"
692
- yield {"text": tw, "lang": "en", "source": "jigsaw_civil_comments", "domain": "toxicity", "safety_tag": tag}
693
- elif "text" in _names:
694
- cols = [c for c in ["text", "lang", "source", "domain", "safety_tag"] if c in pf.schema.names]
695
- for batch in pf.iter_batches(batch_size=2048, columns=cols):
696
- d = batch.to_pydict()
697
- n = len(d.get("text", []))
698
- for i in range(n):
699
- yield {k: d[k][i] for k in cols}
700
- elif "generated_full_text" in pf.schema.names:
701
- for batch in pf.iter_batches(batch_size=2048, columns=["generated_full_text"]):
702
- d = batch.to_pydict()
703
- n = len(d.get("generated_full_text", []))
704
- for i in range(n):
705
- yield {"text": str(d["generated_full_text"][i]), "lang": "en", "source": "science_textbooks", "domain": "science", "safety_tag": "safe"}
706
- elif "src" in pf.schema.names and "tgt" in pf.schema.names:
707
- for batch in pf.iter_batches(batch_size=2048, columns=["src", "tgt"]):
708
- d = batch.to_pydict()
709
- n = len(d.get("src", []))
710
- for i in range(n):
711
- s = str(d["src"][i]).strip()
712
- t = str(d["tgt"][i]).strip()
713
- if len(s) > 10 and len(t) > 10:
714
- if i % 2 == 0:
715
- txt = f"English: {s}\nHindi: {t}"
716
- else:
717
- txt = f"Hindi: {t}\nEnglish: {s}"
718
- yield {"text": txt, "lang": "bilingual", "source": "samanantar_translation", "domain": "translation", "safety_tag": "safe"}
719
- elif "context" in pf.schema.names and "question" in pf.schema.names and "response" in pf.schema.names:
720
- for batch in pf.iter_batches(batch_size=2048, columns=["context", "question", "response"]):
721
- d = batch.to_pydict()
722
- n = len(d.get("question", []))
723
- for i in range(n):
724
- ctx = str(d["context"][i]).strip()
725
- q = str(d["question"][i]).strip()
726
- ans = str(d["response"][i]).strip()
727
- parts = []
728
- if ctx and len(ctx) > 30:
729
- parts.append(f"### Case Background:\n{ctx}")
730
- parts.append(f"### Legal Query:\n{q}")
731
- parts.append(f"### Legal Determination:\n{ans}")
732
- yield {"text": "\n\n".join(parts), "lang": "en", "source": "indian_courts_case_law", "domain": "governance_law", "safety_tag": "safe"}
733
- elif "instruction" in pf.schema.names and "output" in pf.schema.names:
734
- cols = [c for c in ["instruction", "input", "output"] if c in pf.schema.names]
735
- for batch in pf.iter_batches(batch_size=2048, columns=cols):
736
- d = batch.to_pydict()
737
- n = len(d["instruction"])
738
- for i in range(n):
739
- ins = str(d["instruction"][i]).strip()
740
- inp = str(d["input"][i]).strip() if "input" in d else ""
741
- ans = str(d["output"][i]).strip()
742
- if len(ins) > 5 and len(ans) > 5:
743
- if inp and len(inp) > 5:
744
- txt = f"### Instruction:\n{ins}\n\n### Input:\n{inp}\n\n### Response:\n{ans}"
745
- else:
746
- txt = f"### Instruction:\n{ins}\n\n### Response:\n{ans}"
747
- lg = "hinglish" if any('\u0900' <= ch <= '\u097F' for ch in txt[:500]) else "en"
748
- yield {"text": txt, "lang": lg, "source": "instruction_following", "domain": "instruction", "safety_tag": "safe"}
749
- elif "tweet" in pf.schema.names:
750
- for batch in pf.iter_batches(batch_size=2048, columns=["tweet"]):
751
- d = batch.to_pydict()
752
- n = len(d.get("tweet", []))
753
- for i in range(n):
754
- tw = str(d["tweet"][i]).strip()
755
- if len(tw) > 5:
756
- yield {"text": tw, "lang": "en", "source": "hate_speech_offensive", "domain": "toxicity", "safety_tag": "toxic"}
757
- elif "question" in pf.schema.names and "answer" in pf.schema.names:
758
- # Standalone Q/A pairs (math/problem sets without context column)
759
- for batch in pf.iter_batches(batch_size=2048, columns=["question", "answer"]):
760
- d = batch.to_pydict()
761
- n = len(d.get("question", []))
762
- for i in range(n):
763
- q = str(d["question"][i]).strip()
764
- a = str(d["answer"][i]).strip()
765
- if len(q) > 5 and len(a) > 5:
766
- yield {"text": f"### Problem:\n{q}\n\n### Solution:\n{a}", "lang": "en", "source": "math_qa_pairs", "domain": "reasoning_math", "safety_tag": "safe"}
767
- elif "Patient" in pf.schema.names and "Doctor" in pf.schema.names:
768
- cols = [c for c in ["Description", "Patient", "Doctor"] if c in pf.schema.names]
769
- for batch in pf.iter_batches(batch_size=2048, columns=cols):
770
- d = batch.to_pydict()
771
- n = len(d["Patient"])
772
- for i in range(n):
773
- desc = str(d["Description"][i]).strip() if "Description" in d else ""
774
- pat = str(d["Patient"][i]).strip()
775
- doc = str(d["Doctor"][i]).strip()
776
- if pat and doc:
777
- txt = f"### Medical Case:\n{desc}\n\n### Patient:\n{pat}\n\n### Doctor:\n{doc}" if desc else f"### Patient:\n{pat}\n\n### Doctor:\n{doc}"
778
- yield {"text": txt, "lang": "en", "source": "ai_medical_dialogues", "domain": "health", "safety_tag": "safe"}
779
- except Exception:
780
- continue
 
 
 
 
 
 
 
 
781
 
782
  def _stream_shuffled(gen, buffer_size=20000, seed=42):
783
  if buffer_size <= 0:
@@ -820,8 +1105,9 @@ def main():
820
  mix_max_rows = int(C("mix_max_rows", 2000000))
821
  allow_toxic = safety_mode in ("research", "full")
822
  allow_uncensored = safety_mode in ("full",)
 
823
  if R0:
824
- print(f"[hf] safety={safety_mode} (toxic={allow_toxic}, uncensored={allow_uncensored}) "
825
  f"max_blocks={max_blocks_cfg} max_rows={max_rows_cfg or 'unlimited'} "
826
  f"enforce_mix={enforce_mix} lang_mix={lang_mix if enforce_mix else 'OFF (shuffle-buffer order)'}"
827
  + (f" natural_shuffle={natural_shuffle}" if (not enforce_mix and natural_shuffle > 0) else "")
@@ -839,9 +1125,13 @@ def main():
839
  return allow_toxic
840
  if tag == "uncensored":
841
  return allow_uncensored
 
 
842
  s = f"{r.get('source','')} {r.get('domain','')}".lower()
843
  if "uncensor" in s or "unfilter" in s or "vortex" in s:
844
  return allow_uncensored
 
 
845
  if any(k in s for k in ("toxic", "hate", "offensive", "profan", "civil_comments", "hasoc", "prism")):
846
  return allow_toxic
847
  return True
@@ -849,79 +1139,124 @@ def main():
849
  def __init__(self, stream, tok, seq_len, max_blocks=50000, max_rows=None,
850
  enforce_mix=False, lang_mix=None, mix_seed=42, mix_scan_mult=5,
851
  mix_max_rows=2000000):
852
- buckets = {"hinglish": [], "hindi": [], "english": []}
853
- eos = tok.token_to_id("<eos>")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
854
  if eos is None:
855
  eos = 2
856
- cap_tok = max_blocks * seq_len
857
- weights = {k: float((lang_mix or {}).get(k, 0)) for k in buckets}
858
  if sum(weights.values()) <= 0:
859
  weights = {"hinglish": 0.4, "hindi": 0.3, "english": 0.3}
860
  tot_w = sum(weights.values())
861
  quota = {k: cap_tok * w / tot_w for k, w in weights.items()}
862
- scan_cap = cap_tok * (mix_scan_mult if enforce_mix else 1)
863
  n_rows = n_kept = n_safe_skip = n_empty = n_noschema = 0
864
  lang_rows = {"hinglish": 0, "hindi": 0, "english": 0}
865
- for row in stream:
 
 
866
  n_rows += 1
867
- if max_rows and n_rows > max_rows:
868
  break
869
  txt = row.get("text", "") if isinstance(row, dict) else ""
870
  if not txt or not str(txt).strip():
871
  n_empty += 1
872
  continue
 
 
 
 
 
873
  if not _safety_ok(row if isinstance(row, dict) else {}):
874
  n_safe_skip += 1
875
  continue
876
- txt = str(txt)
877
  try:
878
  tag = get_lang_tag(row.get("lang") or row.get("source") or txt)
879
  except Exception:
880
  n_noschema += 1
881
  continue
882
  bucket = "hinglish" if "hinglish" in tag else ("hindi" if "hindi" in tag else "english")
883
- # enforce mode: quota-full bucket skip (RAM bounded, rare lang tak scan pahunche)
884
- if enforce_mix and len(buckets[bucket]) >= quota[bucket]:
885
  n_kept += 1
886
  if all(len(buckets[k]) >= quota[k] for k in buckets):
887
  break
888
- if n_rows >= (max_rows or mix_max_rows):
889
  break
890
  continue
891
- tag_id = tok.token_to_id(tag) if tok is not None else None
892
  if tag_id is not None:
893
  buckets[bucket].append(tag_id)
894
- buckets[bucket].extend(tok.encode(txt).ids)
895
  buckets[bucket].append(eos)
896
  lang_rows[bucket] += 1
897
  n_kept += 1
898
  total_tok = sum(len(v) for v in buckets.values())
899
- if enforce_mix:
900
- # stop: sab quotas full YA scan bound (rare lang ho to graceful degrade)
901
  if all(len(buckets[k]) >= quota[k] for k in buckets):
902
  break
903
- if n_rows >= (max_rows or mix_max_rows):
904
  break
905
  if total_tok >= scan_cap:
906
  break
907
  elif total_tok >= cap_tok:
908
  break
 
 
 
 
 
 
909
  if R0:
910
- print(f"[hf] rows scanned={n_rows:,} kept={n_kept:,} mix_rows={lang_rows} "
911
  f"skip_empty={n_empty:,} skip_safety({safety_mode})={n_safe_skip:,} skip_schema={n_noschema:,}")
912
  import random as _rng
913
- if enforce_mix:
914
- # Weighted block sampler: quota tak lo; kami ho to bachi avail se bharo.
915
  per_lang_blocks = {}
916
  for k, v in buckets.items():
917
  arr = torch.tensor(v, dtype=torch.long)
918
- n = (len(arr) // seq_len) * seq_len
919
- per_lang_blocks[k] = arr[:n].view(-1, seq_len) if n > 0 else torch.zeros((0, seq_len), dtype=torch.long)
 
920
  avail = {k: len(v) for k, v in per_lang_blocks.items()}
921
- quota_b = {k: int(quota[k] // seq_len) for k in avail}
922
  take = {k: min(avail[k], quota_b[k]) for k in avail}
923
- short = max_blocks - sum(take.values())
924
- if short > 0: # rare lang short -> leftover avail se proportional fill
925
  left = {k: avail[k] - take[k] for k in avail}
926
  tot_left = sum(left.values()) or 1
927
  for k in avail:
@@ -929,24 +1264,34 @@ def main():
929
  take[k] += extra
930
  chosen = [per_lang_blocks[k][:take[k]] for k in avail if take[k] > 0]
931
  if not chosen:
932
- self.blocks = torch.zeros((1, seq_len), dtype=torch.long)
933
  else:
934
  allb = torch.cat(chosen, dim=0)
935
  idx = list(range(len(allb)))
936
- _rng.Random(mix_seed).shuffle(idx)
937
  self.blocks = allb[idx]
938
  if R0:
939
  print(f"[hf] mix ENFORCED target={weights}: blocks_taken={take} total={len(self.blocks)} (avail={avail})")
940
  else:
 
 
 
 
 
941
  ids = buckets["hinglish"] + buckets["hindi"] + buckets["english"]
942
  if R0:
943
- print(f"[hf] mix NATURAL (rule OFF): prefetched {len(ids)} tokens -> {len(ids)//max(seq_len,1)} blocks")
944
  ids = torch.tensor(ids, dtype=torch.long)
945
- n = (len(ids)//seq_len)*seq_len
946
- self.blocks = ids[:n].view(-1, seq_len) if n>0 else torch.zeros((1,seq_len),dtype=torch.long)
947
- def __len__(self): return len(self.blocks)
948
- def __getitem__(self,i):
949
- x=self.blocks[i]; return x[:-1], x[1:]
 
 
 
 
 
950
  ds = HFStreamDataset(hf_ds, tok, seq_len+1, max_blocks=max_blocks_cfg, max_rows=max_rows_cfg,
951
  enforce_mix=enforce_mix, lang_mix=lang_mix, mix_seed=seed_r,
952
  mix_scan_mult=mix_scan_mult, mix_max_rows=mix_max_rows)
@@ -1003,14 +1348,23 @@ def main():
1003
  ids.append(eos)
1004
  import torch as _torch
1005
  ids_t = _torch.tensor(ids, dtype=_torch.long)
1006
- n = (len(ids_t) // (seq_len + 1)) * (seq_len + 1)
1007
- blocks = ids_t[:n].view(-1, seq_len + 1) if n > 0 else _torch.zeros((1, seq_len + 1), dtype=_torch.long)
1008
  class _DummyDS(_torch.utils.data.Dataset):
1009
- def __init__(self, blocks): self.blocks = blocks
 
 
 
 
 
 
 
 
 
 
 
1010
  def __len__(self): return len(self.blocks)
1011
  def __getitem__(self, i):
1012
  x = self.blocks[i]; return x[:-1], x[1:]
1013
- ds = _DummyDS(blocks)
1014
  elif ds is not None:
1015
  pass # ds already built via HF streaming above
1016
  else:
@@ -1029,7 +1383,12 @@ def main():
1029
  if len(ds) > n_train:
1030
  train_ids = all_idx[:n_train]
1031
  val_ids = all_idx[n_train:]
1032
- val_ds = torch.utils.data.Subset(ds, val_ids)
 
 
 
 
 
1033
  else:
1034
  train_ids = all_idx
1035
  val_ds = None
@@ -1071,6 +1430,12 @@ def main():
1071
  + (" [dist broadcast r0->all]" if (use_dist and not args.smoke) else "")) if R0 else None
1072
  elif not args.smoke and device.type == "cuda" and R0:
1073
  print(f"[batch] MANUAL LOCK: micro={batch} accum={accum} eff={batch * accum} tok/step={batch * accum * seq_len * (world if use_dist else 1)} amp={'bf16' if use_bf16 else 'fp16'}")
 
 
 
 
 
 
1074
  if use_dist and not args.smoke:
1075
  # MoE top-2 routing me kuch experts kabhi-kabhi empty rehte hain -> find_unused_parameters=True (safe)
1076
  model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[device.index], find_unused_parameters=True)
@@ -1081,13 +1446,68 @@ def main():
1081
  if R0:
1082
  print(f"[model] total={total/1e6:.1f}M active={active/1e6:.1f}M device={device}" + (f" (DDP x{world})" if use_dist else ""))
1083
 
 
 
 
 
 
 
 
1084
  decay_params = [p for p in model.parameters() if p.requires_grad and p.ndim >= 2]
1085
  no_decay_params = [p for p in model.parameters() if p.requires_grad and p.ndim < 2]
1086
  optim_groups = [
1087
  {"params": decay_params, "weight_decay": wd},
1088
  {"params": no_decay_params, "weight_decay": 0.0},
1089
  ]
1090
- opt = torch.optim.AdamW(optim_groups, lr=lr, betas=tuple(C("betas", [0.9, 0.95])) if not args.smoke else (0.9, 0.95))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1091
  use_amp = use_bf16 or use_fp16
1092
  amp_dtype = torch.bfloat16 if use_bf16 else torch.float16
1093
  scaler = torch.amp.GradScaler("cuda") if (use_fp16 and device.type == "cuda") else None
@@ -1167,6 +1587,28 @@ def main():
1167
  start = load_ckpt(resolved_ckpt, ckpt_model, opt, scaler=scaler)
1168
  if R0:
1169
  print(f"[resume] successfully resumed at step {start} from {resolved_ckpt}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1170
  else:
1171
  if R0:
1172
  print(f"[resume] checkpoint '{resume}' not found locally or on HF Hub — starting fresh from step 0")
@@ -1194,13 +1636,67 @@ def main():
1194
  t0 = time.time()
1195
  step = start
1196
  idx = 0
 
 
 
1197
  tok_per_step = batch * accum * seq_len * (world if use_dist else 1) # DDP: global tokens/step
1198
  pbar = None # Industry standard: clean stdout lines (no carriage-return progress bar in web terminal)
1199
- rng.shuffle(train_ids)
1200
  while step < max_steps:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1201
  if idx >= len(train_ids):
1202
- rng.shuffle(train_ids)
1203
- idx = 0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1204
  # accumulate (batch_size fix: har micro-step me batch samples stack karo)
1205
  opt.zero_grad(set_to_none=True)
1206
  acc_loss = 0.0
@@ -1245,12 +1741,50 @@ def main():
1245
  layer_avg = [sum(acc_usages[s][l_idx][e] for s in range(len(acc_usages))) / len(acc_usages) for e in range(n_exp)]
1246
  avg_usages.append(layer_avg)
1247
  aux = {"moe_aux": round(acc_moe_aux, 4), "mt_loss": round(acc_mt_loss, 4), "usages": avg_usages}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1248
  if scaler is not None:
1249
  scaler.unscale_(opt)
1250
- torch.nn.utils.clip_grad_norm_(model.parameters(), clip)
 
 
 
 
1251
  cur_lr = cosine_lr(step + 1, max_steps, warmup, lr, min_lr)
 
1252
  for g in opt.param_groups:
1253
- g["lr"] = cur_lr
 
 
1254
  if scaler is not None:
1255
  scaler.step(opt)
1256
  scaler.update()
@@ -1273,13 +1807,22 @@ def main():
1273
  max_expert_share = round(max(avg_u), 3)
1274
  if max_expert_share > 0.60:
1275
  print(f"[warn] ROUTER IMBALANCE: max expert share {max_expert_share} > 0.60!", flush=True)
1276
- msg = {"step": step, "of": max_steps, "pct": round(pct, 2), "loss": round(acc_loss, 4),
1277
- "lr": cur_lr, "tok_per_sec": round(tps, 1),
1278
- "elapsed_sec": round(dt, 1), "eta_sec": round(eta, 1), "aux": aux}
 
 
 
 
 
 
1279
  if max_expert_share is not None:
1280
  msg["max_expert_share"] = max_expert_share
 
 
1281
  # Essential clean log line
1282
- print(f"[step {step:,}/{max_steps:,}] ({pct:.2f}%) | loss: {acc_loss:.4f} | lr: {cur_lr:.2e} | {tps:,.0f} tok/s | ETA: {fmt_hms(eta)}", flush=True)
 
1283
  if logf is not None:
1284
  logf.write(json.dumps(msg) + "\n")
1285
  logf.flush()
 
25
  from torch.utils.data import Dataset
26
 
27
  sys.path.insert(0, str(Path(__file__).parent))
28
+ try:
29
+ from viu_moe import ViuArgs, Viu1MoE, load_args as load_model_args
30
+ except ImportError:
31
+ from viu1_moe import ViuArgs, Viu1MoE, load_args as load_model_args
32
 
33
  try:
34
  import yaml
 
48
 
49
  # ---------------- data ----------------
50
  def get_lang_tag(name_or_text: str) -> str:
51
+ n = str(name_or_text).lower().strip()
52
+ # Explicit language codes and names
53
+ if n in ("en", "eng", "english") or "english" in n:
 
 
 
54
  return "<|english|>"
55
+ if n in ("hi", "hin", "hindi") or "hindi" in n:
56
+ return "<|hindi|>"
57
+ if n in ("hinglish", "hi-en", "en-hi", "hi_en", "bilingual") or "hinglish" in n:
58
+ return "<|hinglish|>"
59
+ # Simple heuristic for raw text: Devanagari -> hindi, else ascii -> english, else hinglish
60
  if any('\u0900' <= ch <= '\u097F' for ch in n):
61
  return "<|hindi|>"
62
+ if n.isascii():
63
+ return "<|english|>"
64
  return "<|hinglish|>"
65
 
66
 
 
87
  ids.append(eos)
88
  if len(ids) > warn_tokens:
89
  print(f"[warn] {len(ids)/1e6:.1f}M tokens RAM me — bade corpus ke liye memmap/HF-streaming use karo.")
90
+ self.raw_ids = torch.tensor(ids, dtype=torch.long)
91
+ self.seq_len = seq_len
92
+ self.rebuild_blocks()
93
+
94
+ def rebuild_blocks(self):
95
+ n = (len(self.raw_ids) // self.seq_len) * self.seq_len
96
+ self.blocks = self.raw_ids[:n].view(-1, self.seq_len) if n > 0 else torch.zeros((1, self.seq_len), dtype=torch.long)
97
+
98
+ def set_seq_len(self, new_seq_len):
99
+ if new_seq_len == self.seq_len:
100
+ return
101
+ self.seq_len = new_seq_len
102
+ self.rebuild_blocks()
103
 
104
  def __len__(self):
105
  return len(self.blocks)
 
118
  return files
119
 
120
 
121
+ # ---------------- curriculum & sched ----------------
122
+ def resolve_curriculum_stage(step, stages):
123
+ """Return (stage_idx, stage_dict) based on current step."""
124
+ if not stages:
125
+ return 0, {}
126
+ for idx, st in enumerate(stages):
127
+ m = st.get("max_step")
128
+ if m is None or step < m:
129
+ return idx, st
130
+ return len(stages) - 1, stages[-1]
131
+
132
+
133
  def fmt_hms(sec):
134
  sec = max(int(sec), 0)
135
  h, sec = divmod(sec, 3600)
 
144
  return min_lr + 0.5 * (lr - min_lr) * (1 + math.cos(math.pi * min(p, 1.0)))
145
 
146
 
147
+ # ---------------- Muon Optimizer (Keller Jordan / Kimi K2) ----------------
148
+ def zeropower_via_newtonschulz5(G, steps=5, eps=1e-7):
149
+ """
150
+ Newton-Schulz quintic iteration to approximate matrix polar decomposition.
151
+ Stabilizes and orthogonalizes momentum updates for 2D weight matrices.
152
+ """
153
+ assert len(G.shape) == 2
154
+ a, b, c = (3.4445, -4.7750, 2.0315)
155
+ X = G.bfloat16() if (G.is_cuda and torch.cuda.is_bf16_supported()) else G.float()
156
+ X = X / (X.norm() + eps)
157
+ transposed = False
158
+ if X.size(0) > X.size(1):
159
+ X = X.T
160
+ transposed = True
161
+ for _ in range(steps):
162
+ A = X @ X.T
163
+ B = b * A + c * A @ A
164
+ X = a * X + B @ X
165
+ if transposed:
166
+ X = X.T
167
+ return X.to(dtype=G.dtype)
168
+
169
+
170
+ class Muon(torch.optim.Optimizer):
171
+ """
172
+ Muon (MomentUm Orthogonalized by Newton-schulz) Optimizer.
173
+ Applies orthogonalized momentum updates to 2D internal weight matrices.
174
+ """
175
+ def __init__(self, params, lr=0.02, momentum=0.95, weight_decay=0.01, ns_steps=5):
176
+ defaults = dict(lr=lr, momentum=momentum, weight_decay=weight_decay, ns_steps=ns_steps)
177
+ super().__init__(params, defaults)
178
+
179
+ @torch.no_grad()
180
+ def step(self, closure=None):
181
+ loss = None
182
+ if closure is not None:
183
+ with torch.enable_grad():
184
+ loss = closure()
185
+
186
+ for group in self.param_groups:
187
+ lr = group["lr"]
188
+ momentum = group["momentum"]
189
+ wd = group["weight_decay"]
190
+ steps = group["ns_steps"]
191
+
192
+ for p in group["params"]:
193
+ if p.grad is None:
194
+ continue
195
+ g = p.grad
196
+ state = self.state[p]
197
+
198
+ # Initialize momentum buffer
199
+ if "momentum_buffer" not in state:
200
+ state["momentum_buffer"] = torch.zeros_like(g)
201
+
202
+ buf = state["momentum_buffer"]
203
+ buf.mul_(momentum).add_(g)
204
+
205
+ # Orthogonalize via Newton-Schulz
206
+ u = zeropower_via_newtonschulz5(buf, steps=steps)
207
+
208
+ # Scale update based on aspect ratio
209
+ scale = max(1.0, math.sqrt(p.size(0) / p.size(1)))
210
+ u.mul_(scale)
211
+
212
+ # Weight decay
213
+ if wd > 0:
214
+ p.mul_(1.0 - lr * wd)
215
+
216
+ # Apply update
217
+ p.add_(u, alpha=-lr)
218
+
219
+ return loss
220
+
221
+
222
+ class CombinedOptimizer:
223
+ """Combines Muon (for 2D linear weights) and AdamW (for 1D norms, embeddings, routers)."""
224
+ def __init__(self, optimizers):
225
+ self.optimizers = optimizers
226
+ self.param_groups = []
227
+ for opt in optimizers:
228
+ self.param_groups.extend(opt.param_groups)
229
+
230
+ def step(self, closure=None):
231
+ loss = None
232
+ for opt in self.optimizers:
233
+ res = opt.step(closure=closure)
234
+ if res is not None:
235
+ loss = res
236
+ return loss
237
+
238
+ def zero_grad(self, set_to_none=True):
239
+ for opt in self.optimizers:
240
+ opt.zero_grad(set_to_none=set_to_none)
241
+
242
+ @property
243
+ def state(self):
244
+ # Merged param->state view so torch AMP GradScaler.unscale_() works
245
+ # (it reads optimizer.state[p]). FP16-only path; BF16 configs never touch it.
246
+ merged = {}
247
+ for opt in self.optimizers:
248
+ try:
249
+ merged.update(opt.state)
250
+ except Exception:
251
+ pass
252
+ return merged
253
+
254
+ def state_dict(self):
255
+ return {"opts": [opt.state_dict() for opt in self.optimizers]}
256
+
257
+ def load_state_dict(self, state_dict):
258
+ if isinstance(state_dict, dict) and "opts" in state_dict:
259
+ for opt, sd in zip(self.optimizers, state_dict["opts"]):
260
+ opt.load_state_dict(sd)
261
+ elif len(self.optimizers) > 1:
262
+ try:
263
+ self.optimizers[1].load_state_dict(state_dict)
264
+ except Exception:
265
+ pass
266
+
267
+
268
+ def split_muon_params(model):
269
+ """
270
+ Separates parameters into:
271
+ - Muon: 2D internal linear weights in MLA attention and MoE experts
272
+ - AdamW: Embeddings, RMSNorms, routers, biases, output heads
273
+ """
274
+ muon_params = []
275
+ adamw_params = []
276
+
277
+ for name, p in model.named_parameters():
278
+ if not p.requires_grad:
279
+ continue
280
+ # 1D params (norms, biases) always AdamW
281
+ if p.ndim < 2:
282
+ adamw_params.append(p)
283
+ # Embeddings, router heads, and unembedding heads use AdamW
284
+ elif "tok_emb" in name or "head" in name or "router" in name:
285
+ adamw_params.append(p)
286
+ else:
287
+ muon_params.append(p)
288
+
289
+ return muon_params, adamw_params
290
+
291
+
292
+
293
  def pick_device(pref=None):
294
  """Auto GPU: sabse zyada free VRAM wala cuda, warna cpu. --device se override."""
295
  if pref:
 
346
  + fragmentation ka explicit reserve chahiye, warna wrap ke baad training step-1 pe OOM."""
347
  model.train()
348
  best = 1
349
+ # Reserve: Muon fp32 momentum (~1.7B 2D params ~= 6.8GB) + AdamW8bit states +
350
+ # DDP grad buckets + fragmentation. 1.856B model => try 8GB, warna 4GB, warna 2GB.
351
  adamw_buf = None
352
  if device.type == "cuda":
353
+ for n_elems in (2_000_000_000, 1_000_000_000, 500_000_000): # ~8.0 / 4.0 / 2.0 GB
354
  try:
355
  adamw_buf = torch.empty((n_elems,), dtype=torch.float32, device=device)
356
  break
 
384
  return best
385
 
386
 
387
+ def maybe_apply_fp8(model, want, device, R0=True):
388
+ """EXPERIMENTAL 2025-tech (Blackwell/5090 only): torchao Float8 GEMM training.
389
+ Default OFF. Optimizer (Muon/AdamW) original-dtype params dekhta hai, isliye
390
+ compatible hai. Koi bhi condition miss -> loud BF16 fallback, silent nahi."""
391
+ if not want:
392
+ return model, "fp8=off"
393
+ if device.type != "cuda":
394
+ if R0:
395
+ print("[fp8] CPU box — BF16 fallback (FP8 needs Blackwell CUDA)")
396
+ return model, "fp8=off(cpu)"
397
+ try:
398
+ major, _ = torch.cuda.get_device_capability(device)
399
+ except Exception:
400
+ major = 0
401
+ if major < 9:
402
+ if R0:
403
+ print("[fp8] pre-Blackwell GPU (sm<90) — BF16 fallback (FP8 needs sm_90+)")
404
+ return model, "fp8=off(arch)"
405
+ try:
406
+ from torchao.float8 import convert_to_float8_training
407
+ except ImportError:
408
+ if R0:
409
+ print("[fp8] torchao missing (pip install torchao --index-url https://download.pytorch.org/whl/nightly/cu128) — BF16 fallback")
410
+ return model, "fp8=off(no-torchao)"
411
+ try:
412
+ convert_to_float8_training(model)
413
+ if R0:
414
+ print("[fp8] Float8 training ENABLED via torchao (EXPERIMENTAL — pehle 500 steps loss-watch karo)")
415
+ return model, "fp8=on"
416
+ except Exception as e:
417
+ if R0:
418
+ print(f"[fp8] convert failed ({str(e)[:120]}) — BF16 fallback")
419
+ return model, "fp8=off(err)"
420
+
421
+
422
+ def maybe_apply_compile(model, want, mode, is_smoke, R0=True, device=None, vocab_size=48000):
423
+ """torch.compile (2024-25 mature): 10-30% free speedup. Smoke me skip (fast test).
424
+ DDP-wrap SE PEHLE lagta hai taaki ckpt keys prefix-free rahein.
425
+ Fail-FAST: chhota warmup forward backend verify karta hai (missing triton/compiler
426
+ pehle step pe crash ke bajaye turant eager fallback). Pehla compiled step slow
427
+ hota hai (one-time codegen, minutes) — ye normal hai."""
428
+ if not want:
429
+ return model, "compile=off"
430
+ if is_smoke:
431
+ if R0:
432
+ print("[compile] smoke me skip (fast CPU test chahiye)")
433
+ return model, "compile=off(smoke)"
434
+ try:
435
+ model = torch.compile(model, mode=mode, fullgraph=False)
436
+ if device is not None and str(device) != "cpu":
437
+ with torch.no_grad():
438
+ _ids = torch.randint(0, vocab_size, (1, 16), device=device)
439
+ _ = model(_ids)
440
+ if R0:
441
+ print(f"[compile] torch.compile ON (mode={mode}, fullgraph=False, backend-verified, DDP-se-pehle)")
442
+ return model, f"compile=on({mode})"
443
+ except Exception as e:
444
+ if R0:
445
+ print(f"[compile] backend unavailable ({str(e)[:100]}) — eager fallback (training safe hai)")
446
+ try:
447
+ # compiled wrapper hata ke original module wapas (eager, safe)
448
+ inner = model._orig_mod if hasattr(model, "_orig_mod") else model
449
+ if inner is not model:
450
+ return inner, "compile=off(err-fallback)"
451
+ except Exception:
452
+ pass
453
+ return model, "compile=off(err)"
454
+
455
+
456
  def save_slim_ckpt(path, model, cfg=None):
457
+ """Inference & HF deployment ke liye slim checkpoint (~3.7GB BF16 for 1.856B, optimizer states ke bina)."""
458
  path = Path(path)
459
  path.parent.mkdir(parents=True, exist_ok=True)
460
  state = {k: v.to(dtype=torch.bfloat16).cpu() for k, v in model.state_dict().items()}
 
633
  # val PPL ke liye pure LM loss chahiye — MoE-aux + mt-loss hatao
634
  base = loss.item()
635
  try:
636
+ mt_w = float(aux.get("mt_loss_weight", 0.1)) # match viu_moe.py forward()
637
+ base = base - float(aux.get("moe_aux", 0.0)) - mt_w * float(aux.get("mt_loss", 0.0))
638
  except Exception:
639
  pass
640
  tot += base
 
667
  # Fragmentation kam karo (T4 pe large-alloc OOM ke baad reserved-but-unused dikhta tha)
668
  os.environ.setdefault("PYTORCH_ALLOC_CONF", "expandable_segments:True")
669
  ap = argparse.ArgumentParser()
670
+ ap.add_argument("--config", default="../configs/train_rtx5090.yaml")
671
+ ap.add_argument("--model_config", default="../configs/model_config.yaml")
672
  ap.add_argument("--smoke", action="store_true", help="CPU pe tiny model + 3 steps test")
673
  ap.add_argument("--steps", "--max_steps", dest="steps", type=int, default=None, help="Max training steps override")
674
  ap.add_argument("--resume", default=None, help="Checkpoint path to resume training from")
 
679
  ap.add_argument("--hf_split", default="train", help="Dataset split (default: train)")
680
  ap.add_argument("--batch_size", type=int, default=None, help="Micro-batch size override")
681
  ap.add_argument("--grad_accum", type=int, default=None, help="Gradient accumulation steps override")
682
+ ap.add_argument("--seq_len", type=int, default=None, help="Sequence length override (bypasses curriculum)")
683
+ ap.add_argument("--curriculum", dest="curriculum", action="store_true", default=None, help="Enable 3-stage sequence length curriculum (512 -> 1024 -> 2048)")
684
+ ap.add_argument("--no_curriculum", "--no-curriculum", dest="curriculum", action="store_false", help="Disable sequence length curriculum")
685
  ap.add_argument("--no_auto_tune", action="store_true", help="Disable auto-tune and enforce exact batch_size and grad_accum")
686
+ ap.add_argument("--optimizer", default=None, choices=["muon", "adamw", "paged_adamw_8bit", "8bit"], help="Optimizer choice (muon, adamw, paged_adamw_8bit)")
687
+ ap.add_argument("--muon_lr", type=float, default=None, help="Muon learning rate override (default: 0.02)")
688
+ ap.add_argument("--muon_weight_decay", type=float, default=None, help="Muon weight decay override (default: 0.01)")
689
+ ap.add_argument("--compile", action="store_true", help="torch.compile on (10-30%% free speedup, default off)")
690
+ ap.add_argument("--fp8", action="store_true", help="torchao Float8 training (Blackwell/5090 only, EXPERIMENTAL)")
691
  args = ap.parse_args()
692
 
693
  base = Path(__file__).parent
 
722
  if args.smoke:
723
  print("[mode] SMOKE — tiny model, CPU, 3 steps")
724
  margs = ViuArgs(dim=64, n_layers=2, n_heads=4, n_kv_heads=2, vocab_size=512,
725
+ max_seq_len=64, sliding_window=32, moe_every=2,
726
+ moe_num_experts=4, moe_num_routed_experts=4, moe_shared_experts=1,
727
  multi_token=False, switch_gate=False)
728
  seq_len, batch, max_steps = 32, 2, 3
729
  data_dir, tok_path = base / "../../data/raw", None
 
734
  device = torch.device("cpu")
735
  use_bf16 = False
736
  use_fp16 = False
737
+ use_curriculum = False
738
+ curriculum_stages = None
739
+ curr_stage_idx = 0
740
  else:
741
  margs = load_model_args(args.model_config if Path(args.model_config).exists()
742
  else str(base / args.model_config))
743
+ # Sequence Length Curriculum Setup (512 -> 1024 -> 2048)
744
+ use_curriculum = bool(C("curriculum", False))
745
+ if args.curriculum is not None:
746
+ use_curriculum = bool(args.curriculum)
747
+ if args.seq_len is not None:
748
+ use_curriculum = False # explicit --seq_len overrides curriculum
749
+
750
+ curriculum_stages = C("curriculum_stages", [
751
+ {"max_step": 50000, "seq_len": 512, "batch_size": 8, "grad_accum": 64},
752
+ {"max_step": 200000, "seq_len": 1024, "batch_size": 8, "grad_accum": 32},
753
+ {"max_step": None, "seq_len": 2048, "batch_size": 4, "grad_accum": 32},
754
+ ]) if use_curriculum else None
755
+
756
+ curr_stage_idx = 0
757
+ if use_curriculum:
758
+ curr_stage_idx, curr_stage = resolve_curriculum_stage(0, curriculum_stages)
759
+ seq_len = int(curr_stage["seq_len"])
760
+ batch = int(args.batch_size or curr_stage["batch_size"])
761
+ accum = int(args.grad_accum or curr_stage["grad_accum"])
762
+ if R0:
763
+ print(f"[curriculum] Initialized Stage 1/{len(curriculum_stages)}: "
764
+ f"seq_len={seq_len} batch={batch} accum={accum} (target {batch*accum*seq_len:,} tok/step)")
765
+ else:
766
+ seq_len = int(args.seq_len or C("seq_len", 1024))
767
+ batch = int(args.batch_size or C("batch_size", 2))
768
+ accum = int(args.grad_accum or C("grad_accum", 8))
769
+ _cfg_steps = C("max_steps", None)
770
+ max_steps = args.steps or (int(_cfg_steps) if _cfg_steps not in ("auto", "none", "null", None) else None)
771
  data_dir = resolve(C("data_dir", "../../data/raw"))
772
  tok_path = resolve(C("tokenizer_path", "../../tokenizer/outputs/tokenizer.json"))
773
  lr, min_lr, warmup, wd = C("lr", 3e-4), C("min_lr", 3e-5), C("warmup_steps", 2500), C("weight_decay", 0.1)
774
+ clip = float(C("grad_clip", 1.0))
775
  eval_every, save_every, log_every = C("eval_every", 1000), C("save_every", 5000), C("log_every", 50)
776
  sample_every = C("sample_every", 1000)
777
  ckpt_dir = resolve(C("ckpt_dir", "../checkpoints"))
778
  log_dir = resolve(C("log_dir", "../../logs"))
779
+ hf_repo = C("hf_repo", "ViuAI/Viu-1.5B-MoE")
780
  hf_token = os.environ.get("HF_TOKEN")
781
  device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
782
  if args.device:
 
837
  import shutil
838
  try:
839
  real_tok = hf_hub_download(
840
+ repo_id=hf_repo or "ViuAI/Viu-1.5B-MoE",
841
  filename="tokenizer/outputs/tokenizer.json",
842
  token=hf_token
843
  )
 
882
  print(f"[hf] streaming dataset {hf_dataset} (native PyArrow streaming)")
883
  # Non-5-col schema folders (AUDIT A3) — inko stream me mat lo, warna
884
  # shuffle/interleave par CastError (wikipedia: title/text/url vs text/lang/...).
885
+ # Note: distilled/, science/, and translation/ (Samanantar) handled in _stream_parquet_rows.
886
+ # Exclude only truly unformatted raw directories (already packaged into domains/)
887
+ # and empty/non-parquet files. All 1,874 valid Parquet files stream natively.
888
  # ZERO EXCLUSIONS: Stream 100% of all files in repository
889
  NON_UNIFIED_PREFIXES = ()
890
  # Corpus-slice rotation (Option A: full-data coverage round-by-round).
 
940
 
941
  def _stream_parquet_rows(files, repo_id):
942
  for fp in files:
943
+ p = f"datasets/{repo_id}/{fp}"
944
+ for _attempt in range(3):
945
+ try:
946
+ with _fs.open(p, "rb") as f:
947
+ pf = _pq.ParquetFile(f)
948
+ _names = pf.schema.names
949
+ if "toxicity" in _names and "text" in _names:
950
+ # Jigsaw civil_comments: score columns -> safety_tag (label preserved as tag, not text)
951
+ cols = ["text", "toxicity", "severe_toxicity", "obscene", "threat", "insult", "identity_attack", "sexual_explicit"]
952
+ cols = [c for c in cols if c in _names]
953
+ for batch in pf.iter_batches(batch_size=2048, columns=cols):
954
+ d = batch.to_pydict()
955
+ n = len(d.get("text", []))
956
+ for i in range(n):
957
+ tw = str(d["text"][i])
958
+ if len(tw.strip()) <= 5:
959
+ continue
960
+ mx = 0.0
961
+ for sc in cols[1:]:
 
 
 
962
  try:
963
+ v = float(d[sc][i])
964
+ except Exception:
965
  v = 0.0
966
  if v > mx:
967
  mx = v
968
+ tag = "toxic" if mx >= 0.5 else "safe"
969
+ yield {"text": tw, "lang": "en", "source": "jigsaw_civil_comments", "domain": "toxicity", "safety_tag": tag}
970
+ elif "text" in _names:
971
+ cols = [c for c in ["text", "lang", "source", "domain", "safety_tag"] if c in pf.schema.names]
972
+ for batch in pf.iter_batches(batch_size=2048, columns=cols):
973
+ d = batch.to_pydict()
974
+ n = len(d.get("text", []))
975
+ for i in range(n):
976
+ yield {k: d[k][i] for k in cols}
977
+ elif "generated_full_text" in pf.schema.names:
978
+ for batch in pf.iter_batches(batch_size=2048, columns=["generated_full_text"]):
979
+ d = batch.to_pydict()
980
+ n = len(d.get("generated_full_text", []))
981
+ for i in range(n):
982
+ yield {"text": str(d["generated_full_text"][i]), "lang": "en", "source": "science_textbooks", "domain": "science", "safety_tag": "safe"}
983
+ elif "src" in pf.schema.names and "tgt" in pf.schema.names:
984
+ for batch in pf.iter_batches(batch_size=2048, columns=["src", "tgt"]):
985
+ d = batch.to_pydict()
986
+ n = len(d.get("src", []))
987
+ for i in range(n):
988
+ s = str(d["src"][i]).strip()
989
+ t = str(d["tgt"][i]).strip()
990
+ if len(s) > 10 and len(t) > 10:
991
+ if i % 2 == 0:
992
+ txt = f"English: {s}\nHindi: {t}"
993
+ else:
994
+ txt = f"Hindi: {t}\nEnglish: {s}"
995
+ yield {"text": txt, "lang": "bilingual", "source": "samanantar_translation", "domain": "translation", "safety_tag": "safe"}
996
+ elif "context" in pf.schema.names and "question" in pf.schema.names and "response" in pf.schema.names:
997
+ for batch in pf.iter_batches(batch_size=2048, columns=["context", "question", "response"]):
998
+ d = batch.to_pydict()
999
+ n = len(d.get("question", []))
1000
+ for i in range(n):
1001
+ ctx = str(d["context"][i]).strip()
1002
+ q = str(d["question"][i]).strip()
1003
+ ans = str(d["response"][i]).strip()
1004
+ parts = []
1005
+ if ctx and len(ctx) > 30:
1006
+ parts.append(f"### Case Background:\n{ctx}")
1007
+ parts.append(f"### Legal Query:\n{q}")
1008
+ parts.append(f"### Legal Determination:\n{ans}")
1009
+ yield {"text": "\n\n".join(parts), "lang": "en", "source": "indian_courts_case_law", "domain": "governance_law", "safety_tag": "safe"}
1010
+ elif "instruction" in pf.schema.names and "output" in pf.schema.names:
1011
+ cols = [c for c in ["instruction", "input", "output"] if c in pf.schema.names]
1012
+ for batch in pf.iter_batches(batch_size=2048, columns=cols):
1013
+ d = batch.to_pydict()
1014
+ n = len(d["instruction"])
1015
+ for i in range(n):
1016
+ ins = str(d["instruction"][i]).strip()
1017
+ inp = str(d["input"][i]).strip() if "input" in d else ""
1018
+ ans = str(d["output"][i]).strip()
1019
+ if len(ins) > 5 and len(ans) > 5:
1020
+ if inp and len(inp) > 5:
1021
+ txt = f"### Instruction:\n{ins}\n\n### Input:\n{inp}\n\n### Response:\n{ans}"
1022
+ else:
1023
+ txt = f"### Instruction:\n{ins}\n\n### Response:\n{ans}"
1024
+ lg = "hinglish" if any('\u0900' <= ch <= '\u097F' for ch in txt[:500]) else "en"
1025
+ yield {"text": txt, "lang": lg, "source": "instruction_following", "domain": "instruction", "safety_tag": "safe"}
1026
+ elif "tweet" in pf.schema.names:
1027
+ for batch in pf.iter_batches(batch_size=2048, columns=["tweet"]):
1028
+ d = batch.to_pydict()
1029
+ n = len(d.get("tweet", []))
1030
+ for i in range(n):
1031
+ tw = str(d["tweet"][i]).strip()
1032
+ if len(tw) > 5:
1033
+ yield {"text": tw, "lang": "en", "source": "hate_speech_offensive", "domain": "toxicity", "safety_tag": "toxic"}
1034
+ elif "question" in pf.schema.names and "answer" in pf.schema.names:
1035
+ # Standalone Q/A pairs (math/problem sets without context column)
1036
+ for batch in pf.iter_batches(batch_size=2048, columns=["question", "answer"]):
1037
+ d = batch.to_pydict()
1038
+ n = len(d.get("question", []))
1039
+ for i in range(n):
1040
+ q = str(d["question"][i]).strip()
1041
+ a = str(d["answer"][i]).strip()
1042
+ if len(q) > 5 and len(a) > 5:
1043
+ yield {"text": f"### Problem:\n{q}\n\n### Solution:\n{a}", "lang": "en", "source": "math_qa_pairs", "domain": "reasoning_math", "safety_tag": "safe"}
1044
+ elif "Patient" in pf.schema.names and "Doctor" in pf.schema.names:
1045
+ cols = [c for c in ["Description", "Patient", "Doctor"] if c in pf.schema.names]
1046
+ for batch in pf.iter_batches(batch_size=2048, columns=cols):
1047
+ d = batch.to_pydict()
1048
+ n = len(d["Patient"])
1049
+ for i in range(n):
1050
+ desc = str(d["Description"][i]).strip() if "Description" in d else ""
1051
+ pat = str(d["Patient"][i]).strip()
1052
+ doc = str(d["Doctor"][i]).strip()
1053
+ if pat and doc:
1054
+ txt = f"### Medical Case:\n{desc}\n\n### Patient:\n{pat}\n\n### Doctor:\n{doc}" if desc else f"### Patient:\n{pat}\n\n### Doctor:\n{doc}"
1055
+ yield {"text": txt, "lang": "en", "source": "ai_medical_dialogues", "domain": "health", "safety_tag": "safe"}
1056
+ break # success — move to next file
1057
+ except GeneratorExit:
1058
+ return # caller closed the generator
1059
+ except Exception as e:
1060
+ if _attempt < 2:
1061
+ import time as _time
1062
+ _time.sleep(2 ** _attempt) # backoff: 1s, 2s
1063
+ else:
1064
+ if R0:
1065
+ print(f"[hf] WARNING: skipping {fp} after 3 retries: {e}")
1066
 
1067
  def _stream_shuffled(gen, buffer_size=20000, seed=42):
1068
  if buffer_size <= 0:
 
1105
  mix_max_rows = int(C("mix_max_rows", 2000000))
1106
  allow_toxic = safety_mode in ("research", "full")
1107
  allow_uncensored = safety_mode in ("full",)
1108
+ allow_erotica = safety_mode in ("full",)
1109
  if R0:
1110
+ print(f"[hf] safety={safety_mode} (toxic={allow_toxic}, uncensored={allow_uncensored}, erotica={allow_erotica}) "
1111
  f"max_blocks={max_blocks_cfg} max_rows={max_rows_cfg or 'unlimited'} "
1112
  f"enforce_mix={enforce_mix} lang_mix={lang_mix if enforce_mix else 'OFF (shuffle-buffer order)'}"
1113
  + (f" natural_shuffle={natural_shuffle}" if (not enforce_mix and natural_shuffle > 0) else "")
 
1125
  return allow_toxic
1126
  if tag == "uncensored":
1127
  return allow_uncensored
1128
+ if tag in ("sexual_explicit", "erotica", "erotic"):
1129
+ return allow_erotica
1130
  s = f"{r.get('source','')} {r.get('domain','')}".lower()
1131
  if "uncensor" in s or "unfilter" in s or "vortex" in s:
1132
  return allow_uncensored
1133
+ if "erotic" in s:
1134
+ return allow_erotica
1135
  if any(k in s for k in ("toxic", "hate", "offensive", "profan", "civil_comments", "hasoc", "prism")):
1136
  return allow_toxic
1137
  return True
 
1139
  def __init__(self, stream, tok, seq_len, max_blocks=50000, max_rows=None,
1140
  enforce_mix=False, lang_mix=None, mix_seed=42, mix_scan_mult=5,
1141
  mix_max_rows=2000000):
1142
+ self.stream = stream
1143
+ self.tok = tok
1144
+ self.seq_len = seq_len
1145
+ self.max_blocks = max_blocks
1146
+ self.max_rows = max_rows
1147
+ self.enforce_mix = enforce_mix
1148
+ self.lang_mix = lang_mix
1149
+ self.mix_seed = mix_seed
1150
+ self.mix_scan_mult = mix_scan_mult
1151
+ self.mix_max_rows = mix_max_rows
1152
+ self.blocks = torch.zeros((0, seq_len), dtype=torch.long)
1153
+ self.stream_exhausted = False
1154
+ self.refill_count = 0
1155
+ self._remainder = {"hinglish": [], "hindi": [], "english": []} # carry-over tokens across refill()
1156
+ self.refill()
1157
+
1158
+ def set_seq_len(self, new_seq_len):
1159
+ """Dynamically adjust block size for curriculum training.
1160
+ Re-chunks unconsumed tokens and remainder buffer without loss."""
1161
+ if new_seq_len == self.seq_len:
1162
+ return
1163
+ self.seq_len = new_seq_len
1164
+ # Collect unconsumed blocks back into remainder buffer
1165
+ if hasattr(self, "blocks") and len(self.blocks) > 0:
1166
+ unconsumed = self.blocks.flatten().tolist()
1167
+ self._remainder["english"].extend(unconsumed)
1168
+ self.blocks = torch.zeros((0, self.seq_len), dtype=torch.long)
1169
+ self.refill()
1170
+
1171
+ def refill(self):
1172
+ if self.stream_exhausted:
1173
+ return 0
1174
+ self.refill_count += 1
1175
+ buckets = {"hinglish": list(self._remainder.get("hinglish", [])),
1176
+ "hindi": list(self._remainder.get("hindi", [])),
1177
+ "english": list(self._remainder.get("english", []))}
1178
+ eos = self.tok.token_to_id("<eos>")
1179
  if eos is None:
1180
  eos = 2
1181
+ cap_tok = self.max_blocks * self.seq_len
1182
+ weights = {k: float((self.lang_mix or {}).get(k, 0)) for k in buckets}
1183
  if sum(weights.values()) <= 0:
1184
  weights = {"hinglish": 0.4, "hindi": 0.3, "english": 0.3}
1185
  tot_w = sum(weights.values())
1186
  quota = {k: cap_tok * w / tot_w for k, w in weights.items()}
1187
+ scan_cap = cap_tok * (self.mix_scan_mult if self.enforce_mix else 1)
1188
  n_rows = n_kept = n_safe_skip = n_empty = n_noschema = 0
1189
  lang_rows = {"hinglish": 0, "hindi": 0, "english": 0}
1190
+ rows_found = False
1191
+ for row in self.stream:
1192
+ rows_found = True
1193
  n_rows += 1
1194
+ if self.max_rows and n_rows > self.max_rows:
1195
  break
1196
  txt = row.get("text", "") if isinstance(row, dict) else ""
1197
  if not txt or not str(txt).strip():
1198
  n_empty += 1
1199
  continue
1200
+ txt = str(txt).strip()
1201
+ # Auditor filter: skip degenerate colon strings (e.g. NCERT colon scrapes ': : : : :')
1202
+ if len(txt) < 5 or (len(txt) < 40 and txt.count(':') >= 3 and txt.count(':') > len(txt) / 3):
1203
+ n_empty += 1
1204
+ continue
1205
  if not _safety_ok(row if isinstance(row, dict) else {}):
1206
  n_safe_skip += 1
1207
  continue
 
1208
  try:
1209
  tag = get_lang_tag(row.get("lang") or row.get("source") or txt)
1210
  except Exception:
1211
  n_noschema += 1
1212
  continue
1213
  bucket = "hinglish" if "hinglish" in tag else ("hindi" if "hindi" in tag else "english")
1214
+ if self.enforce_mix and len(buckets[bucket]) >= quota[bucket]:
 
1215
  n_kept += 1
1216
  if all(len(buckets[k]) >= quota[k] for k in buckets):
1217
  break
1218
+ if n_rows >= (self.max_rows or self.mix_max_rows):
1219
  break
1220
  continue
1221
+ tag_id = self.tok.token_to_id(tag) if self.tok is not None else None
1222
  if tag_id is not None:
1223
  buckets[bucket].append(tag_id)
1224
+ buckets[bucket].extend(self.tok.encode(txt).ids)
1225
  buckets[bucket].append(eos)
1226
  lang_rows[bucket] += 1
1227
  n_kept += 1
1228
  total_tok = sum(len(v) for v in buckets.values())
1229
+ if self.enforce_mix:
 
1230
  if all(len(buckets[k]) >= quota[k] for k in buckets):
1231
  break
1232
+ if n_rows >= (self.max_rows or self.mix_max_rows):
1233
  break
1234
  if total_tok >= scan_cap:
1235
  break
1236
  elif total_tok >= cap_tok:
1237
  break
1238
+
1239
+ total_read = sum(len(v) for v in buckets.values())
1240
+ if not rows_found or total_read == 0:
1241
+ self.stream_exhausted = True
1242
+ return 0
1243
+
1244
  if R0:
1245
+ print(f"[hf] chunk {self.refill_count}: scanned={n_rows:,} kept={n_kept:,} mix_rows={lang_rows} "
1246
  f"skip_empty={n_empty:,} skip_safety({safety_mode})={n_safe_skip:,} skip_schema={n_noschema:,}")
1247
  import random as _rng
1248
+ if self.enforce_mix:
 
1249
  per_lang_blocks = {}
1250
  for k, v in buckets.items():
1251
  arr = torch.tensor(v, dtype=torch.long)
1252
+ n = (len(arr) // self.seq_len) * self.seq_len
1253
+ self._remainder[k] = arr[n:].tolist() # carry over tail tokens
1254
+ per_lang_blocks[k] = arr[:n].view(-1, self.seq_len) if n > 0 else torch.zeros((0, self.seq_len), dtype=torch.long)
1255
  avail = {k: len(v) for k, v in per_lang_blocks.items()}
1256
+ quota_b = {k: int(quota[k] // self.seq_len) for k in avail}
1257
  take = {k: min(avail[k], quota_b[k]) for k in avail}
1258
+ short = self.max_blocks - sum(take.values())
1259
+ if short > 0:
1260
  left = {k: avail[k] - take[k] for k in avail}
1261
  tot_left = sum(left.values()) or 1
1262
  for k in avail:
 
1264
  take[k] += extra
1265
  chosen = [per_lang_blocks[k][:take[k]] for k in avail if take[k] > 0]
1266
  if not chosen:
1267
+ self.blocks = torch.zeros((1, self.seq_len), dtype=torch.long)
1268
  else:
1269
  allb = torch.cat(chosen, dim=0)
1270
  idx = list(range(len(allb)))
1271
+ _rng.Random(self.mix_seed + self.refill_count).shuffle(idx)
1272
  self.blocks = allb[idx]
1273
  if R0:
1274
  print(f"[hf] mix ENFORCED target={weights}: blocks_taken={take} total={len(self.blocks)} (avail={avail})")
1275
  else:
1276
+ # Save per-lang remainders before merging
1277
+ for k in ["hinglish", "hindi", "english"]:
1278
+ arr_k = torch.tensor(buckets[k], dtype=torch.long)
1279
+ n_k = (len(arr_k) // self.seq_len) * self.seq_len
1280
+ self._remainder[k] = arr_k[n_k:].tolist()
1281
  ids = buckets["hinglish"] + buckets["hindi"] + buckets["english"]
1282
  if R0:
1283
+ print(f"[hf] mix NATURAL (chunk {self.refill_count}): prefetched {len(ids):,} tokens -> {len(ids)//max(self.seq_len,1):,} blocks")
1284
  ids = torch.tensor(ids, dtype=torch.long)
1285
+ n = (len(ids) // self.seq_len) * self.seq_len
1286
+ self.blocks = ids[:n].view(-1, self.seq_len) if n > 0 else torch.zeros((1, self.seq_len), dtype=torch.long)
1287
+ return len(self.blocks)
1288
+
1289
+ def __len__(self):
1290
+ return len(self.blocks)
1291
+
1292
+ def __getitem__(self, i):
1293
+ x = self.blocks[i]
1294
+ return x[:-1], x[1:]
1295
  ds = HFStreamDataset(hf_ds, tok, seq_len+1, max_blocks=max_blocks_cfg, max_rows=max_rows_cfg,
1296
  enforce_mix=enforce_mix, lang_mix=lang_mix, mix_seed=seed_r,
1297
  mix_scan_mult=mix_scan_mult, mix_max_rows=mix_max_rows)
 
1348
  ids.append(eos)
1349
  import torch as _torch
1350
  ids_t = _torch.tensor(ids, dtype=_torch.long)
 
 
1351
  class _DummyDS(_torch.utils.data.Dataset):
1352
+ def __init__(self, ids_t, seq_len):
1353
+ self.ids_t = ids_t
1354
+ self.seq_len = seq_len
1355
+ self.rebuild()
1356
+ def rebuild(self):
1357
+ n = (len(self.ids_t) // self.seq_len) * self.seq_len
1358
+ self.blocks = self.ids_t[:n].view(-1, self.seq_len) if n > 0 else _torch.zeros((1, self.seq_len), dtype=_torch.long)
1359
+ def set_seq_len(self, new_seq_len):
1360
+ if new_seq_len == self.seq_len:
1361
+ return
1362
+ self.seq_len = new_seq_len
1363
+ self.rebuild()
1364
  def __len__(self): return len(self.blocks)
1365
  def __getitem__(self, i):
1366
  x = self.blocks[i]; return x[:-1], x[1:]
1367
+ ds = _DummyDS(ids_t, seq_len + 1)
1368
  elif ds is not None:
1369
  pass # ds already built via HF streaming above
1370
  else:
 
1383
  if len(ds) > n_train:
1384
  train_ids = all_idx[:n_train]
1385
  val_ids = all_idx[n_train:]
1386
+ class _FrozenValDS(Dataset):
1387
+ def __init__(self, blocks): self.blocks = blocks
1388
+ def __len__(self): return len(self.blocks)
1389
+ def __getitem__(self, i):
1390
+ x = self.blocks[i]; return x[:-1], x[1:]
1391
+ val_ds = _FrozenValDS(ds.blocks[val_ids].clone())
1392
  else:
1393
  train_ids = all_idx
1394
  val_ds = None
 
1430
  + (" [dist broadcast r0->all]" if (use_dist and not args.smoke) else "")) if R0 else None
1431
  elif not args.smoke and device.type == "cuda" and R0:
1432
  print(f"[batch] MANUAL LOCK: micro={batch} accum={accum} eff={batch * accum} tok/step={batch * accum * seq_len * (world if use_dist else 1)} amp={'bf16' if use_bf16 else 'fp16'}")
1433
+ # 2026 speed stack (opt-in, DDP-wrap se pehle): FP8 phir compile
1434
+ if not args.smoke:
1435
+ model, _fp8info = maybe_apply_fp8(model, bool(args.fp8 or C("fp8", False)), device, R0)
1436
+ model, _cmpinfo = maybe_apply_compile(model, bool(args.compile or C("compile", False)),
1437
+ str(C("compile_mode", "reduce-overhead")), args.smoke, R0,
1438
+ device=device, vocab_size=margs.vocab_size)
1439
  if use_dist and not args.smoke:
1440
  # MoE top-2 routing me kuch experts kabhi-kabhi empty rehte hain -> find_unused_parameters=True (safe)
1441
  model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[device.index], find_unused_parameters=True)
 
1446
  if R0:
1447
  print(f"[model] total={total/1e6:.1f}M active={active/1e6:.1f}M device={device}" + (f" (DDP x{world})" if use_dist else ""))
1448
 
1449
+ tok_per_step = batch * accum * seq_len * (world if use_dist else 1)
1450
+ if (max_steps is None or max_steps <= 0) and not args.smoke:
1451
+ total_tokens = int(C("total_tokens", 126_500_000_000))
1452
+ max_steps = max(1, total_tokens // tok_per_step)
1453
+ if R0:
1454
+ print(f"[auto] max_steps dynamically computed from dataset ({total_tokens/1e9:.1f}B tokens / {tok_per_step:,} tok/step): {max_steps:,} steps")
1455
+
1456
  decay_params = [p for p in model.parameters() if p.requires_grad and p.ndim >= 2]
1457
  no_decay_params = [p for p in model.parameters() if p.requires_grad and p.ndim < 2]
1458
  optim_groups = [
1459
  {"params": decay_params, "weight_decay": wd},
1460
  {"params": no_decay_params, "weight_decay": 0.0},
1461
  ]
1462
+
1463
+ if bool(C("grad_ckpt", False)):
1464
+ target = model.module if hasattr(model, "module") else model
1465
+ if hasattr(target, "gradient_checkpointing_enable"):
1466
+ target.gradient_checkpointing_enable()
1467
+ if R0:
1468
+ print("[model] gradient checkpointing enabled (VRAM activation memory halved for 48L)")
1469
+ elif R0:
1470
+ print("[model] WARN: grad_ckpt requested but model has no gradient_checkpointing_enable()")
1471
+
1472
+ opt_name = str(args.optimizer or C("optimizer", "adamw")).lower()
1473
+ if opt_name == "muon":
1474
+ muon_p, adamw_p = split_muon_params(model)
1475
+ muon_lr = float(args.muon_lr if getattr(args, "muon_lr", None) is not None else C("muon_lr", 0.02))
1476
+ muon_wd = float(args.muon_weight_decay if getattr(args, "muon_weight_decay", None) is not None else C("muon_weight_decay", 0.01))
1477
+ opt_muon = Muon(muon_p, lr=muon_lr, momentum=float(C("muon_momentum", 0.95)), weight_decay=muon_wd)
1478
+ adamw_groups = [
1479
+ {"params": [p for p in adamw_p if p.ndim >= 2], "weight_decay": wd},
1480
+ {"params": [p for p in adamw_p if p.ndim < 2], "weight_decay": 0.0},
1481
+ ]
1482
+ if device.type == "cuda":
1483
+ try:
1484
+ import bitsandbytes as bnb
1485
+ opt_adamw = bnb.optim.PagedAdamW8bit(adamw_groups, lr=lr, betas=tuple(C("betas", [0.9, 0.95])))
1486
+ if R0:
1487
+ print(f"[opt] Muon ({len(muon_p)} 2D weights, lr={muon_lr}, wd={muon_wd}) + PagedAdamW8bit ({len(adamw_p)} 1D/emb/router, lr={lr}, wd={wd}) hybrid loaded!")
1488
+ except ImportError:
1489
+ opt_adamw = torch.optim.AdamW(adamw_groups, lr=lr, betas=tuple(C("betas", [0.9, 0.95])))
1490
+ if R0:
1491
+ print(f"[opt] Muon ({len(muon_p)} 2D weights, lr={muon_lr}, wd={muon_wd}) + AdamW ({len(adamw_p)} 1D/emb/router, lr={lr}, wd={wd}) hybrid loaded!")
1492
+ else:
1493
+ opt_adamw = torch.optim.AdamW(adamw_groups, lr=lr, betas=tuple(C("betas", [0.9, 0.95])))
1494
+ if R0:
1495
+ print(f"[opt] Muon ({len(muon_p)} 2D weights, lr={muon_lr}, wd={muon_wd}) + AdamW ({len(adamw_p)} 1D/emb/router, lr={lr}, wd={wd}) hybrid loaded (CPU)!")
1496
+ opt = CombinedOptimizer([opt_muon, opt_adamw])
1497
+ elif opt_name in ("8bit", "paged_adamw_8bit", "adamw8bit") and device.type == "cuda":
1498
+ try:
1499
+ import bitsandbytes as bnb
1500
+ opt = bnb.optim.PagedAdamW8bit(
1501
+ optim_groups, lr=lr, betas=tuple(C("betas", [0.9, 0.95])) if not args.smoke else (0.9, 0.95)
1502
+ )
1503
+ if R0:
1504
+ print("[opt] loaded bitsandbytes PagedAdamW8bit (VRAM memory optimized for 1.5B)")
1505
+ except ImportError:
1506
+ if R0:
1507
+ print("[opt] bitsandbytes not found, falling back to standard AdamW")
1508
+ opt = torch.optim.AdamW(optim_groups, lr=lr, betas=tuple(C("betas", [0.9, 0.95])) if not args.smoke else (0.9, 0.95))
1509
+ else:
1510
+ opt = torch.optim.AdamW(optim_groups, lr=lr, betas=tuple(C("betas", [0.9, 0.95])) if not args.smoke else (0.9, 0.95))
1511
  use_amp = use_bf16 or use_fp16
1512
  amp_dtype = torch.bfloat16 if use_bf16 else torch.float16
1513
  scaler = torch.amp.GradScaler("cuda") if (use_fp16 and device.type == "cuda") else None
 
1587
  start = load_ckpt(resolved_ckpt, ckpt_model, opt, scaler=scaler)
1588
  if R0:
1589
  print(f"[resume] successfully resumed at step {start} from {resolved_ckpt}")
1590
+ if use_curriculum:
1591
+ new_stage_idx, new_stage = resolve_curriculum_stage(start, curriculum_stages)
1592
+ if new_stage_idx != curr_stage_idx:
1593
+ curr_stage_idx = new_stage_idx
1594
+ seq_len = int(new_stage["seq_len"])
1595
+ batch = int(args.batch_size or new_stage["batch_size"])
1596
+ accum = int(args.grad_accum or new_stage["grad_accum"])
1597
+ if use_dist and not args.smoke:
1598
+ accum = max(1, accum // world)
1599
+ if hasattr(ds, "set_seq_len"):
1600
+ ds.set_seq_len(seq_len + 1)
1601
+ train_ids = list(range(len(ds)))
1602
+ if use_dist and not args.smoke:
1603
+ train_ids = train_ids[rank::world]
1604
+ if not train_ids:
1605
+ train_ids = [0]
1606
+ rng.shuffle(train_ids)
1607
+ idx = 0
1608
+ tok_per_step = batch * accum * seq_len * (world if use_dist else 1)
1609
+ if R0:
1610
+ print(f"[curriculum] Resumed into Stage {curr_stage_idx + 1}/{len(curriculum_stages)}: "
1611
+ f"seq_len={seq_len} batch={batch} accum={accum} (tok/step={tok_per_step:,})", flush=True)
1612
  else:
1613
  if R0:
1614
  print(f"[resume] checkpoint '{resume}' not found locally or on HF Hub — starting fresh from step 0")
 
1636
  t0 = time.time()
1637
  step = start
1638
  idx = 0
1639
+ nan_streak = 0
1640
+ max_nan_streak = int(C("max_nan_streak", 25))
1641
+ usage_log_every = int(C("usage_log_every", 100))
1642
  tok_per_step = batch * accum * seq_len * (world if use_dist else 1) # DDP: global tokens/step
1643
  pbar = None # Industry standard: clean stdout lines (no carriage-return progress bar in web terminal)
 
1644
  while step < max_steps:
1645
+ # Check Automatic Curriculum Upgrade
1646
+ if use_curriculum:
1647
+ new_stage_idx, new_stage = resolve_curriculum_stage(step, curriculum_stages)
1648
+ if new_stage_idx != curr_stage_idx:
1649
+ old_seq = seq_len
1650
+ curr_stage_idx = new_stage_idx
1651
+ seq_len = int(new_stage["seq_len"])
1652
+ batch = int(args.batch_size or new_stage["batch_size"])
1653
+ accum = int(args.grad_accum or new_stage["grad_accum"])
1654
+ if use_dist and not args.smoke:
1655
+ accum = max(1, accum // world)
1656
+ tok_per_step = batch * accum * seq_len * (world if use_dist else 1)
1657
+
1658
+ if hasattr(ds, "set_seq_len"):
1659
+ ds.set_seq_len(seq_len + 1)
1660
+ train_ids = list(range(len(ds)))
1661
+ if use_dist and not args.smoke:
1662
+ train_ids = train_ids[rank::world]
1663
+ if not train_ids:
1664
+ train_ids = [0]
1665
+ rng.shuffle(train_ids)
1666
+ idx = 0
1667
+
1668
+ if R0:
1669
+ print("\n" + "=" * 80)
1670
+ print(f"[CURRICULUM UPGRADE] Step {step:,}: Upgraded to Stage {curr_stage_idx + 1}/{len(curriculum_stages)}!")
1671
+ print(f" * Context Length: {old_seq} -> {seq_len} tokens")
1672
+ print(f" * Micro Batch Size: {batch}")
1673
+ print(f" * Grad Accumulation: {accum}")
1674
+ print(f" * Global Tok/Step: {tok_per_step:,} (Constant Token Budget Maintained)")
1675
+ print("=" * 80 + "\n", flush=True)
1676
+
1677
  if idx >= len(train_ids):
1678
+ if hasattr(ds, "refill") and not args.smoke:
1679
+ refilled = ds.refill()
1680
+ if refilled > 0:
1681
+ train_ids = list(range(len(ds)))
1682
+ if use_dist and not args.smoke:
1683
+ train_ids = train_ids[rank::world]
1684
+ if not train_ids:
1685
+ train_ids = [0]
1686
+ rng.shuffle(train_ids)
1687
+ idx = 0
1688
+ if R0:
1689
+ print(f"[stream] Buffer dynamically refilled: loaded {len(ds):,} fresh blocks from 125B+ token stream (step {step:,})", flush=True)
1690
+ else:
1691
+ if getattr(ds, "stream_exhausted", False):
1692
+ if R0:
1693
+ print(f"[stream] All 2,109 dataset shards processed! Stream complete at step {step:,}. Exiting to save final checkpoint...", flush=True)
1694
+ break
1695
+ rng.shuffle(train_ids)
1696
+ idx = 0
1697
+ else:
1698
+ rng.shuffle(train_ids)
1699
+ idx = 0
1700
  # accumulate (batch_size fix: har micro-step me batch samples stack karo)
1701
  opt.zero_grad(set_to_none=True)
1702
  acc_loss = 0.0
 
1741
  layer_avg = [sum(acc_usages[s][l_idx][e] for s in range(len(acc_usages))) / len(acc_usages) for e in range(n_exp)]
1742
  avg_usages.append(layer_avg)
1743
  aux = {"moe_aux": round(acc_moe_aux, 4), "mt_loss": round(acc_mt_loss, 4), "usages": avg_usages}
1744
+ # NaN/Inf guard: poisoned step ko skip karo (optimizer state bachao).
1745
+ # DDP me decision GLOBAL hoti hai (kisi ek rank pe NaN => sab skip),
1746
+ # warna ranks ke params desync ho jayenge. Streak badhti jaye to
1747
+ # divergence maan ke emergency ckpt + sab ranks ek saath ruko.
1748
+ local_bad = not math.isfinite(acc_loss)
1749
+ do_skip = local_bad
1750
+ if use_dist and not args.smoke:
1751
+ try:
1752
+ import torch.distributed as _dist
1753
+ _t = torch.tensor([1 if local_bad else 0], device=device)
1754
+ _dist.all_reduce(_t, op=_dist.ReduceOp.MAX)
1755
+ do_skip = bool(_t.item())
1756
+ except Exception:
1757
+ pass
1758
+ if do_skip:
1759
+ nan_streak += 1
1760
+ opt.zero_grad(set_to_none=True)
1761
+ if R0:
1762
+ print(f"[nan-guard] step {step+1}: non-finite loss ({acc_loss}) — update skipped (streak {nan_streak}/{max_nan_streak})", flush=True)
1763
+ if logf is not None:
1764
+ logf.write(json.dumps({"step": step + 1, "skipped": True, "loss": str(acc_loss),
1765
+ "nan_streak": nan_streak}) + "\n")
1766
+ logf.flush()
1767
+ if nan_streak >= max_nan_streak and not args.smoke:
1768
+ if R0:
1769
+ print(f"[nan-guard] DIVERGENCE: {nan_streak} consecutive NaN steps — emergency ckpt + abort", flush=True)
1770
+ save_ckpt(Path(ckpt_dir) / f"emergency_nan_{step+1}.pt", ckpt_model, opt, step, cfg, scaler=scaler)
1771
+ break
1772
+ step += 1
1773
+ continue
1774
+ nan_streak = 0
1775
  if scaler is not None:
1776
  scaler.unscale_(opt)
1777
+ total_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), clip)
1778
+ try:
1779
+ total_norm = float(total_norm)
1780
+ except Exception:
1781
+ total_norm = -1.0
1782
  cur_lr = cosine_lr(step + 1, max_steps, warmup, lr, min_lr)
1783
+ lr_ratio = cur_lr / max(lr, 1e-9)
1784
  for g in opt.param_groups:
1785
+ if "initial_lr" not in g:
1786
+ g["initial_lr"] = g["lr"]
1787
+ g["lr"] = g["initial_lr"] * lr_ratio
1788
  if scaler is not None:
1789
  scaler.step(opt)
1790
  scaler.update()
 
1807
  max_expert_share = round(max(avg_u), 3)
1808
  if max_expert_share > 0.60:
1809
  print(f"[warn] ROUTER IMBALANCE: max expert share {max_expert_share} > 0.60!", flush=True)
1810
+ # Full 48x28 usage matrix sirf kabhi-kabhi log karo (log-bloat bachao);
1811
+ # har line me max_expert_share + grad_norm kaafi hai.
1812
+ aux_log = dict(aux) if isinstance(aux, dict) else aux
1813
+ if isinstance(aux_log, dict) and step % usage_log_every != 0 and step > 5 and step != max_steps:
1814
+ aux_log = dict(aux_log)
1815
+ aux_log["usages"] = []
1816
+ msg = {"step": step, "of": max_steps, "pct": round(pct, 2), "seq_len": seq_len, "loss": round(acc_loss, 4),
1817
+ "lr": cur_lr, "grad_norm": round(total_norm, 3), "tok_per_sec": round(tps, 1),
1818
+ "elapsed_sec": round(dt, 1), "eta_sec": round(eta, 1), "aux": aux_log}
1819
  if max_expert_share is not None:
1820
  msg["max_expert_share"] = max_expert_share
1821
+ if use_curriculum:
1822
+ msg["curriculum_stage"] = curr_stage_idx + 1
1823
  # Essential clean log line
1824
+ cur_tag = f"seq: {seq_len} (S{curr_stage_idx+1})" if use_curriculum else f"seq: {seq_len}"
1825
+ print(f"[step {step:,}/{max_steps:,}] ({pct:.2f}%) | {cur_tag} | loss: {acc_loss:.4f} | gn: {total_norm:.2f} | lr: {cur_lr:.2e} | {tps:,.0f} tok/s | ETA: {fmt_hms(eta)}", flush=True)
1826
  if logf is not None:
1827
  logf.write(json.dumps(msg) + "\n")
1828
  logf.flush()