Sync model/scripts/train.py with 3-stage sequence length curriculum (512->1024->2048)
Browse files- model/scripts/train.py +734 -191
model/scripts/train.py
CHANGED
|
@@ -25,9 +25,10 @@ import torch.nn.functional as F
|
|
| 25 |
from torch.utils.data import Dataset
|
| 26 |
|
| 27 |
sys.path.insert(0, str(Path(__file__).parent))
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
|
|
|
| 31 |
|
| 32 |
try:
|
| 33 |
import yaml
|
|
@@ -47,16 +48,19 @@ except ImportError:
|
|
| 47 |
|
| 48 |
# ---------------- data ----------------
|
| 49 |
def get_lang_tag(name_or_text: str) -> str:
|
| 50 |
-
n = str(name_or_text).lower()
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
if "hindi" in n:
|
| 54 |
-
return "<|hindi|>"
|
| 55 |
-
if "english" in n:
|
| 56 |
return "<|english|>"
|
| 57 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 58 |
if any('\u0900' <= ch <= '\u097F' for ch in n):
|
| 59 |
return "<|hindi|>"
|
|
|
|
|
|
|
| 60 |
return "<|hinglish|>"
|
| 61 |
|
| 62 |
|
|
@@ -83,9 +87,19 @@ class TxtPackDataset(Dataset):
|
|
| 83 |
ids.append(eos)
|
| 84 |
if len(ids) > warn_tokens:
|
| 85 |
print(f"[warn] {len(ids)/1e6:.1f}M tokens RAM me — bade corpus ke liye memmap/HF-streaming use karo.")
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
self.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
|
| 90 |
def __len__(self):
|
| 91 |
return len(self.blocks)
|
|
@@ -104,7 +118,18 @@ def collect_txt(data_dir):
|
|
| 104 |
return files
|
| 105 |
|
| 106 |
|
| 107 |
-
# ---------------- sched ----------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 108 |
def fmt_hms(sec):
|
| 109 |
sec = max(int(sec), 0)
|
| 110 |
h, sec = divmod(sec, 3600)
|
|
@@ -119,6 +144,152 @@ def cosine_lr(step, max_steps, warmup, lr, min_lr):
|
|
| 119 |
return min_lr + 0.5 * (lr - min_lr) * (1 + math.cos(math.pi * min(p, 1.0)))
|
| 120 |
|
| 121 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
def pick_device(pref=None):
|
| 123 |
"""Auto GPU: sabse zyada free VRAM wala cuda, warna cpu. --device se override."""
|
| 124 |
if pref:
|
|
@@ -175,11 +346,11 @@ def probe_micro_batch(model, seq_len, vocab_size, device, amp_dtype):
|
|
| 175 |
+ fragmentation ka explicit reserve chahiye, warna wrap ke baad training step-1 pe OOM."""
|
| 176 |
model.train()
|
| 177 |
best = 1
|
| 178 |
-
# Reserve:
|
| 179 |
-
#
|
| 180 |
adamw_buf = None
|
| 181 |
if device.type == "cuda":
|
| 182 |
-
for n_elems in (
|
| 183 |
try:
|
| 184 |
adamw_buf = torch.empty((n_elems,), dtype=torch.float32, device=device)
|
| 185 |
break
|
|
@@ -213,8 +384,77 @@ def probe_micro_batch(model, seq_len, vocab_size, device, amp_dtype):
|
|
| 213 |
return best
|
| 214 |
|
| 215 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 216 |
def save_slim_ckpt(path, model, cfg=None):
|
| 217 |
-
"""Inference & HF deployment ke liye slim checkpoint (~
|
| 218 |
path = Path(path)
|
| 219 |
path.parent.mkdir(parents=True, exist_ok=True)
|
| 220 |
state = {k: v.to(dtype=torch.bfloat16).cpu() for k, v in model.state_dict().items()}
|
|
@@ -393,7 +633,8 @@ def evaluate(model, ds, device, max_batches=20, use_amp=False, amp_dtype=torch.f
|
|
| 393 |
# val PPL ke liye pure LM loss chahiye — MoE-aux + mt-loss hatao
|
| 394 |
base = loss.item()
|
| 395 |
try:
|
| 396 |
-
|
|
|
|
| 397 |
except Exception:
|
| 398 |
pass
|
| 399 |
tot += base
|
|
@@ -426,8 +667,8 @@ def main():
|
|
| 426 |
# Fragmentation kam karo (T4 pe large-alloc OOM ke baad reserved-but-unused dikhta tha)
|
| 427 |
os.environ.setdefault("PYTORCH_ALLOC_CONF", "expandable_segments:True")
|
| 428 |
ap = argparse.ArgumentParser()
|
| 429 |
-
ap.add_argument("--config", default="../configs/
|
| 430 |
-
ap.add_argument("--model_config", default="../configs/
|
| 431 |
ap.add_argument("--smoke", action="store_true", help="CPU pe tiny model + 3 steps test")
|
| 432 |
ap.add_argument("--steps", "--max_steps", dest="steps", type=int, default=None, help="Max training steps override")
|
| 433 |
ap.add_argument("--resume", default=None, help="Checkpoint path to resume training from")
|
|
@@ -438,7 +679,15 @@ def main():
|
|
| 438 |
ap.add_argument("--hf_split", default="train", help="Dataset split (default: train)")
|
| 439 |
ap.add_argument("--batch_size", type=int, default=None, help="Micro-batch size override")
|
| 440 |
ap.add_argument("--grad_accum", type=int, default=None, help="Gradient accumulation steps override")
|
|
|
|
|
|
|
|
|
|
| 441 |
ap.add_argument("--no_auto_tune", action="store_true", help="Disable auto-tune and enforce exact batch_size and grad_accum")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 442 |
args = ap.parse_args()
|
| 443 |
|
| 444 |
base = Path(__file__).parent
|
|
@@ -473,7 +722,8 @@ def main():
|
|
| 473 |
if args.smoke:
|
| 474 |
print("[mode] SMOKE — tiny model, CPU, 3 steps")
|
| 475 |
margs = ViuArgs(dim=64, n_layers=2, n_heads=4, n_kv_heads=2, vocab_size=512,
|
| 476 |
-
max_seq_len=64, sliding_window=32, moe_every=2,
|
|
|
|
| 477 |
multi_token=False, switch_gate=False)
|
| 478 |
seq_len, batch, max_steps = 32, 2, 3
|
| 479 |
data_dir, tok_path = base / "../../data/raw", None
|
|
@@ -484,21 +734,49 @@ def main():
|
|
| 484 |
device = torch.device("cpu")
|
| 485 |
use_bf16 = False
|
| 486 |
use_fp16 = False
|
|
|
|
|
|
|
|
|
|
| 487 |
else:
|
| 488 |
margs = load_model_args(args.model_config if Path(args.model_config).exists()
|
| 489 |
else str(base / args.model_config))
|
| 490 |
-
|
| 491 |
-
|
| 492 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 493 |
data_dir = resolve(C("data_dir", "../../data/raw"))
|
| 494 |
tok_path = resolve(C("tokenizer_path", "../../tokenizer/outputs/tokenizer.json"))
|
| 495 |
lr, min_lr, warmup, wd = C("lr", 3e-4), C("min_lr", 3e-5), C("warmup_steps", 2500), C("weight_decay", 0.1)
|
| 496 |
-
|
| 497 |
eval_every, save_every, log_every = C("eval_every", 1000), C("save_every", 5000), C("log_every", 50)
|
| 498 |
sample_every = C("sample_every", 1000)
|
| 499 |
ckpt_dir = resolve(C("ckpt_dir", "../checkpoints"))
|
| 500 |
log_dir = resolve(C("log_dir", "../../logs"))
|
| 501 |
-
hf_repo = C("hf_repo", "ViuAI/
|
| 502 |
hf_token = os.environ.get("HF_TOKEN")
|
| 503 |
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 504 |
if args.device:
|
|
@@ -559,7 +837,7 @@ def main():
|
|
| 559 |
import shutil
|
| 560 |
try:
|
| 561 |
real_tok = hf_hub_download(
|
| 562 |
-
repo_id=hf_repo or "ViuAI/
|
| 563 |
filename="tokenizer/outputs/tokenizer.json",
|
| 564 |
token=hf_token
|
| 565 |
)
|
|
@@ -604,7 +882,9 @@ def main():
|
|
| 604 |
print(f"[hf] streaming dataset {hf_dataset} (native PyArrow streaming)")
|
| 605 |
# Non-5-col schema folders (AUDIT A3) — inko stream me mat lo, warna
|
| 606 |
# shuffle/interleave par CastError (wikipedia: title/text/url vs text/lang/...).
|
| 607 |
-
#
|
|
|
|
|
|
|
| 608 |
# ZERO EXCLUSIONS: Stream 100% of all files in repository
|
| 609 |
NON_UNIFIED_PREFIXES = ()
|
| 610 |
# Corpus-slice rotation (Option A: full-data coverage round-by-round).
|
|
@@ -660,124 +940,129 @@ def main():
|
|
| 660 |
|
| 661 |
def _stream_parquet_rows(files, repo_id):
|
| 662 |
for fp in files:
|
| 663 |
-
|
| 664 |
-
|
| 665 |
-
|
| 666 |
-
|
| 667 |
-
|
| 668 |
-
|
| 669 |
-
|
| 670 |
-
|
| 671 |
-
|
| 672 |
-
|
| 673 |
-
|
| 674 |
-
|
| 675 |
-
|
| 676 |
-
|
| 677 |
-
|
| 678 |
-
|
| 679 |
-
|
| 680 |
-
|
| 681 |
-
|
| 682 |
-
mx = 0.0
|
| 683 |
-
for score_col in ("target", "toxic", "severe_toxicity", "insult"):
|
| 684 |
-
if score_col in d and d[score_col][i] is not None:
|
| 685 |
try:
|
| 686 |
-
v = float(d[
|
| 687 |
-
except
|
| 688 |
v = 0.0
|
| 689 |
if v > mx:
|
| 690 |
mx = v
|
| 691 |
-
|
| 692 |
-
|
| 693 |
-
|
| 694 |
-
|
| 695 |
-
|
| 696 |
-
|
| 697 |
-
|
| 698 |
-
|
| 699 |
-
|
| 700 |
-
|
| 701 |
-
|
| 702 |
-
|
| 703 |
-
|
| 704 |
-
|
| 705 |
-
|
| 706 |
-
|
| 707 |
-
|
| 708 |
-
|
| 709 |
-
|
| 710 |
-
|
| 711 |
-
|
| 712 |
-
|
| 713 |
-
|
| 714 |
-
|
| 715 |
-
|
| 716 |
-
|
| 717 |
-
|
| 718 |
-
|
| 719 |
-
|
| 720 |
-
|
| 721 |
-
|
| 722 |
-
|
| 723 |
-
|
| 724 |
-
|
| 725 |
-
|
| 726 |
-
|
| 727 |
-
|
| 728 |
-
|
| 729 |
-
|
| 730 |
-
|
| 731 |
-
|
| 732 |
-
|
| 733 |
-
|
| 734 |
-
|
| 735 |
-
|
| 736 |
-
|
| 737 |
-
|
| 738 |
-
|
| 739 |
-
|
| 740 |
-
|
| 741 |
-
|
| 742 |
-
|
| 743 |
-
|
| 744 |
-
|
| 745 |
-
|
| 746 |
-
|
| 747 |
-
|
| 748 |
-
|
| 749 |
-
|
| 750 |
-
|
| 751 |
-
|
| 752 |
-
|
| 753 |
-
|
| 754 |
-
|
| 755 |
-
|
| 756 |
-
|
| 757 |
-
|
| 758 |
-
|
| 759 |
-
|
| 760 |
-
|
| 761 |
-
|
| 762 |
-
|
| 763 |
-
|
| 764 |
-
|
| 765 |
-
|
| 766 |
-
|
| 767 |
-
|
| 768 |
-
|
| 769 |
-
|
| 770 |
-
|
| 771 |
-
|
| 772 |
-
|
| 773 |
-
|
| 774 |
-
|
| 775 |
-
|
| 776 |
-
|
| 777 |
-
|
| 778 |
-
|
| 779 |
-
|
| 780 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 781 |
|
| 782 |
def _stream_shuffled(gen, buffer_size=20000, seed=42):
|
| 783 |
if buffer_size <= 0:
|
|
@@ -820,8 +1105,9 @@ def main():
|
|
| 820 |
mix_max_rows = int(C("mix_max_rows", 2000000))
|
| 821 |
allow_toxic = safety_mode in ("research", "full")
|
| 822 |
allow_uncensored = safety_mode in ("full",)
|
|
|
|
| 823 |
if R0:
|
| 824 |
-
print(f"[hf] safety={safety_mode} (toxic={allow_toxic}, uncensored={allow_uncensored}) "
|
| 825 |
f"max_blocks={max_blocks_cfg} max_rows={max_rows_cfg or 'unlimited'} "
|
| 826 |
f"enforce_mix={enforce_mix} lang_mix={lang_mix if enforce_mix else 'OFF (shuffle-buffer order)'}"
|
| 827 |
+ (f" natural_shuffle={natural_shuffle}" if (not enforce_mix and natural_shuffle > 0) else "")
|
|
@@ -839,9 +1125,13 @@ def main():
|
|
| 839 |
return allow_toxic
|
| 840 |
if tag == "uncensored":
|
| 841 |
return allow_uncensored
|
|
|
|
|
|
|
| 842 |
s = f"{r.get('source','')} {r.get('domain','')}".lower()
|
| 843 |
if "uncensor" in s or "unfilter" in s or "vortex" in s:
|
| 844 |
return allow_uncensored
|
|
|
|
|
|
|
| 845 |
if any(k in s for k in ("toxic", "hate", "offensive", "profan", "civil_comments", "hasoc", "prism")):
|
| 846 |
return allow_toxic
|
| 847 |
return True
|
|
@@ -849,79 +1139,124 @@ def main():
|
|
| 849 |
def __init__(self, stream, tok, seq_len, max_blocks=50000, max_rows=None,
|
| 850 |
enforce_mix=False, lang_mix=None, mix_seed=42, mix_scan_mult=5,
|
| 851 |
mix_max_rows=2000000):
|
| 852 |
-
|
| 853 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 854 |
if eos is None:
|
| 855 |
eos = 2
|
| 856 |
-
cap_tok = max_blocks * seq_len
|
| 857 |
-
weights = {k: float((lang_mix or {}).get(k, 0)) for k in buckets}
|
| 858 |
if sum(weights.values()) <= 0:
|
| 859 |
weights = {"hinglish": 0.4, "hindi": 0.3, "english": 0.3}
|
| 860 |
tot_w = sum(weights.values())
|
| 861 |
quota = {k: cap_tok * w / tot_w for k, w in weights.items()}
|
| 862 |
-
scan_cap = cap_tok * (mix_scan_mult if enforce_mix else 1)
|
| 863 |
n_rows = n_kept = n_safe_skip = n_empty = n_noschema = 0
|
| 864 |
lang_rows = {"hinglish": 0, "hindi": 0, "english": 0}
|
| 865 |
-
|
|
|
|
|
|
|
| 866 |
n_rows += 1
|
| 867 |
-
if max_rows and n_rows > max_rows:
|
| 868 |
break
|
| 869 |
txt = row.get("text", "") if isinstance(row, dict) else ""
|
| 870 |
if not txt or not str(txt).strip():
|
| 871 |
n_empty += 1
|
| 872 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 873 |
if not _safety_ok(row if isinstance(row, dict) else {}):
|
| 874 |
n_safe_skip += 1
|
| 875 |
continue
|
| 876 |
-
txt = str(txt)
|
| 877 |
try:
|
| 878 |
tag = get_lang_tag(row.get("lang") or row.get("source") or txt)
|
| 879 |
except Exception:
|
| 880 |
n_noschema += 1
|
| 881 |
continue
|
| 882 |
bucket = "hinglish" if "hinglish" in tag else ("hindi" if "hindi" in tag else "english")
|
| 883 |
-
|
| 884 |
-
if enforce_mix and len(buckets[bucket]) >= quota[bucket]:
|
| 885 |
n_kept += 1
|
| 886 |
if all(len(buckets[k]) >= quota[k] for k in buckets):
|
| 887 |
break
|
| 888 |
-
if n_rows >= (max_rows or mix_max_rows):
|
| 889 |
break
|
| 890 |
continue
|
| 891 |
-
tag_id = tok.token_to_id(tag) if tok is not None else None
|
| 892 |
if tag_id is not None:
|
| 893 |
buckets[bucket].append(tag_id)
|
| 894 |
-
buckets[bucket].extend(tok.encode(txt).ids)
|
| 895 |
buckets[bucket].append(eos)
|
| 896 |
lang_rows[bucket] += 1
|
| 897 |
n_kept += 1
|
| 898 |
total_tok = sum(len(v) for v in buckets.values())
|
| 899 |
-
if enforce_mix:
|
| 900 |
-
# stop: sab quotas full YA scan bound (rare lang ho to graceful degrade)
|
| 901 |
if all(len(buckets[k]) >= quota[k] for k in buckets):
|
| 902 |
break
|
| 903 |
-
if n_rows >= (max_rows or mix_max_rows):
|
| 904 |
break
|
| 905 |
if total_tok >= scan_cap:
|
| 906 |
break
|
| 907 |
elif total_tok >= cap_tok:
|
| 908 |
break
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 909 |
if R0:
|
| 910 |
-
print(f"[hf]
|
| 911 |
f"skip_empty={n_empty:,} skip_safety({safety_mode})={n_safe_skip:,} skip_schema={n_noschema:,}")
|
| 912 |
import random as _rng
|
| 913 |
-
if enforce_mix:
|
| 914 |
-
# Weighted block sampler: quota tak lo; kami ho to bachi avail se bharo.
|
| 915 |
per_lang_blocks = {}
|
| 916 |
for k, v in buckets.items():
|
| 917 |
arr = torch.tensor(v, dtype=torch.long)
|
| 918 |
-
n = (len(arr) // seq_len) * seq_len
|
| 919 |
-
|
|
|
|
| 920 |
avail = {k: len(v) for k, v in per_lang_blocks.items()}
|
| 921 |
-
quota_b = {k: int(quota[k] // seq_len) for k in avail}
|
| 922 |
take = {k: min(avail[k], quota_b[k]) for k in avail}
|
| 923 |
-
short = max_blocks - sum(take.values())
|
| 924 |
-
if short > 0:
|
| 925 |
left = {k: avail[k] - take[k] for k in avail}
|
| 926 |
tot_left = sum(left.values()) or 1
|
| 927 |
for k in avail:
|
|
@@ -929,24 +1264,34 @@ def main():
|
|
| 929 |
take[k] += extra
|
| 930 |
chosen = [per_lang_blocks[k][:take[k]] for k in avail if take[k] > 0]
|
| 931 |
if not chosen:
|
| 932 |
-
self.blocks = torch.zeros((1, seq_len), dtype=torch.long)
|
| 933 |
else:
|
| 934 |
allb = torch.cat(chosen, dim=0)
|
| 935 |
idx = list(range(len(allb)))
|
| 936 |
-
_rng.Random(mix_seed).shuffle(idx)
|
| 937 |
self.blocks = allb[idx]
|
| 938 |
if R0:
|
| 939 |
print(f"[hf] mix ENFORCED target={weights}: blocks_taken={take} total={len(self.blocks)} (avail={avail})")
|
| 940 |
else:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 941 |
ids = buckets["hinglish"] + buckets["hindi"] + buckets["english"]
|
| 942 |
if R0:
|
| 943 |
-
print(f"[hf] mix NATURAL (
|
| 944 |
ids = torch.tensor(ids, dtype=torch.long)
|
| 945 |
-
n = (len(ids)//seq_len)*seq_len
|
| 946 |
-
self.blocks = ids[:n].view(-1, seq_len) if n>0 else torch.zeros((1,seq_len),dtype=torch.long)
|
| 947 |
-
|
| 948 |
-
|
| 949 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 950 |
ds = HFStreamDataset(hf_ds, tok, seq_len+1, max_blocks=max_blocks_cfg, max_rows=max_rows_cfg,
|
| 951 |
enforce_mix=enforce_mix, lang_mix=lang_mix, mix_seed=seed_r,
|
| 952 |
mix_scan_mult=mix_scan_mult, mix_max_rows=mix_max_rows)
|
|
@@ -1003,14 +1348,23 @@ def main():
|
|
| 1003 |
ids.append(eos)
|
| 1004 |
import torch as _torch
|
| 1005 |
ids_t = _torch.tensor(ids, dtype=_torch.long)
|
| 1006 |
-
n = (len(ids_t) // (seq_len + 1)) * (seq_len + 1)
|
| 1007 |
-
blocks = ids_t[:n].view(-1, seq_len + 1) if n > 0 else _torch.zeros((1, seq_len + 1), dtype=_torch.long)
|
| 1008 |
class _DummyDS(_torch.utils.data.Dataset):
|
| 1009 |
-
def __init__(self,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1010 |
def __len__(self): return len(self.blocks)
|
| 1011 |
def __getitem__(self, i):
|
| 1012 |
x = self.blocks[i]; return x[:-1], x[1:]
|
| 1013 |
-
ds = _DummyDS(
|
| 1014 |
elif ds is not None:
|
| 1015 |
pass # ds already built via HF streaming above
|
| 1016 |
else:
|
|
@@ -1029,7 +1383,12 @@ def main():
|
|
| 1029 |
if len(ds) > n_train:
|
| 1030 |
train_ids = all_idx[:n_train]
|
| 1031 |
val_ids = all_idx[n_train:]
|
| 1032 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1033 |
else:
|
| 1034 |
train_ids = all_idx
|
| 1035 |
val_ds = None
|
|
@@ -1071,6 +1430,12 @@ def main():
|
|
| 1071 |
+ (" [dist broadcast r0->all]" if (use_dist and not args.smoke) else "")) if R0 else None
|
| 1072 |
elif not args.smoke and device.type == "cuda" and R0:
|
| 1073 |
print(f"[batch] MANUAL LOCK: micro={batch} accum={accum} eff={batch * accum} tok/step={batch * accum * seq_len * (world if use_dist else 1)} amp={'bf16' if use_bf16 else 'fp16'}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1074 |
if use_dist and not args.smoke:
|
| 1075 |
# MoE top-2 routing me kuch experts kabhi-kabhi empty rehte hain -> find_unused_parameters=True (safe)
|
| 1076 |
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[device.index], find_unused_parameters=True)
|
|
@@ -1081,13 +1446,68 @@ def main():
|
|
| 1081 |
if R0:
|
| 1082 |
print(f"[model] total={total/1e6:.1f}M active={active/1e6:.1f}M device={device}" + (f" (DDP x{world})" if use_dist else ""))
|
| 1083 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1084 |
decay_params = [p for p in model.parameters() if p.requires_grad and p.ndim >= 2]
|
| 1085 |
no_decay_params = [p for p in model.parameters() if p.requires_grad and p.ndim < 2]
|
| 1086 |
optim_groups = [
|
| 1087 |
{"params": decay_params, "weight_decay": wd},
|
| 1088 |
{"params": no_decay_params, "weight_decay": 0.0},
|
| 1089 |
]
|
| 1090 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1091 |
use_amp = use_bf16 or use_fp16
|
| 1092 |
amp_dtype = torch.bfloat16 if use_bf16 else torch.float16
|
| 1093 |
scaler = torch.amp.GradScaler("cuda") if (use_fp16 and device.type == "cuda") else None
|
|
@@ -1167,6 +1587,28 @@ def main():
|
|
| 1167 |
start = load_ckpt(resolved_ckpt, ckpt_model, opt, scaler=scaler)
|
| 1168 |
if R0:
|
| 1169 |
print(f"[resume] successfully resumed at step {start} from {resolved_ckpt}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1170 |
else:
|
| 1171 |
if R0:
|
| 1172 |
print(f"[resume] checkpoint '{resume}' not found locally or on HF Hub — starting fresh from step 0")
|
|
@@ -1194,13 +1636,67 @@ def main():
|
|
| 1194 |
t0 = time.time()
|
| 1195 |
step = start
|
| 1196 |
idx = 0
|
|
|
|
|
|
|
|
|
|
| 1197 |
tok_per_step = batch * accum * seq_len * (world if use_dist else 1) # DDP: global tokens/step
|
| 1198 |
pbar = None # Industry standard: clean stdout lines (no carriage-return progress bar in web terminal)
|
| 1199 |
-
rng.shuffle(train_ids)
|
| 1200 |
while step < max_steps:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1201 |
if idx >= len(train_ids):
|
| 1202 |
-
|
| 1203 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1204 |
# accumulate (batch_size fix: har micro-step me batch samples stack karo)
|
| 1205 |
opt.zero_grad(set_to_none=True)
|
| 1206 |
acc_loss = 0.0
|
|
@@ -1245,12 +1741,50 @@ def main():
|
|
| 1245 |
layer_avg = [sum(acc_usages[s][l_idx][e] for s in range(len(acc_usages))) / len(acc_usages) for e in range(n_exp)]
|
| 1246 |
avg_usages.append(layer_avg)
|
| 1247 |
aux = {"moe_aux": round(acc_moe_aux, 4), "mt_loss": round(acc_mt_loss, 4), "usages": avg_usages}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1248 |
if scaler is not None:
|
| 1249 |
scaler.unscale_(opt)
|
| 1250 |
-
torch.nn.utils.clip_grad_norm_(model.parameters(), clip)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1251 |
cur_lr = cosine_lr(step + 1, max_steps, warmup, lr, min_lr)
|
|
|
|
| 1252 |
for g in opt.param_groups:
|
| 1253 |
-
|
|
|
|
|
|
|
| 1254 |
if scaler is not None:
|
| 1255 |
scaler.step(opt)
|
| 1256 |
scaler.update()
|
|
@@ -1273,13 +1807,22 @@ def main():
|
|
| 1273 |
max_expert_share = round(max(avg_u), 3)
|
| 1274 |
if max_expert_share > 0.60:
|
| 1275 |
print(f"[warn] ROUTER IMBALANCE: max expert share {max_expert_share} > 0.60!", flush=True)
|
| 1276 |
-
|
| 1277 |
-
|
| 1278 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1279 |
if max_expert_share is not None:
|
| 1280 |
msg["max_expert_share"] = max_expert_share
|
|
|
|
|
|
|
| 1281 |
# Essential clean log line
|
| 1282 |
-
|
|
|
|
| 1283 |
if logf is not None:
|
| 1284 |
logf.write(json.dumps(msg) + "\n")
|
| 1285 |
logf.flush()
|
|
|
|
| 25 |
from torch.utils.data import Dataset
|
| 26 |
|
| 27 |
sys.path.insert(0, str(Path(__file__).parent))
|
| 28 |
+
try:
|
| 29 |
+
from viu_moe import ViuArgs, Viu1MoE, load_args as load_model_args
|
| 30 |
+
except ImportError:
|
| 31 |
+
from viu1_moe import ViuArgs, Viu1MoE, load_args as load_model_args
|
| 32 |
|
| 33 |
try:
|
| 34 |
import yaml
|
|
|
|
| 48 |
|
| 49 |
# ---------------- data ----------------
|
| 50 |
def get_lang_tag(name_or_text: str) -> str:
|
| 51 |
+
n = str(name_or_text).lower().strip()
|
| 52 |
+
# Explicit language codes and names
|
| 53 |
+
if n in ("en", "eng", "english") or "english" in n:
|
|
|
|
|
|
|
|
|
|
| 54 |
return "<|english|>"
|
| 55 |
+
if n in ("hi", "hin", "hindi") or "hindi" in n:
|
| 56 |
+
return "<|hindi|>"
|
| 57 |
+
if n in ("hinglish", "hi-en", "en-hi", "hi_en", "bilingual") or "hinglish" in n:
|
| 58 |
+
return "<|hinglish|>"
|
| 59 |
+
# Simple heuristic for raw text: Devanagari -> hindi, else ascii -> english, else hinglish
|
| 60 |
if any('\u0900' <= ch <= '\u097F' for ch in n):
|
| 61 |
return "<|hindi|>"
|
| 62 |
+
if n.isascii():
|
| 63 |
+
return "<|english|>"
|
| 64 |
return "<|hinglish|>"
|
| 65 |
|
| 66 |
|
|
|
|
| 87 |
ids.append(eos)
|
| 88 |
if len(ids) > warn_tokens:
|
| 89 |
print(f"[warn] {len(ids)/1e6:.1f}M tokens RAM me — bade corpus ke liye memmap/HF-streaming use karo.")
|
| 90 |
+
self.raw_ids = torch.tensor(ids, dtype=torch.long)
|
| 91 |
+
self.seq_len = seq_len
|
| 92 |
+
self.rebuild_blocks()
|
| 93 |
+
|
| 94 |
+
def rebuild_blocks(self):
|
| 95 |
+
n = (len(self.raw_ids) // self.seq_len) * self.seq_len
|
| 96 |
+
self.blocks = self.raw_ids[:n].view(-1, self.seq_len) if n > 0 else torch.zeros((1, self.seq_len), dtype=torch.long)
|
| 97 |
+
|
| 98 |
+
def set_seq_len(self, new_seq_len):
|
| 99 |
+
if new_seq_len == self.seq_len:
|
| 100 |
+
return
|
| 101 |
+
self.seq_len = new_seq_len
|
| 102 |
+
self.rebuild_blocks()
|
| 103 |
|
| 104 |
def __len__(self):
|
| 105 |
return len(self.blocks)
|
|
|
|
| 118 |
return files
|
| 119 |
|
| 120 |
|
| 121 |
+
# ---------------- curriculum & sched ----------------
|
| 122 |
+
def resolve_curriculum_stage(step, stages):
|
| 123 |
+
"""Return (stage_idx, stage_dict) based on current step."""
|
| 124 |
+
if not stages:
|
| 125 |
+
return 0, {}
|
| 126 |
+
for idx, st in enumerate(stages):
|
| 127 |
+
m = st.get("max_step")
|
| 128 |
+
if m is None or step < m:
|
| 129 |
+
return idx, st
|
| 130 |
+
return len(stages) - 1, stages[-1]
|
| 131 |
+
|
| 132 |
+
|
| 133 |
def fmt_hms(sec):
|
| 134 |
sec = max(int(sec), 0)
|
| 135 |
h, sec = divmod(sec, 3600)
|
|
|
|
| 144 |
return min_lr + 0.5 * (lr - min_lr) * (1 + math.cos(math.pi * min(p, 1.0)))
|
| 145 |
|
| 146 |
|
| 147 |
+
# ---------------- Muon Optimizer (Keller Jordan / Kimi K2) ----------------
|
| 148 |
+
def zeropower_via_newtonschulz5(G, steps=5, eps=1e-7):
|
| 149 |
+
"""
|
| 150 |
+
Newton-Schulz quintic iteration to approximate matrix polar decomposition.
|
| 151 |
+
Stabilizes and orthogonalizes momentum updates for 2D weight matrices.
|
| 152 |
+
"""
|
| 153 |
+
assert len(G.shape) == 2
|
| 154 |
+
a, b, c = (3.4445, -4.7750, 2.0315)
|
| 155 |
+
X = G.bfloat16() if (G.is_cuda and torch.cuda.is_bf16_supported()) else G.float()
|
| 156 |
+
X = X / (X.norm() + eps)
|
| 157 |
+
transposed = False
|
| 158 |
+
if X.size(0) > X.size(1):
|
| 159 |
+
X = X.T
|
| 160 |
+
transposed = True
|
| 161 |
+
for _ in range(steps):
|
| 162 |
+
A = X @ X.T
|
| 163 |
+
B = b * A + c * A @ A
|
| 164 |
+
X = a * X + B @ X
|
| 165 |
+
if transposed:
|
| 166 |
+
X = X.T
|
| 167 |
+
return X.to(dtype=G.dtype)
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
class Muon(torch.optim.Optimizer):
|
| 171 |
+
"""
|
| 172 |
+
Muon (MomentUm Orthogonalized by Newton-schulz) Optimizer.
|
| 173 |
+
Applies orthogonalized momentum updates to 2D internal weight matrices.
|
| 174 |
+
"""
|
| 175 |
+
def __init__(self, params, lr=0.02, momentum=0.95, weight_decay=0.01, ns_steps=5):
|
| 176 |
+
defaults = dict(lr=lr, momentum=momentum, weight_decay=weight_decay, ns_steps=ns_steps)
|
| 177 |
+
super().__init__(params, defaults)
|
| 178 |
+
|
| 179 |
+
@torch.no_grad()
|
| 180 |
+
def step(self, closure=None):
|
| 181 |
+
loss = None
|
| 182 |
+
if closure is not None:
|
| 183 |
+
with torch.enable_grad():
|
| 184 |
+
loss = closure()
|
| 185 |
+
|
| 186 |
+
for group in self.param_groups:
|
| 187 |
+
lr = group["lr"]
|
| 188 |
+
momentum = group["momentum"]
|
| 189 |
+
wd = group["weight_decay"]
|
| 190 |
+
steps = group["ns_steps"]
|
| 191 |
+
|
| 192 |
+
for p in group["params"]:
|
| 193 |
+
if p.grad is None:
|
| 194 |
+
continue
|
| 195 |
+
g = p.grad
|
| 196 |
+
state = self.state[p]
|
| 197 |
+
|
| 198 |
+
# Initialize momentum buffer
|
| 199 |
+
if "momentum_buffer" not in state:
|
| 200 |
+
state["momentum_buffer"] = torch.zeros_like(g)
|
| 201 |
+
|
| 202 |
+
buf = state["momentum_buffer"]
|
| 203 |
+
buf.mul_(momentum).add_(g)
|
| 204 |
+
|
| 205 |
+
# Orthogonalize via Newton-Schulz
|
| 206 |
+
u = zeropower_via_newtonschulz5(buf, steps=steps)
|
| 207 |
+
|
| 208 |
+
# Scale update based on aspect ratio
|
| 209 |
+
scale = max(1.0, math.sqrt(p.size(0) / p.size(1)))
|
| 210 |
+
u.mul_(scale)
|
| 211 |
+
|
| 212 |
+
# Weight decay
|
| 213 |
+
if wd > 0:
|
| 214 |
+
p.mul_(1.0 - lr * wd)
|
| 215 |
+
|
| 216 |
+
# Apply update
|
| 217 |
+
p.add_(u, alpha=-lr)
|
| 218 |
+
|
| 219 |
+
return loss
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
class CombinedOptimizer:
|
| 223 |
+
"""Combines Muon (for 2D linear weights) and AdamW (for 1D norms, embeddings, routers)."""
|
| 224 |
+
def __init__(self, optimizers):
|
| 225 |
+
self.optimizers = optimizers
|
| 226 |
+
self.param_groups = []
|
| 227 |
+
for opt in optimizers:
|
| 228 |
+
self.param_groups.extend(opt.param_groups)
|
| 229 |
+
|
| 230 |
+
def step(self, closure=None):
|
| 231 |
+
loss = None
|
| 232 |
+
for opt in self.optimizers:
|
| 233 |
+
res = opt.step(closure=closure)
|
| 234 |
+
if res is not None:
|
| 235 |
+
loss = res
|
| 236 |
+
return loss
|
| 237 |
+
|
| 238 |
+
def zero_grad(self, set_to_none=True):
|
| 239 |
+
for opt in self.optimizers:
|
| 240 |
+
opt.zero_grad(set_to_none=set_to_none)
|
| 241 |
+
|
| 242 |
+
@property
|
| 243 |
+
def state(self):
|
| 244 |
+
# Merged param->state view so torch AMP GradScaler.unscale_() works
|
| 245 |
+
# (it reads optimizer.state[p]). FP16-only path; BF16 configs never touch it.
|
| 246 |
+
merged = {}
|
| 247 |
+
for opt in self.optimizers:
|
| 248 |
+
try:
|
| 249 |
+
merged.update(opt.state)
|
| 250 |
+
except Exception:
|
| 251 |
+
pass
|
| 252 |
+
return merged
|
| 253 |
+
|
| 254 |
+
def state_dict(self):
|
| 255 |
+
return {"opts": [opt.state_dict() for opt in self.optimizers]}
|
| 256 |
+
|
| 257 |
+
def load_state_dict(self, state_dict):
|
| 258 |
+
if isinstance(state_dict, dict) and "opts" in state_dict:
|
| 259 |
+
for opt, sd in zip(self.optimizers, state_dict["opts"]):
|
| 260 |
+
opt.load_state_dict(sd)
|
| 261 |
+
elif len(self.optimizers) > 1:
|
| 262 |
+
try:
|
| 263 |
+
self.optimizers[1].load_state_dict(state_dict)
|
| 264 |
+
except Exception:
|
| 265 |
+
pass
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
def split_muon_params(model):
|
| 269 |
+
"""
|
| 270 |
+
Separates parameters into:
|
| 271 |
+
- Muon: 2D internal linear weights in MLA attention and MoE experts
|
| 272 |
+
- AdamW: Embeddings, RMSNorms, routers, biases, output heads
|
| 273 |
+
"""
|
| 274 |
+
muon_params = []
|
| 275 |
+
adamw_params = []
|
| 276 |
+
|
| 277 |
+
for name, p in model.named_parameters():
|
| 278 |
+
if not p.requires_grad:
|
| 279 |
+
continue
|
| 280 |
+
# 1D params (norms, biases) always AdamW
|
| 281 |
+
if p.ndim < 2:
|
| 282 |
+
adamw_params.append(p)
|
| 283 |
+
# Embeddings, router heads, and unembedding heads use AdamW
|
| 284 |
+
elif "tok_emb" in name or "head" in name or "router" in name:
|
| 285 |
+
adamw_params.append(p)
|
| 286 |
+
else:
|
| 287 |
+
muon_params.append(p)
|
| 288 |
+
|
| 289 |
+
return muon_params, adamw_params
|
| 290 |
+
|
| 291 |
+
|
| 292 |
+
|
| 293 |
def pick_device(pref=None):
|
| 294 |
"""Auto GPU: sabse zyada free VRAM wala cuda, warna cpu. --device se override."""
|
| 295 |
if pref:
|
|
|
|
| 346 |
+ fragmentation ka explicit reserve chahiye, warna wrap ke baad training step-1 pe OOM."""
|
| 347 |
model.train()
|
| 348 |
best = 1
|
| 349 |
+
# Reserve: Muon fp32 momentum (~1.7B 2D params ~= 6.8GB) + AdamW8bit states +
|
| 350 |
+
# DDP grad buckets + fragmentation. 1.856B model => try 8GB, warna 4GB, warna 2GB.
|
| 351 |
adamw_buf = None
|
| 352 |
if device.type == "cuda":
|
| 353 |
+
for n_elems in (2_000_000_000, 1_000_000_000, 500_000_000): # ~8.0 / 4.0 / 2.0 GB
|
| 354 |
try:
|
| 355 |
adamw_buf = torch.empty((n_elems,), dtype=torch.float32, device=device)
|
| 356 |
break
|
|
|
|
| 384 |
return best
|
| 385 |
|
| 386 |
|
| 387 |
+
def maybe_apply_fp8(model, want, device, R0=True):
|
| 388 |
+
"""EXPERIMENTAL 2025-tech (Blackwell/5090 only): torchao Float8 GEMM training.
|
| 389 |
+
Default OFF. Optimizer (Muon/AdamW) original-dtype params dekhta hai, isliye
|
| 390 |
+
compatible hai. Koi bhi condition miss -> loud BF16 fallback, silent nahi."""
|
| 391 |
+
if not want:
|
| 392 |
+
return model, "fp8=off"
|
| 393 |
+
if device.type != "cuda":
|
| 394 |
+
if R0:
|
| 395 |
+
print("[fp8] CPU box — BF16 fallback (FP8 needs Blackwell CUDA)")
|
| 396 |
+
return model, "fp8=off(cpu)"
|
| 397 |
+
try:
|
| 398 |
+
major, _ = torch.cuda.get_device_capability(device)
|
| 399 |
+
except Exception:
|
| 400 |
+
major = 0
|
| 401 |
+
if major < 9:
|
| 402 |
+
if R0:
|
| 403 |
+
print("[fp8] pre-Blackwell GPU (sm<90) — BF16 fallback (FP8 needs sm_90+)")
|
| 404 |
+
return model, "fp8=off(arch)"
|
| 405 |
+
try:
|
| 406 |
+
from torchao.float8 import convert_to_float8_training
|
| 407 |
+
except ImportError:
|
| 408 |
+
if R0:
|
| 409 |
+
print("[fp8] torchao missing (pip install torchao --index-url https://download.pytorch.org/whl/nightly/cu128) — BF16 fallback")
|
| 410 |
+
return model, "fp8=off(no-torchao)"
|
| 411 |
+
try:
|
| 412 |
+
convert_to_float8_training(model)
|
| 413 |
+
if R0:
|
| 414 |
+
print("[fp8] Float8 training ENABLED via torchao (EXPERIMENTAL — pehle 500 steps loss-watch karo)")
|
| 415 |
+
return model, "fp8=on"
|
| 416 |
+
except Exception as e:
|
| 417 |
+
if R0:
|
| 418 |
+
print(f"[fp8] convert failed ({str(e)[:120]}) — BF16 fallback")
|
| 419 |
+
return model, "fp8=off(err)"
|
| 420 |
+
|
| 421 |
+
|
| 422 |
+
def maybe_apply_compile(model, want, mode, is_smoke, R0=True, device=None, vocab_size=48000):
|
| 423 |
+
"""torch.compile (2024-25 mature): 10-30% free speedup. Smoke me skip (fast test).
|
| 424 |
+
DDP-wrap SE PEHLE lagta hai taaki ckpt keys prefix-free rahein.
|
| 425 |
+
Fail-FAST: chhota warmup forward backend verify karta hai (missing triton/compiler
|
| 426 |
+
pehle step pe crash ke bajaye turant eager fallback). Pehla compiled step slow
|
| 427 |
+
hota hai (one-time codegen, minutes) — ye normal hai."""
|
| 428 |
+
if not want:
|
| 429 |
+
return model, "compile=off"
|
| 430 |
+
if is_smoke:
|
| 431 |
+
if R0:
|
| 432 |
+
print("[compile] smoke me skip (fast CPU test chahiye)")
|
| 433 |
+
return model, "compile=off(smoke)"
|
| 434 |
+
try:
|
| 435 |
+
model = torch.compile(model, mode=mode, fullgraph=False)
|
| 436 |
+
if device is not None and str(device) != "cpu":
|
| 437 |
+
with torch.no_grad():
|
| 438 |
+
_ids = torch.randint(0, vocab_size, (1, 16), device=device)
|
| 439 |
+
_ = model(_ids)
|
| 440 |
+
if R0:
|
| 441 |
+
print(f"[compile] torch.compile ON (mode={mode}, fullgraph=False, backend-verified, DDP-se-pehle)")
|
| 442 |
+
return model, f"compile=on({mode})"
|
| 443 |
+
except Exception as e:
|
| 444 |
+
if R0:
|
| 445 |
+
print(f"[compile] backend unavailable ({str(e)[:100]}) — eager fallback (training safe hai)")
|
| 446 |
+
try:
|
| 447 |
+
# compiled wrapper hata ke original module wapas (eager, safe)
|
| 448 |
+
inner = model._orig_mod if hasattr(model, "_orig_mod") else model
|
| 449 |
+
if inner is not model:
|
| 450 |
+
return inner, "compile=off(err-fallback)"
|
| 451 |
+
except Exception:
|
| 452 |
+
pass
|
| 453 |
+
return model, "compile=off(err)"
|
| 454 |
+
|
| 455 |
+
|
| 456 |
def save_slim_ckpt(path, model, cfg=None):
|
| 457 |
+
"""Inference & HF deployment ke liye slim checkpoint (~3.7GB BF16 for 1.856B, optimizer states ke bina)."""
|
| 458 |
path = Path(path)
|
| 459 |
path.parent.mkdir(parents=True, exist_ok=True)
|
| 460 |
state = {k: v.to(dtype=torch.bfloat16).cpu() for k, v in model.state_dict().items()}
|
|
|
|
| 633 |
# val PPL ke liye pure LM loss chahiye — MoE-aux + mt-loss hatao
|
| 634 |
base = loss.item()
|
| 635 |
try:
|
| 636 |
+
mt_w = float(aux.get("mt_loss_weight", 0.1)) # match viu_moe.py forward()
|
| 637 |
+
base = base - float(aux.get("moe_aux", 0.0)) - mt_w * float(aux.get("mt_loss", 0.0))
|
| 638 |
except Exception:
|
| 639 |
pass
|
| 640 |
tot += base
|
|
|
|
| 667 |
# Fragmentation kam karo (T4 pe large-alloc OOM ke baad reserved-but-unused dikhta tha)
|
| 668 |
os.environ.setdefault("PYTORCH_ALLOC_CONF", "expandable_segments:True")
|
| 669 |
ap = argparse.ArgumentParser()
|
| 670 |
+
ap.add_argument("--config", default="../configs/train_rtx5090.yaml")
|
| 671 |
+
ap.add_argument("--model_config", default="../configs/model_config.yaml")
|
| 672 |
ap.add_argument("--smoke", action="store_true", help="CPU pe tiny model + 3 steps test")
|
| 673 |
ap.add_argument("--steps", "--max_steps", dest="steps", type=int, default=None, help="Max training steps override")
|
| 674 |
ap.add_argument("--resume", default=None, help="Checkpoint path to resume training from")
|
|
|
|
| 679 |
ap.add_argument("--hf_split", default="train", help="Dataset split (default: train)")
|
| 680 |
ap.add_argument("--batch_size", type=int, default=None, help="Micro-batch size override")
|
| 681 |
ap.add_argument("--grad_accum", type=int, default=None, help="Gradient accumulation steps override")
|
| 682 |
+
ap.add_argument("--seq_len", type=int, default=None, help="Sequence length override (bypasses curriculum)")
|
| 683 |
+
ap.add_argument("--curriculum", dest="curriculum", action="store_true", default=None, help="Enable 3-stage sequence length curriculum (512 -> 1024 -> 2048)")
|
| 684 |
+
ap.add_argument("--no_curriculum", "--no-curriculum", dest="curriculum", action="store_false", help="Disable sequence length curriculum")
|
| 685 |
ap.add_argument("--no_auto_tune", action="store_true", help="Disable auto-tune and enforce exact batch_size and grad_accum")
|
| 686 |
+
ap.add_argument("--optimizer", default=None, choices=["muon", "adamw", "paged_adamw_8bit", "8bit"], help="Optimizer choice (muon, adamw, paged_adamw_8bit)")
|
| 687 |
+
ap.add_argument("--muon_lr", type=float, default=None, help="Muon learning rate override (default: 0.02)")
|
| 688 |
+
ap.add_argument("--muon_weight_decay", type=float, default=None, help="Muon weight decay override (default: 0.01)")
|
| 689 |
+
ap.add_argument("--compile", action="store_true", help="torch.compile on (10-30%% free speedup, default off)")
|
| 690 |
+
ap.add_argument("--fp8", action="store_true", help="torchao Float8 training (Blackwell/5090 only, EXPERIMENTAL)")
|
| 691 |
args = ap.parse_args()
|
| 692 |
|
| 693 |
base = Path(__file__).parent
|
|
|
|
| 722 |
if args.smoke:
|
| 723 |
print("[mode] SMOKE — tiny model, CPU, 3 steps")
|
| 724 |
margs = ViuArgs(dim=64, n_layers=2, n_heads=4, n_kv_heads=2, vocab_size=512,
|
| 725 |
+
max_seq_len=64, sliding_window=32, moe_every=2,
|
| 726 |
+
moe_num_experts=4, moe_num_routed_experts=4, moe_shared_experts=1,
|
| 727 |
multi_token=False, switch_gate=False)
|
| 728 |
seq_len, batch, max_steps = 32, 2, 3
|
| 729 |
data_dir, tok_path = base / "../../data/raw", None
|
|
|
|
| 734 |
device = torch.device("cpu")
|
| 735 |
use_bf16 = False
|
| 736 |
use_fp16 = False
|
| 737 |
+
use_curriculum = False
|
| 738 |
+
curriculum_stages = None
|
| 739 |
+
curr_stage_idx = 0
|
| 740 |
else:
|
| 741 |
margs = load_model_args(args.model_config if Path(args.model_config).exists()
|
| 742 |
else str(base / args.model_config))
|
| 743 |
+
# Sequence Length Curriculum Setup (512 -> 1024 -> 2048)
|
| 744 |
+
use_curriculum = bool(C("curriculum", False))
|
| 745 |
+
if args.curriculum is not None:
|
| 746 |
+
use_curriculum = bool(args.curriculum)
|
| 747 |
+
if args.seq_len is not None:
|
| 748 |
+
use_curriculum = False # explicit --seq_len overrides curriculum
|
| 749 |
+
|
| 750 |
+
curriculum_stages = C("curriculum_stages", [
|
| 751 |
+
{"max_step": 50000, "seq_len": 512, "batch_size": 8, "grad_accum": 64},
|
| 752 |
+
{"max_step": 200000, "seq_len": 1024, "batch_size": 8, "grad_accum": 32},
|
| 753 |
+
{"max_step": None, "seq_len": 2048, "batch_size": 4, "grad_accum": 32},
|
| 754 |
+
]) if use_curriculum else None
|
| 755 |
+
|
| 756 |
+
curr_stage_idx = 0
|
| 757 |
+
if use_curriculum:
|
| 758 |
+
curr_stage_idx, curr_stage = resolve_curriculum_stage(0, curriculum_stages)
|
| 759 |
+
seq_len = int(curr_stage["seq_len"])
|
| 760 |
+
batch = int(args.batch_size or curr_stage["batch_size"])
|
| 761 |
+
accum = int(args.grad_accum or curr_stage["grad_accum"])
|
| 762 |
+
if R0:
|
| 763 |
+
print(f"[curriculum] Initialized Stage 1/{len(curriculum_stages)}: "
|
| 764 |
+
f"seq_len={seq_len} batch={batch} accum={accum} (target {batch*accum*seq_len:,} tok/step)")
|
| 765 |
+
else:
|
| 766 |
+
seq_len = int(args.seq_len or C("seq_len", 1024))
|
| 767 |
+
batch = int(args.batch_size or C("batch_size", 2))
|
| 768 |
+
accum = int(args.grad_accum or C("grad_accum", 8))
|
| 769 |
+
_cfg_steps = C("max_steps", None)
|
| 770 |
+
max_steps = args.steps or (int(_cfg_steps) if _cfg_steps not in ("auto", "none", "null", None) else None)
|
| 771 |
data_dir = resolve(C("data_dir", "../../data/raw"))
|
| 772 |
tok_path = resolve(C("tokenizer_path", "../../tokenizer/outputs/tokenizer.json"))
|
| 773 |
lr, min_lr, warmup, wd = C("lr", 3e-4), C("min_lr", 3e-5), C("warmup_steps", 2500), C("weight_decay", 0.1)
|
| 774 |
+
clip = float(C("grad_clip", 1.0))
|
| 775 |
eval_every, save_every, log_every = C("eval_every", 1000), C("save_every", 5000), C("log_every", 50)
|
| 776 |
sample_every = C("sample_every", 1000)
|
| 777 |
ckpt_dir = resolve(C("ckpt_dir", "../checkpoints"))
|
| 778 |
log_dir = resolve(C("log_dir", "../../logs"))
|
| 779 |
+
hf_repo = C("hf_repo", "ViuAI/Viu-1.5B-MoE")
|
| 780 |
hf_token = os.environ.get("HF_TOKEN")
|
| 781 |
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 782 |
if args.device:
|
|
|
|
| 837 |
import shutil
|
| 838 |
try:
|
| 839 |
real_tok = hf_hub_download(
|
| 840 |
+
repo_id=hf_repo or "ViuAI/Viu-1.5B-MoE",
|
| 841 |
filename="tokenizer/outputs/tokenizer.json",
|
| 842 |
token=hf_token
|
| 843 |
)
|
|
|
|
| 882 |
print(f"[hf] streaming dataset {hf_dataset} (native PyArrow streaming)")
|
| 883 |
# Non-5-col schema folders (AUDIT A3) — inko stream me mat lo, warna
|
| 884 |
# shuffle/interleave par CastError (wikipedia: title/text/url vs text/lang/...).
|
| 885 |
+
# Note: distilled/, science/, and translation/ (Samanantar) handled in _stream_parquet_rows.
|
| 886 |
+
# Exclude only truly unformatted raw directories (already packaged into domains/)
|
| 887 |
+
# and empty/non-parquet files. All 1,874 valid Parquet files stream natively.
|
| 888 |
# ZERO EXCLUSIONS: Stream 100% of all files in repository
|
| 889 |
NON_UNIFIED_PREFIXES = ()
|
| 890 |
# Corpus-slice rotation (Option A: full-data coverage round-by-round).
|
|
|
|
| 940 |
|
| 941 |
def _stream_parquet_rows(files, repo_id):
|
| 942 |
for fp in files:
|
| 943 |
+
p = f"datasets/{repo_id}/{fp}"
|
| 944 |
+
for _attempt in range(3):
|
| 945 |
+
try:
|
| 946 |
+
with _fs.open(p, "rb") as f:
|
| 947 |
+
pf = _pq.ParquetFile(f)
|
| 948 |
+
_names = pf.schema.names
|
| 949 |
+
if "toxicity" in _names and "text" in _names:
|
| 950 |
+
# Jigsaw civil_comments: score columns -> safety_tag (label preserved as tag, not text)
|
| 951 |
+
cols = ["text", "toxicity", "severe_toxicity", "obscene", "threat", "insult", "identity_attack", "sexual_explicit"]
|
| 952 |
+
cols = [c for c in cols if c in _names]
|
| 953 |
+
for batch in pf.iter_batches(batch_size=2048, columns=cols):
|
| 954 |
+
d = batch.to_pydict()
|
| 955 |
+
n = len(d.get("text", []))
|
| 956 |
+
for i in range(n):
|
| 957 |
+
tw = str(d["text"][i])
|
| 958 |
+
if len(tw.strip()) <= 5:
|
| 959 |
+
continue
|
| 960 |
+
mx = 0.0
|
| 961 |
+
for sc in cols[1:]:
|
|
|
|
|
|
|
|
|
|
| 962 |
try:
|
| 963 |
+
v = float(d[sc][i])
|
| 964 |
+
except Exception:
|
| 965 |
v = 0.0
|
| 966 |
if v > mx:
|
| 967 |
mx = v
|
| 968 |
+
tag = "toxic" if mx >= 0.5 else "safe"
|
| 969 |
+
yield {"text": tw, "lang": "en", "source": "jigsaw_civil_comments", "domain": "toxicity", "safety_tag": tag}
|
| 970 |
+
elif "text" in _names:
|
| 971 |
+
cols = [c for c in ["text", "lang", "source", "domain", "safety_tag"] if c in pf.schema.names]
|
| 972 |
+
for batch in pf.iter_batches(batch_size=2048, columns=cols):
|
| 973 |
+
d = batch.to_pydict()
|
| 974 |
+
n = len(d.get("text", []))
|
| 975 |
+
for i in range(n):
|
| 976 |
+
yield {k: d[k][i] for k in cols}
|
| 977 |
+
elif "generated_full_text" in pf.schema.names:
|
| 978 |
+
for batch in pf.iter_batches(batch_size=2048, columns=["generated_full_text"]):
|
| 979 |
+
d = batch.to_pydict()
|
| 980 |
+
n = len(d.get("generated_full_text", []))
|
| 981 |
+
for i in range(n):
|
| 982 |
+
yield {"text": str(d["generated_full_text"][i]), "lang": "en", "source": "science_textbooks", "domain": "science", "safety_tag": "safe"}
|
| 983 |
+
elif "src" in pf.schema.names and "tgt" in pf.schema.names:
|
| 984 |
+
for batch in pf.iter_batches(batch_size=2048, columns=["src", "tgt"]):
|
| 985 |
+
d = batch.to_pydict()
|
| 986 |
+
n = len(d.get("src", []))
|
| 987 |
+
for i in range(n):
|
| 988 |
+
s = str(d["src"][i]).strip()
|
| 989 |
+
t = str(d["tgt"][i]).strip()
|
| 990 |
+
if len(s) > 10 and len(t) > 10:
|
| 991 |
+
if i % 2 == 0:
|
| 992 |
+
txt = f"English: {s}\nHindi: {t}"
|
| 993 |
+
else:
|
| 994 |
+
txt = f"Hindi: {t}\nEnglish: {s}"
|
| 995 |
+
yield {"text": txt, "lang": "bilingual", "source": "samanantar_translation", "domain": "translation", "safety_tag": "safe"}
|
| 996 |
+
elif "context" in pf.schema.names and "question" in pf.schema.names and "response" in pf.schema.names:
|
| 997 |
+
for batch in pf.iter_batches(batch_size=2048, columns=["context", "question", "response"]):
|
| 998 |
+
d = batch.to_pydict()
|
| 999 |
+
n = len(d.get("question", []))
|
| 1000 |
+
for i in range(n):
|
| 1001 |
+
ctx = str(d["context"][i]).strip()
|
| 1002 |
+
q = str(d["question"][i]).strip()
|
| 1003 |
+
ans = str(d["response"][i]).strip()
|
| 1004 |
+
parts = []
|
| 1005 |
+
if ctx and len(ctx) > 30:
|
| 1006 |
+
parts.append(f"### Case Background:\n{ctx}")
|
| 1007 |
+
parts.append(f"### Legal Query:\n{q}")
|
| 1008 |
+
parts.append(f"### Legal Determination:\n{ans}")
|
| 1009 |
+
yield {"text": "\n\n".join(parts), "lang": "en", "source": "indian_courts_case_law", "domain": "governance_law", "safety_tag": "safe"}
|
| 1010 |
+
elif "instruction" in pf.schema.names and "output" in pf.schema.names:
|
| 1011 |
+
cols = [c for c in ["instruction", "input", "output"] if c in pf.schema.names]
|
| 1012 |
+
for batch in pf.iter_batches(batch_size=2048, columns=cols):
|
| 1013 |
+
d = batch.to_pydict()
|
| 1014 |
+
n = len(d["instruction"])
|
| 1015 |
+
for i in range(n):
|
| 1016 |
+
ins = str(d["instruction"][i]).strip()
|
| 1017 |
+
inp = str(d["input"][i]).strip() if "input" in d else ""
|
| 1018 |
+
ans = str(d["output"][i]).strip()
|
| 1019 |
+
if len(ins) > 5 and len(ans) > 5:
|
| 1020 |
+
if inp and len(inp) > 5:
|
| 1021 |
+
txt = f"### Instruction:\n{ins}\n\n### Input:\n{inp}\n\n### Response:\n{ans}"
|
| 1022 |
+
else:
|
| 1023 |
+
txt = f"### Instruction:\n{ins}\n\n### Response:\n{ans}"
|
| 1024 |
+
lg = "hinglish" if any('\u0900' <= ch <= '\u097F' for ch in txt[:500]) else "en"
|
| 1025 |
+
yield {"text": txt, "lang": lg, "source": "instruction_following", "domain": "instruction", "safety_tag": "safe"}
|
| 1026 |
+
elif "tweet" in pf.schema.names:
|
| 1027 |
+
for batch in pf.iter_batches(batch_size=2048, columns=["tweet"]):
|
| 1028 |
+
d = batch.to_pydict()
|
| 1029 |
+
n = len(d.get("tweet", []))
|
| 1030 |
+
for i in range(n):
|
| 1031 |
+
tw = str(d["tweet"][i]).strip()
|
| 1032 |
+
if len(tw) > 5:
|
| 1033 |
+
yield {"text": tw, "lang": "en", "source": "hate_speech_offensive", "domain": "toxicity", "safety_tag": "toxic"}
|
| 1034 |
+
elif "question" in pf.schema.names and "answer" in pf.schema.names:
|
| 1035 |
+
# Standalone Q/A pairs (math/problem sets without context column)
|
| 1036 |
+
for batch in pf.iter_batches(batch_size=2048, columns=["question", "answer"]):
|
| 1037 |
+
d = batch.to_pydict()
|
| 1038 |
+
n = len(d.get("question", []))
|
| 1039 |
+
for i in range(n):
|
| 1040 |
+
q = str(d["question"][i]).strip()
|
| 1041 |
+
a = str(d["answer"][i]).strip()
|
| 1042 |
+
if len(q) > 5 and len(a) > 5:
|
| 1043 |
+
yield {"text": f"### Problem:\n{q}\n\n### Solution:\n{a}", "lang": "en", "source": "math_qa_pairs", "domain": "reasoning_math", "safety_tag": "safe"}
|
| 1044 |
+
elif "Patient" in pf.schema.names and "Doctor" in pf.schema.names:
|
| 1045 |
+
cols = [c for c in ["Description", "Patient", "Doctor"] if c in pf.schema.names]
|
| 1046 |
+
for batch in pf.iter_batches(batch_size=2048, columns=cols):
|
| 1047 |
+
d = batch.to_pydict()
|
| 1048 |
+
n = len(d["Patient"])
|
| 1049 |
+
for i in range(n):
|
| 1050 |
+
desc = str(d["Description"][i]).strip() if "Description" in d else ""
|
| 1051 |
+
pat = str(d["Patient"][i]).strip()
|
| 1052 |
+
doc = str(d["Doctor"][i]).strip()
|
| 1053 |
+
if pat and doc:
|
| 1054 |
+
txt = f"### Medical Case:\n{desc}\n\n### Patient:\n{pat}\n\n### Doctor:\n{doc}" if desc else f"### Patient:\n{pat}\n\n### Doctor:\n{doc}"
|
| 1055 |
+
yield {"text": txt, "lang": "en", "source": "ai_medical_dialogues", "domain": "health", "safety_tag": "safe"}
|
| 1056 |
+
break # success — move to next file
|
| 1057 |
+
except GeneratorExit:
|
| 1058 |
+
return # caller closed the generator
|
| 1059 |
+
except Exception as e:
|
| 1060 |
+
if _attempt < 2:
|
| 1061 |
+
import time as _time
|
| 1062 |
+
_time.sleep(2 ** _attempt) # backoff: 1s, 2s
|
| 1063 |
+
else:
|
| 1064 |
+
if R0:
|
| 1065 |
+
print(f"[hf] WARNING: skipping {fp} after 3 retries: {e}")
|
| 1066 |
|
| 1067 |
def _stream_shuffled(gen, buffer_size=20000, seed=42):
|
| 1068 |
if buffer_size <= 0:
|
|
|
|
| 1105 |
mix_max_rows = int(C("mix_max_rows", 2000000))
|
| 1106 |
allow_toxic = safety_mode in ("research", "full")
|
| 1107 |
allow_uncensored = safety_mode in ("full",)
|
| 1108 |
+
allow_erotica = safety_mode in ("full",)
|
| 1109 |
if R0:
|
| 1110 |
+
print(f"[hf] safety={safety_mode} (toxic={allow_toxic}, uncensored={allow_uncensored}, erotica={allow_erotica}) "
|
| 1111 |
f"max_blocks={max_blocks_cfg} max_rows={max_rows_cfg or 'unlimited'} "
|
| 1112 |
f"enforce_mix={enforce_mix} lang_mix={lang_mix if enforce_mix else 'OFF (shuffle-buffer order)'}"
|
| 1113 |
+ (f" natural_shuffle={natural_shuffle}" if (not enforce_mix and natural_shuffle > 0) else "")
|
|
|
|
| 1125 |
return allow_toxic
|
| 1126 |
if tag == "uncensored":
|
| 1127 |
return allow_uncensored
|
| 1128 |
+
if tag in ("sexual_explicit", "erotica", "erotic"):
|
| 1129 |
+
return allow_erotica
|
| 1130 |
s = f"{r.get('source','')} {r.get('domain','')}".lower()
|
| 1131 |
if "uncensor" in s or "unfilter" in s or "vortex" in s:
|
| 1132 |
return allow_uncensored
|
| 1133 |
+
if "erotic" in s:
|
| 1134 |
+
return allow_erotica
|
| 1135 |
if any(k in s for k in ("toxic", "hate", "offensive", "profan", "civil_comments", "hasoc", "prism")):
|
| 1136 |
return allow_toxic
|
| 1137 |
return True
|
|
|
|
| 1139 |
def __init__(self, stream, tok, seq_len, max_blocks=50000, max_rows=None,
|
| 1140 |
enforce_mix=False, lang_mix=None, mix_seed=42, mix_scan_mult=5,
|
| 1141 |
mix_max_rows=2000000):
|
| 1142 |
+
self.stream = stream
|
| 1143 |
+
self.tok = tok
|
| 1144 |
+
self.seq_len = seq_len
|
| 1145 |
+
self.max_blocks = max_blocks
|
| 1146 |
+
self.max_rows = max_rows
|
| 1147 |
+
self.enforce_mix = enforce_mix
|
| 1148 |
+
self.lang_mix = lang_mix
|
| 1149 |
+
self.mix_seed = mix_seed
|
| 1150 |
+
self.mix_scan_mult = mix_scan_mult
|
| 1151 |
+
self.mix_max_rows = mix_max_rows
|
| 1152 |
+
self.blocks = torch.zeros((0, seq_len), dtype=torch.long)
|
| 1153 |
+
self.stream_exhausted = False
|
| 1154 |
+
self.refill_count = 0
|
| 1155 |
+
self._remainder = {"hinglish": [], "hindi": [], "english": []} # carry-over tokens across refill()
|
| 1156 |
+
self.refill()
|
| 1157 |
+
|
| 1158 |
+
def set_seq_len(self, new_seq_len):
|
| 1159 |
+
"""Dynamically adjust block size for curriculum training.
|
| 1160 |
+
Re-chunks unconsumed tokens and remainder buffer without loss."""
|
| 1161 |
+
if new_seq_len == self.seq_len:
|
| 1162 |
+
return
|
| 1163 |
+
self.seq_len = new_seq_len
|
| 1164 |
+
# Collect unconsumed blocks back into remainder buffer
|
| 1165 |
+
if hasattr(self, "blocks") and len(self.blocks) > 0:
|
| 1166 |
+
unconsumed = self.blocks.flatten().tolist()
|
| 1167 |
+
self._remainder["english"].extend(unconsumed)
|
| 1168 |
+
self.blocks = torch.zeros((0, self.seq_len), dtype=torch.long)
|
| 1169 |
+
self.refill()
|
| 1170 |
+
|
| 1171 |
+
def refill(self):
|
| 1172 |
+
if self.stream_exhausted:
|
| 1173 |
+
return 0
|
| 1174 |
+
self.refill_count += 1
|
| 1175 |
+
buckets = {"hinglish": list(self._remainder.get("hinglish", [])),
|
| 1176 |
+
"hindi": list(self._remainder.get("hindi", [])),
|
| 1177 |
+
"english": list(self._remainder.get("english", []))}
|
| 1178 |
+
eos = self.tok.token_to_id("<eos>")
|
| 1179 |
if eos is None:
|
| 1180 |
eos = 2
|
| 1181 |
+
cap_tok = self.max_blocks * self.seq_len
|
| 1182 |
+
weights = {k: float((self.lang_mix or {}).get(k, 0)) for k in buckets}
|
| 1183 |
if sum(weights.values()) <= 0:
|
| 1184 |
weights = {"hinglish": 0.4, "hindi": 0.3, "english": 0.3}
|
| 1185 |
tot_w = sum(weights.values())
|
| 1186 |
quota = {k: cap_tok * w / tot_w for k, w in weights.items()}
|
| 1187 |
+
scan_cap = cap_tok * (self.mix_scan_mult if self.enforce_mix else 1)
|
| 1188 |
n_rows = n_kept = n_safe_skip = n_empty = n_noschema = 0
|
| 1189 |
lang_rows = {"hinglish": 0, "hindi": 0, "english": 0}
|
| 1190 |
+
rows_found = False
|
| 1191 |
+
for row in self.stream:
|
| 1192 |
+
rows_found = True
|
| 1193 |
n_rows += 1
|
| 1194 |
+
if self.max_rows and n_rows > self.max_rows:
|
| 1195 |
break
|
| 1196 |
txt = row.get("text", "") if isinstance(row, dict) else ""
|
| 1197 |
if not txt or not str(txt).strip():
|
| 1198 |
n_empty += 1
|
| 1199 |
continue
|
| 1200 |
+
txt = str(txt).strip()
|
| 1201 |
+
# Auditor filter: skip degenerate colon strings (e.g. NCERT colon scrapes ': : : : :')
|
| 1202 |
+
if len(txt) < 5 or (len(txt) < 40 and txt.count(':') >= 3 and txt.count(':') > len(txt) / 3):
|
| 1203 |
+
n_empty += 1
|
| 1204 |
+
continue
|
| 1205 |
if not _safety_ok(row if isinstance(row, dict) else {}):
|
| 1206 |
n_safe_skip += 1
|
| 1207 |
continue
|
|
|
|
| 1208 |
try:
|
| 1209 |
tag = get_lang_tag(row.get("lang") or row.get("source") or txt)
|
| 1210 |
except Exception:
|
| 1211 |
n_noschema += 1
|
| 1212 |
continue
|
| 1213 |
bucket = "hinglish" if "hinglish" in tag else ("hindi" if "hindi" in tag else "english")
|
| 1214 |
+
if self.enforce_mix and len(buckets[bucket]) >= quota[bucket]:
|
|
|
|
| 1215 |
n_kept += 1
|
| 1216 |
if all(len(buckets[k]) >= quota[k] for k in buckets):
|
| 1217 |
break
|
| 1218 |
+
if n_rows >= (self.max_rows or self.mix_max_rows):
|
| 1219 |
break
|
| 1220 |
continue
|
| 1221 |
+
tag_id = self.tok.token_to_id(tag) if self.tok is not None else None
|
| 1222 |
if tag_id is not None:
|
| 1223 |
buckets[bucket].append(tag_id)
|
| 1224 |
+
buckets[bucket].extend(self.tok.encode(txt).ids)
|
| 1225 |
buckets[bucket].append(eos)
|
| 1226 |
lang_rows[bucket] += 1
|
| 1227 |
n_kept += 1
|
| 1228 |
total_tok = sum(len(v) for v in buckets.values())
|
| 1229 |
+
if self.enforce_mix:
|
|
|
|
| 1230 |
if all(len(buckets[k]) >= quota[k] for k in buckets):
|
| 1231 |
break
|
| 1232 |
+
if n_rows >= (self.max_rows or self.mix_max_rows):
|
| 1233 |
break
|
| 1234 |
if total_tok >= scan_cap:
|
| 1235 |
break
|
| 1236 |
elif total_tok >= cap_tok:
|
| 1237 |
break
|
| 1238 |
+
|
| 1239 |
+
total_read = sum(len(v) for v in buckets.values())
|
| 1240 |
+
if not rows_found or total_read == 0:
|
| 1241 |
+
self.stream_exhausted = True
|
| 1242 |
+
return 0
|
| 1243 |
+
|
| 1244 |
if R0:
|
| 1245 |
+
print(f"[hf] chunk {self.refill_count}: scanned={n_rows:,} kept={n_kept:,} mix_rows={lang_rows} "
|
| 1246 |
f"skip_empty={n_empty:,} skip_safety({safety_mode})={n_safe_skip:,} skip_schema={n_noschema:,}")
|
| 1247 |
import random as _rng
|
| 1248 |
+
if self.enforce_mix:
|
|
|
|
| 1249 |
per_lang_blocks = {}
|
| 1250 |
for k, v in buckets.items():
|
| 1251 |
arr = torch.tensor(v, dtype=torch.long)
|
| 1252 |
+
n = (len(arr) // self.seq_len) * self.seq_len
|
| 1253 |
+
self._remainder[k] = arr[n:].tolist() # carry over tail tokens
|
| 1254 |
+
per_lang_blocks[k] = arr[:n].view(-1, self.seq_len) if n > 0 else torch.zeros((0, self.seq_len), dtype=torch.long)
|
| 1255 |
avail = {k: len(v) for k, v in per_lang_blocks.items()}
|
| 1256 |
+
quota_b = {k: int(quota[k] // self.seq_len) for k in avail}
|
| 1257 |
take = {k: min(avail[k], quota_b[k]) for k in avail}
|
| 1258 |
+
short = self.max_blocks - sum(take.values())
|
| 1259 |
+
if short > 0:
|
| 1260 |
left = {k: avail[k] - take[k] for k in avail}
|
| 1261 |
tot_left = sum(left.values()) or 1
|
| 1262 |
for k in avail:
|
|
|
|
| 1264 |
take[k] += extra
|
| 1265 |
chosen = [per_lang_blocks[k][:take[k]] for k in avail if take[k] > 0]
|
| 1266 |
if not chosen:
|
| 1267 |
+
self.blocks = torch.zeros((1, self.seq_len), dtype=torch.long)
|
| 1268 |
else:
|
| 1269 |
allb = torch.cat(chosen, dim=0)
|
| 1270 |
idx = list(range(len(allb)))
|
| 1271 |
+
_rng.Random(self.mix_seed + self.refill_count).shuffle(idx)
|
| 1272 |
self.blocks = allb[idx]
|
| 1273 |
if R0:
|
| 1274 |
print(f"[hf] mix ENFORCED target={weights}: blocks_taken={take} total={len(self.blocks)} (avail={avail})")
|
| 1275 |
else:
|
| 1276 |
+
# Save per-lang remainders before merging
|
| 1277 |
+
for k in ["hinglish", "hindi", "english"]:
|
| 1278 |
+
arr_k = torch.tensor(buckets[k], dtype=torch.long)
|
| 1279 |
+
n_k = (len(arr_k) // self.seq_len) * self.seq_len
|
| 1280 |
+
self._remainder[k] = arr_k[n_k:].tolist()
|
| 1281 |
ids = buckets["hinglish"] + buckets["hindi"] + buckets["english"]
|
| 1282 |
if R0:
|
| 1283 |
+
print(f"[hf] mix NATURAL (chunk {self.refill_count}): prefetched {len(ids):,} tokens -> {len(ids)//max(self.seq_len,1):,} blocks")
|
| 1284 |
ids = torch.tensor(ids, dtype=torch.long)
|
| 1285 |
+
n = (len(ids) // self.seq_len) * self.seq_len
|
| 1286 |
+
self.blocks = ids[:n].view(-1, self.seq_len) if n > 0 else torch.zeros((1, self.seq_len), dtype=torch.long)
|
| 1287 |
+
return len(self.blocks)
|
| 1288 |
+
|
| 1289 |
+
def __len__(self):
|
| 1290 |
+
return len(self.blocks)
|
| 1291 |
+
|
| 1292 |
+
def __getitem__(self, i):
|
| 1293 |
+
x = self.blocks[i]
|
| 1294 |
+
return x[:-1], x[1:]
|
| 1295 |
ds = HFStreamDataset(hf_ds, tok, seq_len+1, max_blocks=max_blocks_cfg, max_rows=max_rows_cfg,
|
| 1296 |
enforce_mix=enforce_mix, lang_mix=lang_mix, mix_seed=seed_r,
|
| 1297 |
mix_scan_mult=mix_scan_mult, mix_max_rows=mix_max_rows)
|
|
|
|
| 1348 |
ids.append(eos)
|
| 1349 |
import torch as _torch
|
| 1350 |
ids_t = _torch.tensor(ids, dtype=_torch.long)
|
|
|
|
|
|
|
| 1351 |
class _DummyDS(_torch.utils.data.Dataset):
|
| 1352 |
+
def __init__(self, ids_t, seq_len):
|
| 1353 |
+
self.ids_t = ids_t
|
| 1354 |
+
self.seq_len = seq_len
|
| 1355 |
+
self.rebuild()
|
| 1356 |
+
def rebuild(self):
|
| 1357 |
+
n = (len(self.ids_t) // self.seq_len) * self.seq_len
|
| 1358 |
+
self.blocks = self.ids_t[:n].view(-1, self.seq_len) if n > 0 else _torch.zeros((1, self.seq_len), dtype=_torch.long)
|
| 1359 |
+
def set_seq_len(self, new_seq_len):
|
| 1360 |
+
if new_seq_len == self.seq_len:
|
| 1361 |
+
return
|
| 1362 |
+
self.seq_len = new_seq_len
|
| 1363 |
+
self.rebuild()
|
| 1364 |
def __len__(self): return len(self.blocks)
|
| 1365 |
def __getitem__(self, i):
|
| 1366 |
x = self.blocks[i]; return x[:-1], x[1:]
|
| 1367 |
+
ds = _DummyDS(ids_t, seq_len + 1)
|
| 1368 |
elif ds is not None:
|
| 1369 |
pass # ds already built via HF streaming above
|
| 1370 |
else:
|
|
|
|
| 1383 |
if len(ds) > n_train:
|
| 1384 |
train_ids = all_idx[:n_train]
|
| 1385 |
val_ids = all_idx[n_train:]
|
| 1386 |
+
class _FrozenValDS(Dataset):
|
| 1387 |
+
def __init__(self, blocks): self.blocks = blocks
|
| 1388 |
+
def __len__(self): return len(self.blocks)
|
| 1389 |
+
def __getitem__(self, i):
|
| 1390 |
+
x = self.blocks[i]; return x[:-1], x[1:]
|
| 1391 |
+
val_ds = _FrozenValDS(ds.blocks[val_ids].clone())
|
| 1392 |
else:
|
| 1393 |
train_ids = all_idx
|
| 1394 |
val_ds = None
|
|
|
|
| 1430 |
+ (" [dist broadcast r0->all]" if (use_dist and not args.smoke) else "")) if R0 else None
|
| 1431 |
elif not args.smoke and device.type == "cuda" and R0:
|
| 1432 |
print(f"[batch] MANUAL LOCK: micro={batch} accum={accum} eff={batch * accum} tok/step={batch * accum * seq_len * (world if use_dist else 1)} amp={'bf16' if use_bf16 else 'fp16'}")
|
| 1433 |
+
# 2026 speed stack (opt-in, DDP-wrap se pehle): FP8 phir compile
|
| 1434 |
+
if not args.smoke:
|
| 1435 |
+
model, _fp8info = maybe_apply_fp8(model, bool(args.fp8 or C("fp8", False)), device, R0)
|
| 1436 |
+
model, _cmpinfo = maybe_apply_compile(model, bool(args.compile or C("compile", False)),
|
| 1437 |
+
str(C("compile_mode", "reduce-overhead")), args.smoke, R0,
|
| 1438 |
+
device=device, vocab_size=margs.vocab_size)
|
| 1439 |
if use_dist and not args.smoke:
|
| 1440 |
# MoE top-2 routing me kuch experts kabhi-kabhi empty rehte hain -> find_unused_parameters=True (safe)
|
| 1441 |
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[device.index], find_unused_parameters=True)
|
|
|
|
| 1446 |
if R0:
|
| 1447 |
print(f"[model] total={total/1e6:.1f}M active={active/1e6:.1f}M device={device}" + (f" (DDP x{world})" if use_dist else ""))
|
| 1448 |
|
| 1449 |
+
tok_per_step = batch * accum * seq_len * (world if use_dist else 1)
|
| 1450 |
+
if (max_steps is None or max_steps <= 0) and not args.smoke:
|
| 1451 |
+
total_tokens = int(C("total_tokens", 126_500_000_000))
|
| 1452 |
+
max_steps = max(1, total_tokens // tok_per_step)
|
| 1453 |
+
if R0:
|
| 1454 |
+
print(f"[auto] max_steps dynamically computed from dataset ({total_tokens/1e9:.1f}B tokens / {tok_per_step:,} tok/step): {max_steps:,} steps")
|
| 1455 |
+
|
| 1456 |
decay_params = [p for p in model.parameters() if p.requires_grad and p.ndim >= 2]
|
| 1457 |
no_decay_params = [p for p in model.parameters() if p.requires_grad and p.ndim < 2]
|
| 1458 |
optim_groups = [
|
| 1459 |
{"params": decay_params, "weight_decay": wd},
|
| 1460 |
{"params": no_decay_params, "weight_decay": 0.0},
|
| 1461 |
]
|
| 1462 |
+
|
| 1463 |
+
if bool(C("grad_ckpt", False)):
|
| 1464 |
+
target = model.module if hasattr(model, "module") else model
|
| 1465 |
+
if hasattr(target, "gradient_checkpointing_enable"):
|
| 1466 |
+
target.gradient_checkpointing_enable()
|
| 1467 |
+
if R0:
|
| 1468 |
+
print("[model] gradient checkpointing enabled (VRAM activation memory halved for 48L)")
|
| 1469 |
+
elif R0:
|
| 1470 |
+
print("[model] WARN: grad_ckpt requested but model has no gradient_checkpointing_enable()")
|
| 1471 |
+
|
| 1472 |
+
opt_name = str(args.optimizer or C("optimizer", "adamw")).lower()
|
| 1473 |
+
if opt_name == "muon":
|
| 1474 |
+
muon_p, adamw_p = split_muon_params(model)
|
| 1475 |
+
muon_lr = float(args.muon_lr if getattr(args, "muon_lr", None) is not None else C("muon_lr", 0.02))
|
| 1476 |
+
muon_wd = float(args.muon_weight_decay if getattr(args, "muon_weight_decay", None) is not None else C("muon_weight_decay", 0.01))
|
| 1477 |
+
opt_muon = Muon(muon_p, lr=muon_lr, momentum=float(C("muon_momentum", 0.95)), weight_decay=muon_wd)
|
| 1478 |
+
adamw_groups = [
|
| 1479 |
+
{"params": [p for p in adamw_p if p.ndim >= 2], "weight_decay": wd},
|
| 1480 |
+
{"params": [p for p in adamw_p if p.ndim < 2], "weight_decay": 0.0},
|
| 1481 |
+
]
|
| 1482 |
+
if device.type == "cuda":
|
| 1483 |
+
try:
|
| 1484 |
+
import bitsandbytes as bnb
|
| 1485 |
+
opt_adamw = bnb.optim.PagedAdamW8bit(adamw_groups, lr=lr, betas=tuple(C("betas", [0.9, 0.95])))
|
| 1486 |
+
if R0:
|
| 1487 |
+
print(f"[opt] Muon ({len(muon_p)} 2D weights, lr={muon_lr}, wd={muon_wd}) + PagedAdamW8bit ({len(adamw_p)} 1D/emb/router, lr={lr}, wd={wd}) hybrid loaded!")
|
| 1488 |
+
except ImportError:
|
| 1489 |
+
opt_adamw = torch.optim.AdamW(adamw_groups, lr=lr, betas=tuple(C("betas", [0.9, 0.95])))
|
| 1490 |
+
if R0:
|
| 1491 |
+
print(f"[opt] Muon ({len(muon_p)} 2D weights, lr={muon_lr}, wd={muon_wd}) + AdamW ({len(adamw_p)} 1D/emb/router, lr={lr}, wd={wd}) hybrid loaded!")
|
| 1492 |
+
else:
|
| 1493 |
+
opt_adamw = torch.optim.AdamW(adamw_groups, lr=lr, betas=tuple(C("betas", [0.9, 0.95])))
|
| 1494 |
+
if R0:
|
| 1495 |
+
print(f"[opt] Muon ({len(muon_p)} 2D weights, lr={muon_lr}, wd={muon_wd}) + AdamW ({len(adamw_p)} 1D/emb/router, lr={lr}, wd={wd}) hybrid loaded (CPU)!")
|
| 1496 |
+
opt = CombinedOptimizer([opt_muon, opt_adamw])
|
| 1497 |
+
elif opt_name in ("8bit", "paged_adamw_8bit", "adamw8bit") and device.type == "cuda":
|
| 1498 |
+
try:
|
| 1499 |
+
import bitsandbytes as bnb
|
| 1500 |
+
opt = bnb.optim.PagedAdamW8bit(
|
| 1501 |
+
optim_groups, lr=lr, betas=tuple(C("betas", [0.9, 0.95])) if not args.smoke else (0.9, 0.95)
|
| 1502 |
+
)
|
| 1503 |
+
if R0:
|
| 1504 |
+
print("[opt] loaded bitsandbytes PagedAdamW8bit (VRAM memory optimized for 1.5B)")
|
| 1505 |
+
except ImportError:
|
| 1506 |
+
if R0:
|
| 1507 |
+
print("[opt] bitsandbytes not found, falling back to standard AdamW")
|
| 1508 |
+
opt = torch.optim.AdamW(optim_groups, lr=lr, betas=tuple(C("betas", [0.9, 0.95])) if not args.smoke else (0.9, 0.95))
|
| 1509 |
+
else:
|
| 1510 |
+
opt = torch.optim.AdamW(optim_groups, lr=lr, betas=tuple(C("betas", [0.9, 0.95])) if not args.smoke else (0.9, 0.95))
|
| 1511 |
use_amp = use_bf16 or use_fp16
|
| 1512 |
amp_dtype = torch.bfloat16 if use_bf16 else torch.float16
|
| 1513 |
scaler = torch.amp.GradScaler("cuda") if (use_fp16 and device.type == "cuda") else None
|
|
|
|
| 1587 |
start = load_ckpt(resolved_ckpt, ckpt_model, opt, scaler=scaler)
|
| 1588 |
if R0:
|
| 1589 |
print(f"[resume] successfully resumed at step {start} from {resolved_ckpt}")
|
| 1590 |
+
if use_curriculum:
|
| 1591 |
+
new_stage_idx, new_stage = resolve_curriculum_stage(start, curriculum_stages)
|
| 1592 |
+
if new_stage_idx != curr_stage_idx:
|
| 1593 |
+
curr_stage_idx = new_stage_idx
|
| 1594 |
+
seq_len = int(new_stage["seq_len"])
|
| 1595 |
+
batch = int(args.batch_size or new_stage["batch_size"])
|
| 1596 |
+
accum = int(args.grad_accum or new_stage["grad_accum"])
|
| 1597 |
+
if use_dist and not args.smoke:
|
| 1598 |
+
accum = max(1, accum // world)
|
| 1599 |
+
if hasattr(ds, "set_seq_len"):
|
| 1600 |
+
ds.set_seq_len(seq_len + 1)
|
| 1601 |
+
train_ids = list(range(len(ds)))
|
| 1602 |
+
if use_dist and not args.smoke:
|
| 1603 |
+
train_ids = train_ids[rank::world]
|
| 1604 |
+
if not train_ids:
|
| 1605 |
+
train_ids = [0]
|
| 1606 |
+
rng.shuffle(train_ids)
|
| 1607 |
+
idx = 0
|
| 1608 |
+
tok_per_step = batch * accum * seq_len * (world if use_dist else 1)
|
| 1609 |
+
if R0:
|
| 1610 |
+
print(f"[curriculum] Resumed into Stage {curr_stage_idx + 1}/{len(curriculum_stages)}: "
|
| 1611 |
+
f"seq_len={seq_len} batch={batch} accum={accum} (tok/step={tok_per_step:,})", flush=True)
|
| 1612 |
else:
|
| 1613 |
if R0:
|
| 1614 |
print(f"[resume] checkpoint '{resume}' not found locally or on HF Hub — starting fresh from step 0")
|
|
|
|
| 1636 |
t0 = time.time()
|
| 1637 |
step = start
|
| 1638 |
idx = 0
|
| 1639 |
+
nan_streak = 0
|
| 1640 |
+
max_nan_streak = int(C("max_nan_streak", 25))
|
| 1641 |
+
usage_log_every = int(C("usage_log_every", 100))
|
| 1642 |
tok_per_step = batch * accum * seq_len * (world if use_dist else 1) # DDP: global tokens/step
|
| 1643 |
pbar = None # Industry standard: clean stdout lines (no carriage-return progress bar in web terminal)
|
|
|
|
| 1644 |
while step < max_steps:
|
| 1645 |
+
# Check Automatic Curriculum Upgrade
|
| 1646 |
+
if use_curriculum:
|
| 1647 |
+
new_stage_idx, new_stage = resolve_curriculum_stage(step, curriculum_stages)
|
| 1648 |
+
if new_stage_idx != curr_stage_idx:
|
| 1649 |
+
old_seq = seq_len
|
| 1650 |
+
curr_stage_idx = new_stage_idx
|
| 1651 |
+
seq_len = int(new_stage["seq_len"])
|
| 1652 |
+
batch = int(args.batch_size or new_stage["batch_size"])
|
| 1653 |
+
accum = int(args.grad_accum or new_stage["grad_accum"])
|
| 1654 |
+
if use_dist and not args.smoke:
|
| 1655 |
+
accum = max(1, accum // world)
|
| 1656 |
+
tok_per_step = batch * accum * seq_len * (world if use_dist else 1)
|
| 1657 |
+
|
| 1658 |
+
if hasattr(ds, "set_seq_len"):
|
| 1659 |
+
ds.set_seq_len(seq_len + 1)
|
| 1660 |
+
train_ids = list(range(len(ds)))
|
| 1661 |
+
if use_dist and not args.smoke:
|
| 1662 |
+
train_ids = train_ids[rank::world]
|
| 1663 |
+
if not train_ids:
|
| 1664 |
+
train_ids = [0]
|
| 1665 |
+
rng.shuffle(train_ids)
|
| 1666 |
+
idx = 0
|
| 1667 |
+
|
| 1668 |
+
if R0:
|
| 1669 |
+
print("\n" + "=" * 80)
|
| 1670 |
+
print(f"[CURRICULUM UPGRADE] Step {step:,}: Upgraded to Stage {curr_stage_idx + 1}/{len(curriculum_stages)}!")
|
| 1671 |
+
print(f" * Context Length: {old_seq} -> {seq_len} tokens")
|
| 1672 |
+
print(f" * Micro Batch Size: {batch}")
|
| 1673 |
+
print(f" * Grad Accumulation: {accum}")
|
| 1674 |
+
print(f" * Global Tok/Step: {tok_per_step:,} (Constant Token Budget Maintained)")
|
| 1675 |
+
print("=" * 80 + "\n", flush=True)
|
| 1676 |
+
|
| 1677 |
if idx >= len(train_ids):
|
| 1678 |
+
if hasattr(ds, "refill") and not args.smoke:
|
| 1679 |
+
refilled = ds.refill()
|
| 1680 |
+
if refilled > 0:
|
| 1681 |
+
train_ids = list(range(len(ds)))
|
| 1682 |
+
if use_dist and not args.smoke:
|
| 1683 |
+
train_ids = train_ids[rank::world]
|
| 1684 |
+
if not train_ids:
|
| 1685 |
+
train_ids = [0]
|
| 1686 |
+
rng.shuffle(train_ids)
|
| 1687 |
+
idx = 0
|
| 1688 |
+
if R0:
|
| 1689 |
+
print(f"[stream] Buffer dynamically refilled: loaded {len(ds):,} fresh blocks from 125B+ token stream (step {step:,})", flush=True)
|
| 1690 |
+
else:
|
| 1691 |
+
if getattr(ds, "stream_exhausted", False):
|
| 1692 |
+
if R0:
|
| 1693 |
+
print(f"[stream] All 2,109 dataset shards processed! Stream complete at step {step:,}. Exiting to save final checkpoint...", flush=True)
|
| 1694 |
+
break
|
| 1695 |
+
rng.shuffle(train_ids)
|
| 1696 |
+
idx = 0
|
| 1697 |
+
else:
|
| 1698 |
+
rng.shuffle(train_ids)
|
| 1699 |
+
idx = 0
|
| 1700 |
# accumulate (batch_size fix: har micro-step me batch samples stack karo)
|
| 1701 |
opt.zero_grad(set_to_none=True)
|
| 1702 |
acc_loss = 0.0
|
|
|
|
| 1741 |
layer_avg = [sum(acc_usages[s][l_idx][e] for s in range(len(acc_usages))) / len(acc_usages) for e in range(n_exp)]
|
| 1742 |
avg_usages.append(layer_avg)
|
| 1743 |
aux = {"moe_aux": round(acc_moe_aux, 4), "mt_loss": round(acc_mt_loss, 4), "usages": avg_usages}
|
| 1744 |
+
# NaN/Inf guard: poisoned step ko skip karo (optimizer state bachao).
|
| 1745 |
+
# DDP me decision GLOBAL hoti hai (kisi ek rank pe NaN => sab skip),
|
| 1746 |
+
# warna ranks ke params desync ho jayenge. Streak badhti jaye to
|
| 1747 |
+
# divergence maan ke emergency ckpt + sab ranks ek saath ruko.
|
| 1748 |
+
local_bad = not math.isfinite(acc_loss)
|
| 1749 |
+
do_skip = local_bad
|
| 1750 |
+
if use_dist and not args.smoke:
|
| 1751 |
+
try:
|
| 1752 |
+
import torch.distributed as _dist
|
| 1753 |
+
_t = torch.tensor([1 if local_bad else 0], device=device)
|
| 1754 |
+
_dist.all_reduce(_t, op=_dist.ReduceOp.MAX)
|
| 1755 |
+
do_skip = bool(_t.item())
|
| 1756 |
+
except Exception:
|
| 1757 |
+
pass
|
| 1758 |
+
if do_skip:
|
| 1759 |
+
nan_streak += 1
|
| 1760 |
+
opt.zero_grad(set_to_none=True)
|
| 1761 |
+
if R0:
|
| 1762 |
+
print(f"[nan-guard] step {step+1}: non-finite loss ({acc_loss}) — update skipped (streak {nan_streak}/{max_nan_streak})", flush=True)
|
| 1763 |
+
if logf is not None:
|
| 1764 |
+
logf.write(json.dumps({"step": step + 1, "skipped": True, "loss": str(acc_loss),
|
| 1765 |
+
"nan_streak": nan_streak}) + "\n")
|
| 1766 |
+
logf.flush()
|
| 1767 |
+
if nan_streak >= max_nan_streak and not args.smoke:
|
| 1768 |
+
if R0:
|
| 1769 |
+
print(f"[nan-guard] DIVERGENCE: {nan_streak} consecutive NaN steps — emergency ckpt + abort", flush=True)
|
| 1770 |
+
save_ckpt(Path(ckpt_dir) / f"emergency_nan_{step+1}.pt", ckpt_model, opt, step, cfg, scaler=scaler)
|
| 1771 |
+
break
|
| 1772 |
+
step += 1
|
| 1773 |
+
continue
|
| 1774 |
+
nan_streak = 0
|
| 1775 |
if scaler is not None:
|
| 1776 |
scaler.unscale_(opt)
|
| 1777 |
+
total_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), clip)
|
| 1778 |
+
try:
|
| 1779 |
+
total_norm = float(total_norm)
|
| 1780 |
+
except Exception:
|
| 1781 |
+
total_norm = -1.0
|
| 1782 |
cur_lr = cosine_lr(step + 1, max_steps, warmup, lr, min_lr)
|
| 1783 |
+
lr_ratio = cur_lr / max(lr, 1e-9)
|
| 1784 |
for g in opt.param_groups:
|
| 1785 |
+
if "initial_lr" not in g:
|
| 1786 |
+
g["initial_lr"] = g["lr"]
|
| 1787 |
+
g["lr"] = g["initial_lr"] * lr_ratio
|
| 1788 |
if scaler is not None:
|
| 1789 |
scaler.step(opt)
|
| 1790 |
scaler.update()
|
|
|
|
| 1807 |
max_expert_share = round(max(avg_u), 3)
|
| 1808 |
if max_expert_share > 0.60:
|
| 1809 |
print(f"[warn] ROUTER IMBALANCE: max expert share {max_expert_share} > 0.60!", flush=True)
|
| 1810 |
+
# Full 48x28 usage matrix sirf kabhi-kabhi log karo (log-bloat bachao);
|
| 1811 |
+
# har line me max_expert_share + grad_norm kaafi hai.
|
| 1812 |
+
aux_log = dict(aux) if isinstance(aux, dict) else aux
|
| 1813 |
+
if isinstance(aux_log, dict) and step % usage_log_every != 0 and step > 5 and step != max_steps:
|
| 1814 |
+
aux_log = dict(aux_log)
|
| 1815 |
+
aux_log["usages"] = []
|
| 1816 |
+
msg = {"step": step, "of": max_steps, "pct": round(pct, 2), "seq_len": seq_len, "loss": round(acc_loss, 4),
|
| 1817 |
+
"lr": cur_lr, "grad_norm": round(total_norm, 3), "tok_per_sec": round(tps, 1),
|
| 1818 |
+
"elapsed_sec": round(dt, 1), "eta_sec": round(eta, 1), "aux": aux_log}
|
| 1819 |
if max_expert_share is not None:
|
| 1820 |
msg["max_expert_share"] = max_expert_share
|
| 1821 |
+
if use_curriculum:
|
| 1822 |
+
msg["curriculum_stage"] = curr_stage_idx + 1
|
| 1823 |
# Essential clean log line
|
| 1824 |
+
cur_tag = f"seq: {seq_len} (S{curr_stage_idx+1})" if use_curriculum else f"seq: {seq_len}"
|
| 1825 |
+
print(f"[step {step:,}/{max_steps:,}] ({pct:.2f}%) | {cur_tag} | loss: {acc_loss:.4f} | gn: {total_norm:.2f} | lr: {cur_lr:.2e} | {tps:,.0f} tok/s | ETA: {fmt_hms(eta)}", flush=True)
|
| 1826 |
if logf is not None:
|
| 1827 |
logf.write(json.dumps(msg) + "\n")
|
| 1828 |
logf.flush()
|