fix: DDP auto_tune probe-before-wrap + broadcast micro/accum, R0-only logs, empty-shard guard, safety import path
Browse files- docs/PROGRESS.md +9 -0
- model/scripts/train.py +76 -39
- tokenizer/outputs/tokenizer.json +3 -0
- tokenizer/train_tokenizer_colab.ipynb +30 -25
docs/PROGRESS.md
CHANGED
|
@@ -333,7 +333,16 @@ Har change isi order me hoga — koi step skip nahi:
|
|
| 333 |
2. **HF PUSH** — `ViuAI/ViuMini-MoE-242M` (model repo: code) ya `ViuAI/viu-mini-raw-pretrain` (dataset repo: data/card) par upload karo. Bina push ke Kaggle par purana code chalega.
|
| 334 |
3. **KAGGLE RUN** — har session start me bootstrap cell (snapshot_download from HF model repo) se fresh code lao, phir run karo. Restart-proof workflow.
|
| 335 |
Kaggle par manual file upload se version mismatch hota hai (v1 tokenizer wala case). Isme koi exception nahi hai.
|
|
|
|
|
|
|
| 336 |
- [x] 2026-09-24 v1 tokenizer DELETED (user order): HF model repo se `tokenizer/outputs/tokenizer.json` (v1, 4.23MB, mask typo) hataya — ab galti se v1 use impossible. v2 sirf dataset repo (`tokenizer/tokenizer.json`, 4.06MB) + local me hai. `train_config.yaml` me v2 download instruction added. Ye notes + config HF push pending (agla sync me jayega).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 337 |
|
| 338 |
## Kaise note likhna hai
|
| 339 |
```
|
|
|
|
| 333 |
2. **HF PUSH** — `ViuAI/ViuMini-MoE-242M` (model repo: code) ya `ViuAI/viu-mini-raw-pretrain` (dataset repo: data/card) par upload karo. Bina push ke Kaggle par purana code chalega.
|
| 334 |
3. **KAGGLE RUN** — har session start me bootstrap cell (snapshot_download from HF model repo) se fresh code lao, phir run karo. Restart-proof workflow.
|
| 335 |
Kaggle par manual file upload se version mismatch hota hai (v1 tokenizer wala case). Isme koi exception nahi hai.
|
| 336 |
+
- [x] 2026-09-24 DDP multi-GPU (user: training fast karo): `train.py` me torchrun DDP — stride data shard (overlap zero), grad-sync, rank0-only ckpt/eval/log/push, ckpt bina `module.` prefix (resume-safe), global tok/step xworld. Smoke PASS (local, CPU). NOTE: real 2-GPU test local me possible nahi (no GPU) — Kaggle T4x2 par verify karna hai. HF push done.
|
| 337 |
+
- [x] 2026-09-24 DDP VERIFIED on Kaggle T4x2 (user screenshot): `[dist] rank 0/2 cuda:0 + rank 1/2 cuda:1`, grad-sync every step, dono ranks streaming. Speedup watching me hai (~1.8x expected).
|
| 338 |
- [x] 2026-09-24 v1 tokenizer DELETED (user order): HF model repo se `tokenizer/outputs/tokenizer.json` (v1, 4.23MB, mask typo) hataya — ab galti se v1 use impossible. v2 sirf dataset repo (`tokenizer/tokenizer.json`, 4.06MB) + local me hai. `train_config.yaml` me v2 download instruction added. Ye notes + config HF push pending (agla sync me jayega).
|
| 339 |
+
- [x] 2026-09-24 train.py BUG AUDIT + FIX (user: program me bug check karo):
|
| 340 |
+
- **CRITICAL**: auto_tune micro-batch probe DDP wrap ke BAAD chal raha tha — probe backward me allreduce hota hai; agar dono T4 ka OOM point alag ho to probe-step count mismatch → NCCL hang. FIX: probe AB raw model pe (wrap se pehle), phir rank0 ka (micro, accum) `dist.broadcast` se sab pe sync, uske baad DDP wrap.
|
| 341 |
+
- **HIGH**: `[train]` / `[data]` / `[tok]` / `[hf]` / `[done]` lines dono rank print karte the (interleaved duplicate logs). FIX: sab critical prints R0-only; early `rank/world` env se (startup se hi R0 sahi).
|
| 342 |
+
- **MED**: DDP stride-shard ke baad agar rank ko 0 blocks mile → training loop me `% 0` ZeroDivision crash. FIX: empty-shard guard (fallback 1 block + WARN).
|
| 343 |
+
- **MED**: `from data.scripts.filter_by_safety` import CWD-depend fail hota tha (silent fallback). FIX: repo root `sys.path` me dala.
|
| 344 |
+
- verify: syntax OK + `train.py --smoke` 3-step PASS (loss 6.31, MoE usages OK).
|
| 345 |
+
- NOTE: DDP probe+broadcast fix local CPU pe full-test nahi ho sakta — next Kaggle T4x2 run me `[auto] ... [dist broadcast r0->all]` line dikhni chahiye.
|
| 346 |
|
| 347 |
## Kaise note likhna hai
|
| 348 |
```
|
model/scripts/train.py
CHANGED
|
@@ -22,6 +22,8 @@ import torch.nn.functional as F
|
|
| 22 |
from torch.utils.data import Dataset
|
| 23 |
|
| 24 |
sys.path.insert(0, str(Path(__file__).parent))
|
|
|
|
|
|
|
| 25 |
from viu1_moe import ViuArgs, Viu1MoE, load_args as load_model_args
|
| 26 |
|
| 27 |
try:
|
|
@@ -349,8 +351,10 @@ def main():
|
|
| 349 |
return cand.resolve()
|
| 350 |
return (cfg_dir / pp).resolve()
|
| 351 |
|
| 352 |
-
#
|
| 353 |
-
|
|
|
|
|
|
|
| 354 |
|
| 355 |
if args.smoke:
|
| 356 |
print("[mode] SMOKE — tiny model, CPU, 3 steps")
|
|
@@ -387,19 +391,20 @@ def main():
|
|
| 387 |
device = pick_device()
|
| 388 |
# DDP: torchrun --nproc_per_node=2 se chalao; auto-detect bhi (WORLD_SIZE>1).
|
| 389 |
# NOTE: smoke me DDP kabhi nahi (CPU tiny test).
|
| 390 |
-
|
| 391 |
-
|
| 392 |
-
|
| 393 |
-
|
|
|
|
|
|
|
| 394 |
use_dist = False
|
|
|
|
| 395 |
if use_dist:
|
| 396 |
import torch.distributed as dist
|
| 397 |
-
local_rank = int(os.environ.get("LOCAL_RANK", "0"))
|
| 398 |
-
rank = int(os.environ.get("RANK", local_rank))
|
| 399 |
-
world = int(os.environ.get("WORLD_SIZE", "1"))
|
| 400 |
torch.cuda.set_device(local_rank)
|
| 401 |
device = torch.device(f"cuda:{local_rank}")
|
| 402 |
dist.init_process_group(backend="nccl", rank=rank, world_size=world)
|
|
|
|
| 403 |
print(f"[dist] rank {rank}/{world} on {device} (DDP, grad-sync every step)")
|
| 404 |
R0 = (rank == 0)
|
| 405 |
gname, ggb, bf16_ok = gpu_caps(device) if device.type == "cuda" else ("cpu", 0.0, False)
|
|
@@ -412,10 +417,12 @@ def main():
|
|
| 412 |
tok = None
|
| 413 |
if Tokenizer is not None and tok_path is not None and Path(str(tok_path)).exists():
|
| 414 |
tok = Tokenizer.from_file(str(tok_path))
|
| 415 |
-
|
|
|
|
| 416 |
elif not args.smoke:
|
| 417 |
-
|
| 418 |
-
|
|
|
|
| 419 |
return
|
| 420 |
if args.smoke or tok is None:
|
| 421 |
# smoke/dummy: sample data se chhota BPE on-the-fly? simple: train.py smoke me char-level dummy
|
|
@@ -427,11 +434,13 @@ def main():
|
|
| 427 |
ds = None
|
| 428 |
if hf_dataset and not args.smoke:
|
| 429 |
if tok is None:
|
| 430 |
-
|
|
|
|
| 431 |
return
|
| 432 |
try:
|
| 433 |
from datasets import load_dataset
|
| 434 |
-
|
|
|
|
| 435 |
hf_ds = load_dataset(hf_dataset, split=args.hf_split, streaming=True)
|
| 436 |
# Language-mix rule OPTIONAL (default OFF = natural Hub order).
|
| 437 |
# enforce_mix=true par weighted block sampler chalta hai (lang_mix weights se).
|
|
@@ -450,9 +459,10 @@ def main():
|
|
| 450 |
mix_max_rows = int(C("mix_max_rows", 2000000))
|
| 451 |
allow_toxic = safety_mode in ("research", "full")
|
| 452 |
allow_uncensored = safety_mode in ("full",)
|
| 453 |
-
|
| 454 |
-
|
| 455 |
-
|
|
|
|
| 456 |
try:
|
| 457 |
from data.scripts.filter_by_safety import keep_row as _keep
|
| 458 |
def _safety_ok(r):
|
|
@@ -533,8 +543,9 @@ def main():
|
|
| 533 |
break
|
| 534 |
elif total_tok >= cap_tok:
|
| 535 |
break
|
| 536 |
-
|
| 537 |
-
|
|
|
|
| 538 |
import random as _rng
|
| 539 |
if enforce_mix:
|
| 540 |
# Weighted block sampler: quota tak lo; kami ho to bachi avail se bharo.
|
|
@@ -561,10 +572,12 @@ def main():
|
|
| 561 |
idx = list(range(len(allb)))
|
| 562 |
_rng.Random(mix_seed).shuffle(idx)
|
| 563 |
self.blocks = allb[idx]
|
| 564 |
-
|
|
|
|
| 565 |
else:
|
| 566 |
ids = buckets["hinglish"] + buckets["hindi"] + buckets["english"]
|
| 567 |
-
|
|
|
|
| 568 |
ids = torch.tensor(ids, dtype=torch.long)
|
| 569 |
n = (len(ids)//seq_len)*seq_len
|
| 570 |
self.blocks = ids[:n].view(-1, seq_len) if n>0 else torch.zeros((1,seq_len),dtype=torch.long)
|
|
@@ -575,7 +588,8 @@ def main():
|
|
| 575 |
enforce_mix=enforce_mix, lang_mix=lang_mix, mix_seed=mix_seed,
|
| 576 |
mix_scan_mult=mix_scan_mult, mix_max_rows=mix_max_rows)
|
| 577 |
except Exception as e:
|
| 578 |
-
|
|
|
|
| 579 |
hf_dataset = None
|
| 580 |
ds = None
|
| 581 |
if args.smoke:
|
|
@@ -658,17 +672,19 @@ def main():
|
|
| 658 |
val_ds = None
|
| 659 |
if use_dist and not args.smoke:
|
| 660 |
train_ids = train_ids[rank::world]
|
| 661 |
-
|
| 662 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 663 |
|
| 664 |
model = Viu1MoE(margs).to(device)
|
| 665 |
-
|
| 666 |
-
|
| 667 |
-
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[device.index], find_unused_parameters=True)
|
| 668 |
-
print(f"[dist] rank {rank}: DDP wrapped")
|
| 669 |
-
ckpt_model = model.module if (use_dist and not args.smoke) else model # ckpt bina module. prefix ke (resume-safe)
|
| 670 |
-
total, active = ckpt_model.param_counts()
|
| 671 |
-
print(f"[model] total={total/1e6:.1f}M active={active/1e6:.1f}M device={device} (DDP x{world})" if use_dist else f"[model] total={total/1e6:.1f}M active={active/1e6:.1f}M device={device}")
|
| 672 |
if C("auto_tune", True) and not args.smoke and device.type == "cuda":
|
| 673 |
# eff-batch target same rakho, micro/accum GPU ke hisaab se baanto — har GPU pe same command
|
| 674 |
target_eff = batch * accum
|
|
@@ -677,8 +693,25 @@ def main():
|
|
| 677 |
micro = max(1, min(micro, target_eff))
|
| 678 |
accum = max(1, round(target_eff / micro))
|
| 679 |
batch = micro
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 680 |
print(f"[auto] {gname} ({ggb:.0f}GB, bf16_ok={bf16_ok}) amp={'bf16' if use_bf16 else 'fp16'} "
|
| 681 |
-
f"micro={batch} accum={accum} eff={batch * accum} (target {target_eff}) tok/step={batch * accum * seq_len}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 682 |
|
| 683 |
decay_params = [p for p in model.parameters() if p.requires_grad and p.ndim >= 2]
|
| 684 |
no_decay_params = [p for p in model.parameters() if p.requires_grad and p.ndim < 2]
|
|
@@ -695,16 +728,18 @@ def main():
|
|
| 695 |
resume = None if args.smoke else C("resume", None)
|
| 696 |
if resume and Path(resume).exists():
|
| 697 |
start = load_ckpt(resume, ckpt_model, opt, scaler=scaler)
|
| 698 |
-
|
|
|
|
| 699 |
# HF auto-push (har training ke baad ckpt backup mandatory) — token env HF_TOKEN se
|
| 700 |
do_hf_push = bool(C("hf_push", False)) and not args.smoke
|
| 701 |
hf_repo = C("hf_repo", None)
|
| 702 |
hf_token = os.environ.get("HF_TOKEN")
|
| 703 |
hf_every = int(C("hf_push_every", 0) or 0)
|
| 704 |
if do_hf_push and not hf_repo:
|
| 705 |
-
|
|
|
|
| 706 |
do_hf_push = False
|
| 707 |
-
if do_hf_push and not hf_token:
|
| 708 |
print("[hf-push] HF_TOKEN env me nahi — push skip hoga (token set karo ya manual cell chalao)")
|
| 709 |
|
| 710 |
log_dir = Path(log_dir)
|
|
@@ -782,7 +817,8 @@ def main():
|
|
| 782 |
pbar.update(1)
|
| 783 |
pbar.set_postfix(loss=f"{acc_loss:.4f}", lr=f"{cur_lr:.1e}", refresh=False)
|
| 784 |
|
| 785 |
-
|
|
|
|
| 786 |
dt = time.time() - t0
|
| 787 |
done = max(step - start, 1)
|
| 788 |
rate = done / max(dt, 1e-6)
|
|
@@ -804,7 +840,7 @@ def main():
|
|
| 804 |
share_str = f" max_share {max_expert_share}" if max_expert_share is not None else ""
|
| 805 |
print(f"[train] step {step}/{max_steps} ({pct:.1f}%) loss {acc_loss:.4f} lr {cur_lr:.2e} "
|
| 806 |
f"tok/s {tps:,.0f} elapsed {fmt_hms(dt)} ETA {fmt_hms(eta)}{share_str} aux {aux}", flush=True)
|
| 807 |
-
if
|
| 808 |
logf.write(json.dumps(msg) + "\n")
|
| 809 |
logf.flush()
|
| 810 |
|
|
@@ -833,8 +869,9 @@ def main():
|
|
| 833 |
pbar.close()
|
| 834 |
total_dt = time.time() - t0
|
| 835 |
total_done = max(step - start, 1)
|
| 836 |
-
|
| 837 |
-
|
|
|
|
| 838 |
if R0 and do_hf_push:
|
| 839 |
save_slim_ckpt(Path(ckpt_dir) / f"slim_final_{step}.pt", ckpt_model, cfg)
|
| 840 |
hf_push_ckpt(ckpt_dir, hf_repo, hf_token, only_file=Path(ckpt_dir) / f"slim_final_{step}.pt",
|
|
|
|
| 22 |
from torch.utils.data import Dataset
|
| 23 |
|
| 24 |
sys.path.insert(0, str(Path(__file__).parent))
|
| 25 |
+
# repo root: data.scripts.filter_by_safety import ke liye (Kaggle pe CWD alag hota hai)
|
| 26 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
| 27 |
from viu1_moe import ViuArgs, Viu1MoE, load_args as load_model_args
|
| 28 |
|
| 29 |
try:
|
|
|
|
| 351 |
return cand.resolve()
|
| 352 |
return (cfg_dir / pp).resolve()
|
| 353 |
|
| 354 |
+
# Early rank (startup prints R0-only chahte hain; DDP init non-smoke block me hoga)
|
| 355 |
+
rank = int(os.environ.get("RANK", "0"))
|
| 356 |
+
world = int(os.environ.get("WORLD_SIZE", "1"))
|
| 357 |
+
use_dist, R0 = False, (rank == 0)
|
| 358 |
|
| 359 |
if args.smoke:
|
| 360 |
print("[mode] SMOKE — tiny model, CPU, 3 steps")
|
|
|
|
| 391 |
device = pick_device()
|
| 392 |
# DDP: torchrun --nproc_per_node=2 se chalao; auto-detect bhi (WORLD_SIZE>1).
|
| 393 |
# NOTE: smoke me DDP kabhi nahi (CPU tiny test).
|
| 394 |
+
# rank/world env se hi rakho (early R0 ke saath consistent) — init ke waqt overwrite mat karo.
|
| 395 |
+
local_rank = int(os.environ.get("LOCAL_RANK", "0"))
|
| 396 |
+
use_dist = bool(args.dist or world > 1)
|
| 397 |
+
if use_dist and (not torch.cuda.is_available() or world < 2):
|
| 398 |
+
if R0:
|
| 399 |
+
print("[dist] WARN: multi-GPU env nahi (torchrun bina / 1 GPU) — single-GPU fallback")
|
| 400 |
use_dist = False
|
| 401 |
+
rank, world = 0, 1
|
| 402 |
if use_dist:
|
| 403 |
import torch.distributed as dist
|
|
|
|
|
|
|
|
|
|
| 404 |
torch.cuda.set_device(local_rank)
|
| 405 |
device = torch.device(f"cuda:{local_rank}")
|
| 406 |
dist.init_process_group(backend="nccl", rank=rank, world_size=world)
|
| 407 |
+
if use_dist and not args.smoke:
|
| 408 |
print(f"[dist] rank {rank}/{world} on {device} (DDP, grad-sync every step)")
|
| 409 |
R0 = (rank == 0)
|
| 410 |
gname, ggb, bf16_ok = gpu_caps(device) if device.type == "cuda" else ("cpu", 0.0, False)
|
|
|
|
| 417 |
tok = None
|
| 418 |
if Tokenizer is not None and tok_path is not None and Path(str(tok_path)).exists():
|
| 419 |
tok = Tokenizer.from_file(str(tok_path))
|
| 420 |
+
if R0:
|
| 421 |
+
print(f"[tok] loaded {tok_path} vocab={tok.get_vocab_size()}")
|
| 422 |
elif not args.smoke:
|
| 423 |
+
if R0:
|
| 424 |
+
print(f"[warn] tokenizer nahi mila ({tok_path}) — FINAL tokenizer Phase-2 me banega.")
|
| 425 |
+
print("[error] real training ke liye FINAL tokenizer chahiye. --smoke se test karo ya tokenizer train karo.")
|
| 426 |
return
|
| 427 |
if args.smoke or tok is None:
|
| 428 |
# smoke/dummy: sample data se chhota BPE on-the-fly? simple: train.py smoke me char-level dummy
|
|
|
|
| 434 |
ds = None
|
| 435 |
if hf_dataset and not args.smoke:
|
| 436 |
if tok is None:
|
| 437 |
+
if R0:
|
| 438 |
+
print("[error] HF dataset ke liye tokenizer chahiye — pehle tokenizer train karo.")
|
| 439 |
return
|
| 440 |
try:
|
| 441 |
from datasets import load_dataset
|
| 442 |
+
if R0:
|
| 443 |
+
print(f"[hf] streaming dataset {hf_dataset} split={args.hf_split}")
|
| 444 |
hf_ds = load_dataset(hf_dataset, split=args.hf_split, streaming=True)
|
| 445 |
# Language-mix rule OPTIONAL (default OFF = natural Hub order).
|
| 446 |
# enforce_mix=true par weighted block sampler chalta hai (lang_mix weights se).
|
|
|
|
| 459 |
mix_max_rows = int(C("mix_max_rows", 2000000))
|
| 460 |
allow_toxic = safety_mode in ("research", "full")
|
| 461 |
allow_uncensored = safety_mode in ("full",)
|
| 462 |
+
if R0:
|
| 463 |
+
print(f"[hf] safety={safety_mode} (toxic={allow_toxic}, uncensored={allow_uncensored}) "
|
| 464 |
+
f"max_blocks={max_blocks_cfg} max_rows={max_rows_cfg or 'unlimited'} "
|
| 465 |
+
f"enforce_mix={enforce_mix} lang_mix={lang_mix if enforce_mix else 'OFF (natural order)'}")
|
| 466 |
try:
|
| 467 |
from data.scripts.filter_by_safety import keep_row as _keep
|
| 468 |
def _safety_ok(r):
|
|
|
|
| 543 |
break
|
| 544 |
elif total_tok >= cap_tok:
|
| 545 |
break
|
| 546 |
+
if R0:
|
| 547 |
+
print(f"[hf] rows scanned={n_rows:,} kept={n_kept:,} mix_rows={lang_rows} "
|
| 548 |
+
f"skip_empty={n_empty:,} skip_safety({safety_mode})={n_safe_skip:,} skip_schema={n_noschema:,}")
|
| 549 |
import random as _rng
|
| 550 |
if enforce_mix:
|
| 551 |
# Weighted block sampler: quota tak lo; kami ho to bachi avail se bharo.
|
|
|
|
| 572 |
idx = list(range(len(allb)))
|
| 573 |
_rng.Random(mix_seed).shuffle(idx)
|
| 574 |
self.blocks = allb[idx]
|
| 575 |
+
if R0:
|
| 576 |
+
print(f"[hf] mix ENFORCED target={weights}: blocks_taken={take} total={len(self.blocks)} (avail={avail})")
|
| 577 |
else:
|
| 578 |
ids = buckets["hinglish"] + buckets["hindi"] + buckets["english"]
|
| 579 |
+
if R0:
|
| 580 |
+
print(f"[hf] mix NATURAL (rule OFF): prefetched {len(ids)} tokens -> {len(ids)//max(seq_len,1)} blocks")
|
| 581 |
ids = torch.tensor(ids, dtype=torch.long)
|
| 582 |
n = (len(ids)//seq_len)*seq_len
|
| 583 |
self.blocks = ids[:n].view(-1, seq_len) if n>0 else torch.zeros((1,seq_len),dtype=torch.long)
|
|
|
|
| 588 |
enforce_mix=enforce_mix, lang_mix=lang_mix, mix_seed=mix_seed,
|
| 589 |
mix_scan_mult=mix_scan_mult, mix_max_rows=mix_max_rows)
|
| 590 |
except Exception as e:
|
| 591 |
+
if R0:
|
| 592 |
+
print(f"[hf] streaming failed ({e}), fallback to local data/raw")
|
| 593 |
hf_dataset = None
|
| 594 |
ds = None
|
| 595 |
if args.smoke:
|
|
|
|
| 672 |
val_ds = None
|
| 673 |
if use_dist and not args.smoke:
|
| 674 |
train_ids = train_ids[rank::world]
|
| 675 |
+
if not train_ids:
|
| 676 |
+
# blocks < world ya rank ko empty stride: ZeroDivision crash se pehle guard
|
| 677 |
+
print(f"[dist] WARN rank {rank}: 0 train blocks after stride — fallback 1 block (data overlap)")
|
| 678 |
+
train_ids = [all_idx[rank % len(all_idx)]] if all_idx else [0]
|
| 679 |
+
else:
|
| 680 |
+
if R0 or use_dist: # DDP me stride-count rank-wise useful hai
|
| 681 |
+
print(f"[dist] rank {rank}: {len(train_ids)} train blocks (stride-shard /{world})")
|
| 682 |
+
if R0:
|
| 683 |
+
print(f"[data] blocks={len(ds)} train={len(train_ids)} val={len(val_ids) if val_ds else 0} seq={seq_len} batch={batch} accum={accum} eff_batch={batch*accum} (x{world} ranks = {batch*accum*world} global)")
|
| 684 |
|
| 685 |
model = Viu1MoE(margs).to(device)
|
| 686 |
+
# auto_tune PROBE DDP wrap SE PEHLE (raw model): wrap ke baad probe backward allreduce
|
| 687 |
+
# karta hai — dono ranks ka OOM point alag ho to probe-step count mismatch → NCCL hang.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 688 |
if C("auto_tune", True) and not args.smoke and device.type == "cuda":
|
| 689 |
# eff-batch target same rakho, micro/accum GPU ke hisaab se baanto — har GPU pe same command
|
| 690 |
target_eff = batch * accum
|
|
|
|
| 693 |
micro = max(1, min(micro, target_eff))
|
| 694 |
accum = max(1, round(target_eff / micro))
|
| 695 |
batch = micro
|
| 696 |
+
if use_dist and not args.smoke:
|
| 697 |
+
# ranks ka VRAM alag ho to micro/accum alag → per-step backward count alag → hang.
|
| 698 |
+
# rank0 ka auto_tune choice sab pe broadcast karo.
|
| 699 |
+
import torch.distributed as dist
|
| 700 |
+
_cfg = torch.tensor([batch, accum], device=device, dtype=torch.long)
|
| 701 |
+
dist.broadcast(_cfg, src=0)
|
| 702 |
+
batch, accum = int(_cfg[0]), int(_cfg[1])
|
| 703 |
print(f"[auto] {gname} ({ggb:.0f}GB, bf16_ok={bf16_ok}) amp={'bf16' if use_bf16 else 'fp16'} "
|
| 704 |
+
f"micro={batch} accum={accum} eff={batch * accum} (target {target_eff}) tok/step={batch * accum * seq_len}"
|
| 705 |
+
+ (" [dist broadcast r0->all]" if (use_dist and not args.smoke) else "")) if R0 else None
|
| 706 |
+
if use_dist and not args.smoke:
|
| 707 |
+
# MoE top-2 routing me kuch experts kabhi-kabhi empty rehte hain -> find_unused_parameters=True (safe)
|
| 708 |
+
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[device.index], find_unused_parameters=True)
|
| 709 |
+
if R0:
|
| 710 |
+
print(f"[dist] DDP wrapped (find_unused=True, micro={batch} accum={accum} synced)")
|
| 711 |
+
ckpt_model = model.module if (use_dist and not args.smoke) else model # ckpt bina module. prefix ke (resume-safe)
|
| 712 |
+
total, active = ckpt_model.param_counts()
|
| 713 |
+
if R0:
|
| 714 |
+
print(f"[model] total={total/1e6:.1f}M active={active/1e6:.1f}M device={device}" + (f" (DDP x{world})" if use_dist else ""))
|
| 715 |
|
| 716 |
decay_params = [p for p in model.parameters() if p.requires_grad and p.ndim >= 2]
|
| 717 |
no_decay_params = [p for p in model.parameters() if p.requires_grad and p.ndim < 2]
|
|
|
|
| 728 |
resume = None if args.smoke else C("resume", None)
|
| 729 |
if resume and Path(resume).exists():
|
| 730 |
start = load_ckpt(resume, ckpt_model, opt, scaler=scaler)
|
| 731 |
+
if R0:
|
| 732 |
+
print(f"[resume] step {start} from {resume}")
|
| 733 |
# HF auto-push (har training ke baad ckpt backup mandatory) — token env HF_TOKEN se
|
| 734 |
do_hf_push = bool(C("hf_push", False)) and not args.smoke
|
| 735 |
hf_repo = C("hf_repo", None)
|
| 736 |
hf_token = os.environ.get("HF_TOKEN")
|
| 737 |
hf_every = int(C("hf_push_every", 0) or 0)
|
| 738 |
if do_hf_push and not hf_repo:
|
| 739 |
+
if R0:
|
| 740 |
+
print("[hf-push] WARN: hf_push true par hf_repo missing — push off")
|
| 741 |
do_hf_push = False
|
| 742 |
+
if do_hf_push and not hf_token and R0:
|
| 743 |
print("[hf-push] HF_TOKEN env me nahi — push skip hoga (token set karo ya manual cell chalao)")
|
| 744 |
|
| 745 |
log_dir = Path(log_dir)
|
|
|
|
| 817 |
pbar.update(1)
|
| 818 |
pbar.set_postfix(loss=f"{acc_loss:.4f}", lr=f"{cur_lr:.1e}", refresh=False)
|
| 819 |
|
| 820 |
+
# DDP me [train] lines sirf R0 (warna dono rank interleaved duplicate print)
|
| 821 |
+
if R0 and (step % log_every == 0 or step == max_steps):
|
| 822 |
dt = time.time() - t0
|
| 823 |
done = max(step - start, 1)
|
| 824 |
rate = done / max(dt, 1e-6)
|
|
|
|
| 840 |
share_str = f" max_share {max_expert_share}" if max_expert_share is not None else ""
|
| 841 |
print(f"[train] step {step}/{max_steps} ({pct:.1f}%) loss {acc_loss:.4f} lr {cur_lr:.2e} "
|
| 842 |
f"tok/s {tps:,.0f} elapsed {fmt_hms(dt)} ETA {fmt_hms(eta)}{share_str} aux {aux}", flush=True)
|
| 843 |
+
if logf is not None:
|
| 844 |
logf.write(json.dumps(msg) + "\n")
|
| 845 |
logf.flush()
|
| 846 |
|
|
|
|
| 869 |
pbar.close()
|
| 870 |
total_dt = time.time() - t0
|
| 871 |
total_done = max(step - start, 1)
|
| 872 |
+
if R0:
|
| 873 |
+
print(f"[done] steps {total_done} time {fmt_hms(total_dt)} "
|
| 874 |
+
f"avg_tok/s {tok_per_step * total_done / max(total_dt, 1e-6):,.0f}", flush=True)
|
| 875 |
if R0 and do_hf_push:
|
| 876 |
save_slim_ckpt(Path(ckpt_dir) / f"slim_final_{step}.pt", ckpt_model, cfg)
|
| 877 |
hf_push_ckpt(ckpt_dir, hf_repo, hf_token, only_file=Path(ckpt_dir) / f"slim_final_{step}.pt",
|
tokenizer/outputs/tokenizer.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:669832070927b955ea9f589b7883310abdbe7f63fa5be9922a311a2b3f5d92c9
|
| 3 |
+
size 4254267
|
tokenizer/train_tokenizer_colab.ipynb
CHANGED
|
@@ -4,12 +4,12 @@
|
|
| 4 |
"cell_type": "markdown",
|
| 5 |
"metadata": {},
|
| 6 |
"source": [
|
| 7 |
-
"# ViuMini-MoE-242M: Production Tokenizer Training (Indic
|
| 8 |
"### Architecture: Byte-Level BPE (48,000 Vocab) with Llama-3 / Indic Regex\n",
|
| 9 |
-
"###
|
| 10 |
-
"### Special Tokens: 13 tokens including `<soch>`, `</soch>`, `<|user|>`, `<|assistant|>`\n",
|
| 11 |
"\n",
|
| 12 |
-
"**Instructions**: Run this single cell on Google Colab
|
| 13 |
]
|
| 14 |
},
|
| 15 |
{
|
|
@@ -21,7 +21,7 @@
|
|
| 21 |
"# Step 1: Install Dependencies\n",
|
| 22 |
"!pip install -q tokenizers datasets huggingface_hub pyarrow\n",
|
| 23 |
"\n",
|
| 24 |
-
"# Step 2:
|
| 25 |
"import os\n",
|
| 26 |
"import sys\n",
|
| 27 |
"import time\n",
|
|
@@ -39,8 +39,8 @@
|
|
| 39 |
"VOCAB_SIZE = 48000\n",
|
| 40 |
"TOTAL_TRAIN_LINES = 500000\n",
|
| 41 |
"OUTPUT_TOKENIZER_PATH = \"tokenizer.json\"\n",
|
|
|
|
| 42 |
"def _get_hf_token():\n",
|
| 43 |
-
" # SECURITY: token hardcode mat karo — env / Kaggle / Colab secrets se lo.\n",
|
| 44 |
" tok = os.environ.get(\"HF_TOKEN\") or os.environ.get(\"HUGGING_FACE_HUB_TOKEN\")\n",
|
| 45 |
" if not tok:\n",
|
| 46 |
" try:\n",
|
|
@@ -55,9 +55,10 @@
|
|
| 55 |
" except Exception:\n",
|
| 56 |
" pass\n",
|
| 57 |
" return tok\n",
|
|
|
|
| 58 |
"HF_TOKEN = _get_hf_token()\n",
|
| 59 |
"\n",
|
| 60 |
-
"# Llama-3 / Indic regex: includes \\p{M} so Devanagari
|
| 61 |
"INDIC_LLAMA3_PATTERN = (\n",
|
| 62 |
" r\"(?i:'s|'t|'re|'ve|'m|'ll|'d)|\"\n",
|
| 63 |
" r\"[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\"\n",
|
|
@@ -67,8 +68,9 @@
|
|
| 67 |
" r\"\\s+(?!\\S)|\\s+\"\n",
|
| 68 |
")\n",
|
| 69 |
"\n",
|
|
|
|
| 70 |
"SPECIAL_TOKENS = [\n",
|
| 71 |
-
" \"<pad>\", \"<bos>\", \"<eos>\", \"<unk>\", \"<mask
|
| 72 |
" \"<|hindi|>\", \"<|english|>\", \"<|hinglish|>\",\n",
|
| 73 |
" \"<soch>\", \"</soch>\", \"<|user|>\", \"<|assistant|>\", \"<|system|>\",\n",
|
| 74 |
"]\n",
|
|
@@ -91,7 +93,7 @@
|
|
| 91 |
"]\n",
|
| 92 |
"\n",
|
| 93 |
"print(\"=\" * 70)\n",
|
| 94 |
-
"print(\"Starting
|
| 95 |
"print(\"=\" * 70)\n",
|
| 96 |
"\n",
|
| 97 |
"api = HfApi(token=HF_TOKEN)\n",
|
|
@@ -103,9 +105,10 @@
|
|
| 103 |
"\n",
|
| 104 |
"print(f\"Files on Hub: Hindi: {len(hindi_files)}, Hinglish: {len(hinglish_files)}, English/Reasoning: {len(english_files)}\")\n",
|
| 105 |
"\n",
|
| 106 |
-
"
|
| 107 |
"target_hinglish = int(TOTAL_TRAIN_LINES * 0.35)\n",
|
| 108 |
-
"
|
|
|
|
| 109 |
"\n",
|
| 110 |
"def fetch_stream(file_list, target_count, category_name):\n",
|
| 111 |
" emitted = 0\n",
|
|
@@ -126,13 +129,14 @@
|
|
| 126 |
" print(f\"[Stream] Collected {emitted:,} lines for {category_name}\")\n",
|
| 127 |
"\n",
|
| 128 |
"def interleave_generator():\n",
|
| 129 |
-
" hi_gen = fetch_stream(hindi_files, target_hindi, \"Hindi\")\n",
|
| 130 |
" h_gen = fetch_stream(hinglish_files, target_hinglish, \"Hinglish\")\n",
|
|
|
|
| 131 |
" e_gen = fetch_stream(english_files, target_english, \"English/Reasoning\")\n",
|
| 132 |
"\n",
|
| 133 |
-
" generators = {\"
|
| 134 |
-
"
|
| 135 |
-
"
|
|
|
|
| 136 |
"\n",
|
| 137 |
" total_emitted = 0\n",
|
| 138 |
" while total_emitted < TOTAL_TRAIN_LINES and any(active.values()):\n",
|
|
@@ -179,7 +183,7 @@
|
|
| 179 |
"\n",
|
| 180 |
"# Evaluation\n",
|
| 181 |
"print(\"\\n\" + \"=\" * 70)\n",
|
| 182 |
-
"print(\"TOKENIZER EVALUATION & QUALITY AUDIT\")\n",
|
| 183 |
"print(\"=\" * 70)\n",
|
| 184 |
"\n",
|
| 185 |
"print(\"\\n1. Special Tokens Atomic Isolation Check:\")\n",
|
|
@@ -202,7 +206,7 @@
|
|
| 202 |
"\n",
|
| 203 |
"for cat, ratios in cat_fert.items():\n",
|
| 204 |
" avg_r = sum(ratios) / len(ratios)\n",
|
| 205 |
-
" target = 1.
|
| 206 |
" status = \"EXCELLENT\" if avg_r <= target else (\"ACCEPTABLE\" if avg_r <= target + 0.3 else \"HIGH\")\n",
|
| 207 |
" print(f\" - {cat:10}: {avg_r:.2f} tokens/word (Target: <= {target:.1f}) [{status}]\")\n",
|
| 208 |
"\n",
|
|
@@ -219,14 +223,15 @@
|
|
| 219 |
" print(\" [PASS] 100% Round-trip fidelity confirmed across all benchmark sentences.\")\n",
|
| 220 |
"\n",
|
| 221 |
"# Upload to Hub\n",
|
| 222 |
-
"
|
| 223 |
-
"
|
| 224 |
-
"
|
| 225 |
-
"
|
| 226 |
-
"
|
| 227 |
-
"
|
| 228 |
-
"
|
| 229 |
-
"
|
|
|
|
| 230 |
]
|
| 231 |
}
|
| 232 |
],
|
|
|
|
| 4 |
"cell_type": "markdown",
|
| 5 |
"metadata": {},
|
| 6 |
"source": [
|
| 7 |
+
"# ViuMini-MoE-242M: Production Tokenizer Training (v2 Indic & English Balanced)\n",
|
| 8 |
"### Architecture: Byte-Level BPE (48,000 Vocab) with Llama-3 / Indic Regex\n",
|
| 9 |
+
"### v2 Balanced Mixture: 35% Hinglish + 30% Devanagari Hindi + 35% English & Reasoning\n",
|
| 10 |
+
"### Special Tokens: 13 tokens including `<soch>`, `</soch>`, `<|user|>`, `<|assistant|>`, `<mask\>` (typo fixed)\n",
|
| 11 |
"\n",
|
| 12 |
+
"**Instructions**: Run this single cell on Google Colab or Kaggle. It takes ~3-4 minutes, achieves balanced fertility across Hindi, Hinglish, and English, and uploads the production `tokenizer.json` to Hugging Face Hub."
|
| 13 |
]
|
| 14 |
},
|
| 15 |
{
|
|
|
|
| 21 |
"# Step 1: Install Dependencies\n",
|
| 22 |
"!pip install -q tokenizers datasets huggingface_hub pyarrow\n",
|
| 23 |
"\n",
|
| 24 |
+
"# Step 2: v2 Balanced BPE Training Pipeline\n",
|
| 25 |
"import os\n",
|
| 26 |
"import sys\n",
|
| 27 |
"import time\n",
|
|
|
|
| 39 |
"VOCAB_SIZE = 48000\n",
|
| 40 |
"TOTAL_TRAIN_LINES = 500000\n",
|
| 41 |
"OUTPUT_TOKENIZER_PATH = \"tokenizer.json\"\n",
|
| 42 |
+
"\n",
|
| 43 |
"def _get_hf_token():\n",
|
|
|
|
| 44 |
" tok = os.environ.get(\"HF_TOKEN\") or os.environ.get(\"HUGGING_FACE_HUB_TOKEN\")\n",
|
| 45 |
" if not tok:\n",
|
| 46 |
" try:\n",
|
|
|
|
| 55 |
" except Exception:\n",
|
| 56 |
" pass\n",
|
| 57 |
" return tok\n",
|
| 58 |
+
"\n",
|
| 59 |
"HF_TOKEN = _get_hf_token()\n",
|
| 60 |
"\n",
|
| 61 |
+
"# Llama-3 / Indic regex pattern: includes \\p{M} so Devanagari vowel signs stay attached to consonants\n",
|
| 62 |
"INDIC_LLAMA3_PATTERN = (\n",
|
| 63 |
" r\"(?i:'s|'t|'re|'ve|'m|'ll|'d)|\"\n",
|
| 64 |
" r\"[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\"\n",
|
|
|
|
| 68 |
" r\"\\s+(?!\\S)|\\s+\"\n",
|
| 69 |
")\n",
|
| 70 |
"\n",
|
| 71 |
+
"# v2: <mask|> typo fixed to <mask\>\n",
|
| 72 |
"SPECIAL_TOKENS = [\n",
|
| 73 |
+
" \"<pad>\", \"<bos>\", \"<eos>\", \"<unk>\", \"<mask\>\",\n",
|
| 74 |
" \"<|hindi|>\", \"<|english|>\", \"<|hinglish|>\",\n",
|
| 75 |
" \"<soch>\", \"</soch>\", \"<|user|>\", \"<|assistant|>\", \"<|system|>\",\n",
|
| 76 |
"]\n",
|
|
|
|
| 93 |
"]\n",
|
| 94 |
"\n",
|
| 95 |
"print(\"=\" * 70)\n",
|
| 96 |
+
"print(\"Starting ViuMini-MoE-242M Tokenizer Training (v2 Mix)\")\n",
|
| 97 |
"print(\"=\" * 70)\n",
|
| 98 |
"\n",
|
| 99 |
"api = HfApi(token=HF_TOKEN)\n",
|
|
|
|
| 105 |
"\n",
|
| 106 |
"print(f\"Files on Hub: Hindi: {len(hindi_files)}, Hinglish: {len(hinglish_files)}, English/Reasoning: {len(english_files)}\")\n",
|
| 107 |
"\n",
|
| 108 |
+
"# v2 Mixture: 35% Hinglish + 30% Hindi + 35% English/Reasoning\n",
|
| 109 |
"target_hinglish = int(TOTAL_TRAIN_LINES * 0.35)\n",
|
| 110 |
+
"target_hindi = int(TOTAL_TRAIN_LINES * 0.30)\n",
|
| 111 |
+
"target_english = TOTAL_TRAIN_LINES - target_hinglish - target_hindi\n",
|
| 112 |
"\n",
|
| 113 |
"def fetch_stream(file_list, target_count, category_name):\n",
|
| 114 |
" emitted = 0\n",
|
|
|
|
| 129 |
" print(f\"[Stream] Collected {emitted:,} lines for {category_name}\")\n",
|
| 130 |
"\n",
|
| 131 |
"def interleave_generator():\n",
|
|
|
|
| 132 |
" h_gen = fetch_stream(hinglish_files, target_hinglish, \"Hinglish\")\n",
|
| 133 |
+
" hi_gen = fetch_stream(hindi_files, target_hindi, \"Hindi\")\n",
|
| 134 |
" e_gen = fetch_stream(english_files, target_english, \"English/Reasoning\")\n",
|
| 135 |
"\n",
|
| 136 |
+
" generators = {\"hinglish\": h_gen, \"hindi\": hi_gen, \"english\": e_gen}\n",
|
| 137 |
+
" # Pattern: 7 Hinglish, 6 Hindi, 7 English (35% : 30% : 35%)\n",
|
| 138 |
+
" pattern = [\"hinglish\"] * 7 + [\"hindi\"] * 6 + [\"english\"] * 7\n",
|
| 139 |
+
" active = {\"hinglish\": True, \"hindi\": True, \"english\": True}\n",
|
| 140 |
"\n",
|
| 141 |
" total_emitted = 0\n",
|
| 142 |
" while total_emitted < TOTAL_TRAIN_LINES and any(active.values()):\n",
|
|
|
|
| 183 |
"\n",
|
| 184 |
"# Evaluation\n",
|
| 185 |
"print(\"\\n\" + \"=\" * 70)\n",
|
| 186 |
+
"print(\"TOKENIZER EVALUATION & QUALITY AUDIT (v2 Mix)\")\n",
|
| 187 |
"print(\"=\" * 70)\n",
|
| 188 |
"\n",
|
| 189 |
"print(\"\\n1. Special Tokens Atomic Isolation Check:\")\n",
|
|
|
|
| 206 |
"\n",
|
| 207 |
"for cat, ratios in cat_fert.items():\n",
|
| 208 |
" avg_r = sum(ratios) / len(ratios)\n",
|
| 209 |
+
" target = 1.3 if cat == \"English\" else (1.5 if cat == \"Hinglish\" else 1.8)\n",
|
| 210 |
" status = \"EXCELLENT\" if avg_r <= target else (\"ACCEPTABLE\" if avg_r <= target + 0.3 else \"HIGH\")\n",
|
| 211 |
" print(f\" - {cat:10}: {avg_r:.2f} tokens/word (Target: <= {target:.1f}) [{status}]\")\n",
|
| 212 |
"\n",
|
|
|
|
| 223 |
" print(\" [PASS] 100% Round-trip fidelity confirmed across all benchmark sentences.\")\n",
|
| 224 |
"\n",
|
| 225 |
"# Upload to Hub\n",
|
| 226 |
+
"if HF_TOKEN:\n",
|
| 227 |
+
" print(f\"\\n[Hub] Uploading tokenizer.json to {HF_DATASET_REPO}...\")\n",
|
| 228 |
+
" api.upload_file(\n",
|
| 229 |
+
" path_or_fileobj=OUTPUT_TOKENIZER_PATH,\n",
|
| 230 |
+
" path_in_repo=\"tokenizer/tokenizer.json\",\n",
|
| 231 |
+
" repo_id=HF_DATASET_REPO,\n",
|
| 232 |
+
" repo_type=\"dataset\",\n",
|
| 233 |
+
" )\n",
|
| 234 |
+
" print(f\"[Hub] Successfully uploaded to https://huggingface.co/datasets/{HF_DATASET_REPO}/blob/main/tokenizer/tokenizer.json\")\n"
|
| 235 |
]
|
| 236 |
}
|
| 237 |
],
|