ViuAI commited on
Commit
e52f2b4
·
verified ·
1 Parent(s): 5d2e7a3

Audit fix: Update model/scripts/train.py for ViuMini-Dense-360M

Browse files
Files changed (1) hide show
  1. model/scripts/train.py +4 -3
model/scripts/train.py CHANGED
@@ -1477,10 +1477,11 @@ def main():
1477
  str(C("compile_mode", "reduce-overhead")), args.smoke, R0,
1478
  device=device, vocab_size=margs.vocab_size)
1479
  if use_dist and not args.smoke:
1480
- # MoE routing me agar koi expert kisi micro-batch me skip ho to DDP hang na ho -> find_unused_parameters=True (anti-freeze)
1481
- model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[device.index], find_unused_parameters=True)
 
1482
  if R0:
1483
- print(f"[dist] DDP wrapped (find_unused=True, micro={batch} accum={accum} synced)")
1484
  ckpt_model = model.module if (use_dist and not args.smoke) else model # ckpt bina module. prefix ke (resume-safe)
1485
  total, active = ckpt_model.param_counts()
1486
  if R0:
 
1477
  str(C("compile_mode", "reduce-overhead")), args.smoke, R0,
1478
  device=device, vocab_size=margs.vocab_size)
1479
  if use_dist and not args.smoke:
1480
+ # MoE routing me agar koi expert skip ho to find_unused_parameters=True; Dense model me False (zero overhead)
1481
+ is_moe = getattr(margs, "moe_every", 999) < 900
1482
+ model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[device.index], find_unused_parameters=is_moe)
1483
  if R0:
1484
+ print(f"[dist] DDP wrapped (find_unused={is_moe}, micro={batch} accum={accum} synced)")
1485
  ckpt_model = model.module if (use_dist and not args.smoke) else model # ckpt bina module. prefix ke (resume-safe)
1486
  total, active = ckpt_model.param_counts()
1487
  if R0: