Audit fix: Update model/scripts/train.py for ViuMini-Dense-360M
Browse files- model/scripts/train.py +4 -3
model/scripts/train.py
CHANGED
|
@@ -1477,10 +1477,11 @@ def main():
|
|
| 1477 |
str(C("compile_mode", "reduce-overhead")), args.smoke, R0,
|
| 1478 |
device=device, vocab_size=margs.vocab_size)
|
| 1479 |
if use_dist and not args.smoke:
|
| 1480 |
-
# MoE routing me agar koi expert
|
| 1481 |
-
|
|
|
|
| 1482 |
if R0:
|
| 1483 |
-
print(f"[dist] DDP wrapped (find_unused=
|
| 1484 |
ckpt_model = model.module if (use_dist and not args.smoke) else model # ckpt bina module. prefix ke (resume-safe)
|
| 1485 |
total, active = ckpt_model.param_counts()
|
| 1486 |
if R0:
|
|
|
|
| 1477 |
str(C("compile_mode", "reduce-overhead")), args.smoke, R0,
|
| 1478 |
device=device, vocab_size=margs.vocab_size)
|
| 1479 |
if use_dist and not args.smoke:
|
| 1480 |
+
# MoE routing me agar koi expert skip ho to find_unused_parameters=True; Dense model me False (zero overhead)
|
| 1481 |
+
is_moe = getattr(margs, "moe_every", 999) < 900
|
| 1482 |
+
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[device.index], find_unused_parameters=is_moe)
|
| 1483 |
if R0:
|
| 1484 |
+
print(f"[dist] DDP wrapped (find_unused={is_moe}, micro={batch} accum={accum} synced)")
|
| 1485 |
ckpt_model = model.module if (use_dist and not args.smoke) else model # ckpt bina module. prefix ke (resume-safe)
|
| 1486 |
total, active = ckpt_model.param_counts()
|
| 1487 |
if R0:
|