ViuAI commited on
Commit
2f86ee7
·
verified ·
1 Parent(s): 16e21dc

Update code/train_sft.py with chunked cross-entropy and H200 turbo speed optimizations

Browse files
Files changed (1) hide show
  1. code/train_sft.py +1 -1
code/train_sft.py CHANGED
@@ -771,7 +771,7 @@ def main():
771
  c_logit = raw_m.head(h_i[:, ci:ce, :])
772
  c_lbl = s_lbl[ci:ce].unsqueeze(0)
773
  sample_loss_sum += F.cross_entropy(
774
- c_logit.view(-1, V), c_lbl.view(-1),
775
  ignore_index=-100, reduction="sum"
776
  ).item()
777
 
 
771
  c_logit = raw_m.head(h_i[:, ci:ce, :])
772
  c_lbl = s_lbl[ci:ce].unsqueeze(0)
773
  sample_loss_sum += F.cross_entropy(
774
+ c_logit.reshape(-1, V), c_lbl.reshape(-1),
775
  ignore_index=-100, reduction="sum"
776
  ).item()
777