Update code/train_sft.py with chunked cross-entropy and H200 turbo speed optimizations
Browse files- code/train_sft.py +1 -1
code/train_sft.py
CHANGED
|
@@ -771,7 +771,7 @@ def main():
|
|
| 771 |
c_logit = raw_m.head(h_i[:, ci:ce, :])
|
| 772 |
c_lbl = s_lbl[ci:ce].unsqueeze(0)
|
| 773 |
sample_loss_sum += F.cross_entropy(
|
| 774 |
-
c_logit.
|
| 775 |
ignore_index=-100, reduction="sum"
|
| 776 |
).item()
|
| 777 |
|
|
|
|
| 771 |
c_logit = raw_m.head(h_i[:, ci:ce, :])
|
| 772 |
c_lbl = s_lbl[ci:ce].unsqueeze(0)
|
| 773 |
sample_loss_sum += F.cross_entropy(
|
| 774 |
+
c_logit.reshape(-1, V), c_lbl.reshape(-1),
|
| 775 |
ignore_index=-100, reduction="sum"
|
| 776 |
).item()
|
| 777 |
|