Andrej Karpathy commited on
Commit ·
eaf49a3
1
Parent(s): 31b61d2
fix path which i think was modified during the refactor and this is a bug introduced by claude i believe
Browse files- scripts/chat_sft.py +2 -2
scripts/chat_sft.py
CHANGED
|
@@ -48,7 +48,7 @@ parser.add_argument("--max-seq-len", type=int, default=2048, help="max context l
|
|
| 48 |
parser.add_argument("--device-batch-size", type=int, default=32, help="per-device batch size")
|
| 49 |
parser.add_argument("--total-batch-size", type=int, default=524288, help="total batch size in tokens")
|
| 50 |
# Optimization
|
| 51 |
-
parser.add_argument("--embedding-lr", type=float, default=0.
|
| 52 |
parser.add_argument("--unembedding-lr", type=float, default=0.004, help="learning rate for unembedding parameters (Adam)")
|
| 53 |
parser.add_argument("--matrix-lr", type=float, default=0.02, help="learning rate for matrix parameters (Muon)")
|
| 54 |
parser.add_argument("--weight-decay", type=float, default=0.0, help="weight decay for embedding/unembedding parameters (Adam)")
|
|
@@ -285,7 +285,7 @@ while True:
|
|
| 285 |
# save checkpoint at the end of the run (only on master process)
|
| 286 |
if master_process and last_step and not args.dry_run:
|
| 287 |
output_dirname = args.model_tag if args.model_tag else f"d{depth}" # e.g. d12
|
| 288 |
-
checkpoint_dir = os.path.join(base_dir, "
|
| 289 |
save_checkpoint(
|
| 290 |
checkpoint_dir,
|
| 291 |
step,
|
|
|
|
| 48 |
parser.add_argument("--device-batch-size", type=int, default=32, help="per-device batch size")
|
| 49 |
parser.add_argument("--total-batch-size", type=int, default=524288, help="total batch size in tokens")
|
| 50 |
# Optimization
|
| 51 |
+
parser.add_argument("--embedding-lr", type=float, default=0.3, help="learning rate for embedding parameters (Adam)")
|
| 52 |
parser.add_argument("--unembedding-lr", type=float, default=0.004, help="learning rate for unembedding parameters (Adam)")
|
| 53 |
parser.add_argument("--matrix-lr", type=float, default=0.02, help="learning rate for matrix parameters (Muon)")
|
| 54 |
parser.add_argument("--weight-decay", type=float, default=0.0, help="weight decay for embedding/unembedding parameters (Adam)")
|
|
|
|
| 285 |
# save checkpoint at the end of the run (only on master process)
|
| 286 |
if master_process and last_step and not args.dry_run:
|
| 287 |
output_dirname = args.model_tag if args.model_tag else f"d{depth}" # e.g. d12
|
| 288 |
+
checkpoint_dir = os.path.join(base_dir, "chatsft_checkpoints", output_dirname)
|
| 289 |
save_checkpoint(
|
| 290 |
checkpoint_dir,
|
| 291 |
step,
|