train: escape the literal percent signs in two --help strings (argparse interpolates help text, so '+52 %' crashed --help with 'must be real number, not dict' -- caught by the CPU rehearsal's stage 3)
Browse files- train/train_ounce100m.py +2 -2
train/train_ounce100m.py
CHANGED
|
@@ -259,7 +259,7 @@ def main():
|
|
| 259 |
# that launching the run does not depend on remembering a flag string.
|
| 260 |
ap.add_argument("--seq-len", type=int, default=1024)
|
| 261 |
ap.add_argument("--attn", default="eager", choices=["eager", "sdpa", "flash_attention_2"],
|
| 262 |
-
help="p0c measured eager beating SDPA on Turing; Gate 3 P2/A-B measured +52 % for "
|
| 263 |
"eager at seq 1024 and an OOM at 2048. flash_attention_2 needs sm80+")
|
| 264 |
ap.add_argument("--grad-ckpt", default=True, action=argparse.BooleanOptionalAction,
|
| 265 |
help="on by default: it is what keeps peak memory at ~7 GB instead of 12 GB, and the "
|
|
@@ -288,7 +288,7 @@ def main():
|
|
| 288 |
ap.add_argument("--resume", default="", help="checkpoint dir; empty = auto from Hub latest")
|
| 289 |
ap.add_argument("--val-tokens", type=int, default=2_000_000)
|
| 290 |
ap.add_argument("--log-every", type=int, default=10)
|
| 291 |
-
ap.add_argument("--push-every-steps", type=int, default=0, help="0 = only at 10% boundaries")
|
| 292 |
ap.add_argument("--stop-after-steps", type=int, default=0,
|
| 293 |
help="absolute global step at which THIS session stops training, so it ends on a "
|
| 294 |
"verified checkpoint instead of being killed between two of them. Must be a "
|
|
|
|
| 259 |
# that launching the run does not depend on remembering a flag string.
|
| 260 |
ap.add_argument("--seq-len", type=int, default=1024)
|
| 261 |
ap.add_argument("--attn", default="eager", choices=["eager", "sdpa", "flash_attention_2"],
|
| 262 |
+
help="p0c measured eager beating SDPA on Turing; Gate 3 P2/A-B measured +52 %% for "
|
| 263 |
"eager at seq 1024 and an OOM at 2048. flash_attention_2 needs sm80+")
|
| 264 |
ap.add_argument("--grad-ckpt", default=True, action=argparse.BooleanOptionalAction,
|
| 265 |
help="on by default: it is what keeps peak memory at ~7 GB instead of 12 GB, and the "
|
|
|
|
| 288 |
ap.add_argument("--resume", default="", help="checkpoint dir; empty = auto from Hub latest")
|
| 289 |
ap.add_argument("--val-tokens", type=int, default=2_000_000)
|
| 290 |
ap.add_argument("--log-every", type=int, default=10)
|
| 291 |
+
ap.add_argument("--push-every-steps", type=int, default=0, help="0 = only at 10%% boundaries")
|
| 292 |
ap.add_argument("--stop-after-steps", type=int, default=0,
|
| 293 |
help="absolute global step at which THIS session stops training, so it ends on a "
|
| 294 |
"verified checkpoint instead of being killed between two of them. Must be a "
|