File size: 94,007 Bytes
bdf0b03 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 | {
"version": "1",
"metadata": {
"marimo_version": "0.25.1",
"script_metadata_hash": "ff000607a504b65002f0c642ea5a638f"
},
"cells": [
{
"id": "Hbol",
"code_hash": "1d0db38904205bec4d6f6f6a1f6cec3e",
"outputs": [],
"console": []
},
{
"id": "MJUe",
"code_hash": "8c3add7f504aee86f92a5fd2a080e930",
"outputs": [],
"console": []
},
{
"id": "vblA",
"code_hash": "905d09ca55d5b894841a2106100436a4",
"outputs": [
{
"type": "error",
"ename": "interruption",
"evalue": "This cell was interrupted and needs to be re-run",
"traceback": []
}
],
"console": [
{
"type": "stream",
"name": "stderr",
"text": "<span class=\"codehilite\"><div class=\"highlight\"><pre><span></span><span class=\"gt\">Traceback (most recent call last):</span>\n File <span class=\"nb\">"/tmp/marimo_730/__marimo__cell_vblA_.py"</span>, line <span class=\"m\">3</span>, in <span class=\"n\"><module></span>\n<span class=\"w\"> </span><span class=\"n\">result</span> <span class=\"o\">=</span> <span class=\"n\">subprocess</span><span class=\"o\">.</span><span class=\"n\">run</span><span class=\"p\">(</span>\n<span class=\"w\"> </span> <span class=\"p\">[</span><span class=\"s1\">'python'</span><span class=\"p\">,</span> <span class=\"s1\">'wikitext.py'</span><span class=\"p\">,</span> <span class=\"s1\">'--sources'</span><span class=\"p\">,</span> <span class=\"s1\">'cosmopedia'</span><span class=\"p\">,</span> <span class=\"s1\">'--cosmopedia-mb'</span><span class=\"p\">,</span> <span class=\"s1\">'1000'</span><span class=\"p\">],</span>\n<span class=\"w\"> </span> <span class=\"n\">capture_output</span><span class=\"o\">=</span><span class=\"kc\">True</span><span class=\"p\">,</span> <span class=\"n\">text</span><span class=\"o\">=</span><span class=\"kc\">True</span>\n<span class=\"w\"> </span><span class=\"p\">)</span>\n File <span class=\"nb\">"/usr/local/lib/python3.13/subprocess.py"</span>, line <span class=\"m\">556</span>, in <span class=\"n\">run</span>\n<span class=\"w\"> </span><span class=\"n\">stdout</span><span class=\"p\">,</span> <span class=\"n\">stderr</span> <span class=\"o\">=</span> <span class=\"n\">process</span><span class=\"o\">.</span><span class=\"n\">communicate</span><span class=\"p\">(</span><span class=\"nb\">input</span><span class=\"p\">,</span> <span class=\"n\">timeout</span><span class=\"o\">=</span><span class=\"n\">timeout</span><span class=\"p\">)</span>\n<span class=\"w\"> </span><span class=\"pm\">~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^</span>\n File <span class=\"nb\">"/usr/local/lib/python3.13/subprocess.py"</span>, line <span class=\"m\">1222</span>, in <span class=\"n\">communicate</span>\n<span class=\"w\"> </span><span class=\"n\">stdout</span><span class=\"p\">,</span> <span class=\"n\">stderr</span> <span class=\"o\">=</span> <span class=\"bp\">self</span><span class=\"o\">.</span><span class=\"n\">_communicate</span><span class=\"p\">(</span><span class=\"nb\">input</span><span class=\"p\">,</span> <span class=\"n\">endtime</span><span class=\"p\">,</span> <span class=\"n\">timeout</span><span class=\"p\">)</span>\n<span class=\"w\"> </span><span class=\"pm\">~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^</span>\n File <span class=\"nb\">"/usr/local/lib/python3.13/subprocess.py"</span>, line <span class=\"m\">2154</span>, in <span class=\"n\">_communicate</span>\n<span class=\"w\"> </span><span class=\"n\">ready</span> <span class=\"o\">=</span> <span class=\"n\">selector</span><span class=\"o\">.</span><span class=\"n\">select</span><span class=\"p\">(</span><span class=\"n\">timeout</span><span class=\"p\">)</span>\n File <span class=\"nb\">"/usr/local/lib/python3.13/selectors.py"</span>, line <span class=\"m\">398</span>, in <span class=\"n\">select</span>\n<span class=\"w\"> </span><span class=\"n\">fd_event_list</span> <span class=\"o\">=</span> <span class=\"bp\">self</span><span class=\"o\">.</span><span class=\"n\">_selector</span><span class=\"o\">.</span><span class=\"n\">poll</span><span class=\"p\">(</span><span class=\"n\">timeout</span><span class=\"p\">)</span>\n File <span class=\"nb\">"/usr/local/lib/python3.13/site-packages/marimo/_utils/signals.py"</span>, line <span class=\"m\">33</span>, in <span class=\"n\">__call__</span>\n<span class=\"w\"> </span><span class=\"bp\">self</span><span class=\"o\">.</span><span class=\"n\">_handler</span><span class=\"p\">(</span><span class=\"n\">signum</span><span class=\"p\">,</span> <span class=\"n\">frame</span><span class=\"p\">)</span>\n<span class=\"w\"> </span><span class=\"pm\">~~~~~~~~~~~~~^^^^^^^^^^^^^^^</span>\n File <span class=\"nb\">"/usr/local/lib/python3.13/site-packages/marimo/_runtime/handlers.py"</span>, line <span class=\"m\">74</span>, in <span class=\"n\">interrupt_handler</span>\n<span class=\"w\"> </span><span class=\"k\">raise</span> <span class=\"n\">MarimoInterrupt</span>\n<span class=\"gr\">KeyboardInterrupt</span>\n</pre></div>\n</span>",
"mimetype": "application/vnd.marimo+traceback"
}
]
},
{
"id": "bkHC",
"code_hash": "3e9c348c42c7ba5ad7449c3567bcf13f",
"outputs": [
{
"type": "data",
"data": {
"text/plain": ""
}
}
],
"console": [
{
"type": "stream",
"name": "stdout",
"text": "============================================================\nGPU / NVIDIA DRIVER DIAGNOSTICS\n============================================================\n\n1. Searching for nvidia-smi in common locations:\n /usr/bin/nvidia-smi: \u2705 FOUND\n /usr/local/bin/nvidia-smi: \u274c not found\n /usr/sbin/nvidia-smi: \u274c not found\n /bin/nvidia-smi: \u2705 FOUND\n /opt/nvidia/bin/nvidia-smi: \u274c not found\n\n2. Checking /proc/driver/nvidia (kernel module):\n \u2705 /proc/driver/nvidia EXISTS \u2014 NVIDIA kernel module is loaded\n\n3. Checking /dev/nvidia* device files:\n \u2705 /dev/nvidia-modeset\n \u2705 /dev/nvidia-uvm\n \u2705 /dev/nvidia-uvm-tools\n \u2705 /dev/nvidia1\n \u2705 /dev/nvidiactl\n\n4. Container/VM check:\n Not in a Docker container (no /.dockerenv)\n cgroup info: 7:pids:/2eb5332fc32851d41d6e92fc384df745fdb0c6fa13f4fdf88ad07c56c10555a3\n6:memory:/2eb5332fc32851d41d6e92fc384df745fdb0c6fa13f4fdf88ad07c56c10555a3\n5:job:/2eb5332fc32851d41d6e92fc384df745fdb0c6fa13f4f\n\n5. Current PATH:\n /tmp/uv-venv/bin:/tmp/uv-venv/lib/python3.13/site-packages/nvidia/cu13/bin:/usr/local/lib/python3.13/site-packages/nvidia/cu13/bin:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\n\n6. LD_LIBRARY_PATH:\n /tmp/uv-venv/lib/python3.13/site-packages/nvidia/cu13/lib:/usr/local/lib/python3.13/site-packages/nvidia/cu13/lib:\n\n7. Searching for CUDA installations:\n \u274c /usr/local/cuda not found\n \u274c /usr/local/cuda-12 not found\n \u274c /usr/local/cuda-13 not found\n \u274c /opt/cuda not found\n\n8. Checking lspci for NVIDIA hardware:\n \u274c lspci command not available\n\n9. PyTorch CUDA details:\n PyTorch version: 2.11.0+cu130\n CUDA available: True\n CUDA version (PyTorch built with): 13.0\n \u2705 libcuda.so is loadable\n\n============================================================\n",
"mimetype": "text/plain"
}
]
},
{
"id": "lEQa",
"code_hash": "8ee678e3af92e02ec6c3e6be3ca5bd98",
"outputs": [
{
"type": "data",
"data": {
"text/markdown": "<span class=\"markdown prose dark:prose-invert contents\"><h2 id=\"gpu-diagnosis-summary-molab-rtx-6000-pro-blackwell\">\ud83d\udd0d GPU Diagnosis Summary \u2014 Molab RTX 6000 Pro Blackwell</h2>\n<span class=\"paragraph\">Based on the diagnostic cells in this notebook, here is what I found:</span>\n<h3 id=\"root-cause-the-gpu-is-not-being-exposed-to-this-molab-containerruntime\">Root Cause: <strong>The GPU is not being exposed to this Molab container/runtime</strong></h3>\n<table>\n<thead>\n<tr>\n<th>Check</th>\n<th>Result</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td><code>nvidia-smi</code> binary</td>\n<td>\u274c Not found anywhere on the system</td>\n</tr>\n<tr>\n<td><code>torch.cuda.is_available()</code></td>\n<td>\u274c Returns <code>False</code></td>\n</tr>\n<tr>\n<td><code>/dev/nvidia*</code> devices</td>\n<td>\u274c Not present</td>\n</tr>\n<tr>\n<td><code>/proc/driver/nvidia</code></td>\n<td>\u274c Not present (no kernel module loaded)</td>\n</tr>\n<tr>\n<td><code>future.py</code> benchmark output</td>\n<td>\u274c Ran on <code>device=cpu</code> with <code>peak_mem=0.00 MB</code></td>\n</tr>\n<tr>\n<td><code>nvidia-smi -pm 1</code> command</td>\n<td>\u274c <code>FileNotFoundError: No such file or directory: 'nvidia-smi'</code></td>\n</tr>\n</tbody>\n</table>\n<h3 id=\"key-observations\">Key Observations</h3>\n<ol>\n<li><strong>PyTorch 2.11.0 with CUDA 13.0.2 is installed</strong> \u2014 all CUDA libraries (cuDNN, cuBLAS, NCCL, etc.) are present via pip, but they are <strong>useless without the GPU being exposed to the container</strong>.</li>\n<li><strong>This is a Molab environment</strong> \u2014 the GPU should be available, but the container/runtime was not started with GPU passthrough properly configured.</li>\n<li><strong>No <code>nvidia-smi</code>, no <code>/dev/nvidia*</code>, no <code>/proc/driver/nvidia</code></strong> \u2014 this means the NVIDIA driver is not accessible inside the container at all. The GPU hardware may exist on the host, but the container cannot reach it.</li>\n<li><strong>JAX also has a CUDA plugin mismatch</strong>: <code>jax_cuda13_plugin v0.11.0</code> is incompatible with <code>jaxlib v0.11.2</code>.</li>\n</ol>\n<h3 id=\"what-needs-to-happen\">What Needs to Happen</h3>\n<span class=\"paragraph\">The problem is <strong>not</strong> with your Python scripts or PyTorch installation \u2014 it's that the <strong>Molab container does not have GPU access configured</strong>. See the fix steps below.</span></span>"
}
}
],
"console": []
},
{
"id": "PKri",
"code_hash": "cbb92b645fbce36bb05be35b0050b993",
"outputs": [],
"console": []
},
{
"id": "Xref",
"code_hash": "e5c259217edc1ee4fb636163544d13f9",
"outputs": [],
"console": []
},
{
"id": "SFPL",
"code_hash": "759edb93690b45a7fb4fcbba25445967",
"outputs": [
{
"type": "data",
"data": {
"text/plain": ""
}
}
],
"console": [
{
"type": "stream",
"name": "stdout",
"text": "python: 3.13.11 /tmp/uv-venv/bin/python\ntorch, triton: 2.11.0+cu130 3.6.0\nhelion: Version: 1.5.1\n",
"mimetype": "text/plain"
}
]
},
{
"id": "BYtC",
"code_hash": "66ea21b2bc2ce38743edd9613eac2d87",
"outputs": [],
"console": []
},
{
"id": "RGSE",
"code_hash": "6470f3209ebb8c510b3f5b2f231bfc62",
"outputs": [
{
"type": "data",
"data": {
"text/html": "<pre class='text-xs'>0</pre>"
}
}
],
"console": [
{
"type": "stream",
"name": "stderr",
"text": "/tmp/uv-venv/lib/python3.13/site-packages/torchao/quantization/quant_api.py:1558: SyntaxWarning: invalid escape sequence '\\.'\n * regex for parameter names, must start with `re:`, e.g. `re:language\\.layers\\..+\\.q_proj.weight`.\n",
"mimetype": "text/plain"
},
{
"type": "stream",
"name": "stdout",
"text": "[Tok] Found 6 file(s) in ./training_data\n[Tok] Fingerprint: 83a770868891c636 (document_boundaries=True)\n[Tok] Tokenizing with 6 worker(s) ...\n[Tok] 1/6 files done | 13,452,009 tokens | 6.8s\n[Tok] 2/6 files done | 270,323,821 tokens | 99.7s\n[Tok] 3/6 files done | 770,143,654 tokens | 188.4s\n[Tok] 4/6 files done | 1,779,412,071 tokens | 371.8s\n[Tok] 5/6 files done | 2,873,479,424 tokens | 428.4s\n[Tok] 6/6 files done | 4,179,431,280 tokens | 481.8s\n[Tok] Assembling 4,179,431,280 tokens into ./training_data/.token_cache/tokens_83a770868891c636.tmp.npy ...\n[Tok] Ranges sidecar -> ./training_data/.token_cache/tokens_83a770868891c636.ranges.json (6 entries)\n[Tok] Done in 536.6s | 4,179,431,280 tokens | 16.72 GB\n[Tok] Saved -> ./training_data/.token_cache/tokens_83a770868891c636.npy\n./training_data/.token_cache/tokens_83a770868891c636.npy\n",
"mimetype": "text/plain"
}
]
},
{
"id": "Kclp",
"code_hash": "5a157233f59154540805237fc3f714af",
"outputs": [],
"console": []
},
{
"id": "emfo",
"code_hash": "16f7001224229d5030ea9ad7077bab8a",
"outputs": [
{
"type": "data",
"data": {
"text/html": "<pre class='text-xs'>0</pre>"
}
}
],
"console": [
{
"type": "stream",
"name": "stderr",
"text": "[RANK 0] Detected kernel version 4.19.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n",
"mimetype": "text/plain"
},
{
"type": "stream",
"name": "stdout",
"text": "[Setup] GPU compute capability 12.x (Ampere+) -- TF32 enabled for fp32 matmuls/convs.\n[Setup] Parallelism: plain data-parallel (DDP) only -- no FSDP/tensor-parallel/pipeline-parallel. Appropriate for small device counts (e.g. dual T4); set use_fsdp=True in config.py when sharding is actually needed.\n[Setup] Using device: cuda\n[Setup] Num processes (GPUs): 1\n[Tokenizer] Loaded, vocab_size=12288\n[Dataset] Found 6 files in ./training_data (curriculum OFF)\n[Dataset] Cache hit -> ./training_data/.token_cache/tokens_83a770868891c636.npy (4,179,431,312 tokens)\n[Dataset] train=4,137,636,969 tokens, val=41,794,311 tokens\n[Val] fixed validation set: 33 batches x 8 seqs across 6 file(s) (cosmopedia_1000mb.txt:1, cosmopedia_2000mb.txt:4, fineweb_edu_4000mb.txt:8, littlecurriculum_5000mb.txt:10, smollm_cosmopedia_v2_5000mb.txt:8, tinystories_1000mb.txt:2)\n[Setup] GPU-side batching=True, num_workers=2 (from config.py)\n[Mem] 8.96/101.98 GB used after corpus+batcher on-device (91% free).\n[Engram] layers=[1, 8] max_ngram=3 compressed_vocab=9807 (raw 12288)\n[DocMask] ON: documents split at BOS-after-BOS (bos_id=12287), attention windows [None]\n[Engram] 2,532,736 engram params\n[triton_kernel] loaded in pid=33709\n[TritonAudit] triton_kernel present but unavailable -- eager path.\n[Compile] Compiling model for maximum speed...\n[Compile] reduce-overhead requested via alpha_config.ALPHA_COMPILE_MODE. model.py's lm_head now reads the tied weight via F.linear(x, self.lm_head.weight) instead of the module call, specifically to remove the aliasing ambiguity that caused the earlier 'overwritten by a subsequent run' crash at wte(idx). This is a plausible fix, NOT a verified one -- it hasn't been run against the actual crash on real hardware. VERIFY before trusting a real run: run a few hundred steps, and if val_loss tracks normally (compare against a mode=default run on the same data/seed) and there's no CUDAGraphs RuntimeError, it's confirmed safe. If loss silently diverges or looks wrong without an explicit crash, that's worse than a crash -- stop and revert to mode=default immediately.\n[Compile/diag] reduce-overhead active. reserved_bytes.all.current=8104.0 MiB, num_alloc_retries=0 -- watch this grow abnormally across the first few steps as a sign of multiple cudagraph pools being captured (one per accumulate()/sync branch).\n[Model] Total params: 23.78M\n[Checkpoint] No checkpoint found, starting fresh\nstep 0 | epoch 0.00 | loss 9.4869 | ce 9.4869 | lr 4.00e-06 | tok/s 3,090 | elapsed 1.8min\nstep 50 | epoch 0.00 | loss 8.8555 | ce 8.1037 | lr 1.04e-04 | tok/s 363,404 | elapsed 2.5min\nstep 100 | epoch 0.01 | loss 7.2889 | ce 6.5953 | lr 2.04e-04 | tok/s 517,146 | elapsed 3.0min\nstep 150 | epoch 0.01 | loss 6.0934 | ce 5.7234 | lr 3.04e-04 | tok/s 516,373 | elapsed 3.6min\nstep 200 | epoch 0.02 | loss 5.3833 | ce 5.1494 | lr 4.04e-04 | tok/s 516,285 | elapsed 4.1min\nstep 250 | epoch 0.02 | loss 4.9218 | ce 4.7564 | lr 5.04e-04 | tok/s 516,971 | elapsed 4.6min\nstep 300 | epoch 0.02 | loss 4.6083 | ce 4.5052 | lr 6.04e-04 | tok/s 517,010 | elapsed 5.2min\nstep 350 | epoch 0.03 | loss 4.3647 | ce 4.2413 | lr 7.04e-04 | tok/s 516,763 | elapsed 5.7min\nstep 400 | epoch 0.03 | loss 4.1559 | ce 4.0846 | lr 8.04e-04 | tok/s 516,680 | elapsed 6.2min\nstep 450 | epoch 0.04 | loss 3.9629 | ce 3.9028 | lr 9.04e-04 | tok/s 516,042 | elapsed 6.7min\nstep 500 | epoch 0.04 | loss 3.8155 | ce 3.7532 | lr 1.00e-03 | tok/s 515,703 | elapsed 7.3min\nstep 550 | epoch 0.04 | loss 3.7135 | ce 3.6901 | lr 1.00e-03 | tok/s 515,143 | elapsed 7.8min\nstep 600 | epoch 0.05 | loss 3.6035 | ce 3.5385 | lr 1.00e-03 | tok/s 515,441 | elapsed 8.3min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 601)\nstep 650 | epoch 0.05 | loss 3.5292 | ce 3.5196 | lr 1.00e-03 | tok/s 511,588 | elapsed 8.9min\nstep 700 | epoch 0.06 | loss 3.4675 | ce 3.4325 | lr 1.00e-03 | tok/s 514,574 | elapsed 9.4min\nstep 750 | epoch 0.06 | loss 3.4103 | ce 3.3885 | lr 1.00e-03 | tok/s 514,694 | elapsed 9.9min\nstep 800 | epoch 0.06 | loss 3.3629 | ce 3.3134 | lr 1.00e-03 | tok/s 514,146 | elapsed 10.5min\nstep 850 | epoch 0.07 | loss 3.3342 | ce 3.3325 | lr 1.00e-03 | tok/s 514,010 | elapsed 11.0min\nstep 900 | epoch 0.07 | loss 3.3064 | ce 3.2679 | lr 1.00e-03 | tok/s 513,729 | elapsed 11.5min\nstep 950 | epoch 0.08 | loss 3.2726 | ce 3.2956 | lr 1.00e-03 | tok/s 513,731 | elapsed 12.1min\nstep 1000 | epoch 0.08 | loss 3.2517 | ce 3.1991 | lr 1.00e-03 | tok/s 513,663 | elapsed 12.6min\nstep 1000 | val_loss 3.2163 | val_acc 0.3935 | val_ppl 24.93\nstep 1050 | epoch 0.08 | loss 3.2301 | ce 3.2228 | lr 1.00e-03 | tok/s 477,306 | elapsed 13.2min\nstep 1100 | epoch 0.09 | loss 3.2151 | ce 3.1909 | lr 1.00e-03 | tok/s 513,867 | elapsed 13.7min\nstep 1150 | epoch 0.09 | loss 3.1891 | ce 3.2047 | lr 1.00e-03 | tok/s 513,457 | elapsed 14.2min\nstep 1200 | epoch 0.10 | loss 3.1760 | ce 3.2097 | lr 1.00e-03 | tok/s 513,740 | elapsed 14.8min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 1201)\nstep 1250 | epoch 0.10 | loss 3.1586 | ce 3.1653 | lr 1.00e-03 | tok/s 512,629 | elapsed 15.3min\nstep 1300 | epoch 0.10 | loss 3.1458 | ce 3.1605 | lr 1.00e-03 | tok/s 513,016 | elapsed 15.8min\nstep 1350 | epoch 0.11 | loss 3.1362 | ce 3.1297 | lr 1.00e-03 | tok/s 512,783 | elapsed 16.4min\nstep 1400 | epoch 0.11 | loss 3.1251 | ce 3.0709 | lr 1.00e-03 | tok/s 513,109 | elapsed 16.9min\nstep 1450 | epoch 0.11 | loss 3.1056 | ce 3.0761 | lr 1.00e-03 | tok/s 513,195 | elapsed 17.4min\nstep 1500 | epoch 0.12 | loss 3.1029 | ce 3.0663 | lr 1.00e-03 | tok/s 513,355 | elapsed 17.9min\nstep 1550 | epoch 0.12 | loss 3.0933 | ce 3.1185 | lr 1.00e-03 | tok/s 513,100 | elapsed 18.5min\nstep 1600 | epoch 0.13 | loss 3.0842 | ce 3.0943 | lr 1.00e-03 | tok/s 512,880 | elapsed 19.0min\nstep 1650 | epoch 0.13 | loss 3.0826 | ce 3.0913 | lr 1.00e-03 | tok/s 513,302 | elapsed 19.5min\nstep 1700 | epoch 0.13 | loss 3.0677 | ce 3.0796 | lr 1.00e-03 | tok/s 512,821 | elapsed 20.1min\nstep 1750 | epoch 0.14 | loss 3.0537 | ce 3.0562 | lr 1.00e-03 | tok/s 513,053 | elapsed 20.6min\nstep 1800 | epoch 0.14 | loss 3.0535 | ce 3.0498 | lr 1.00e-03 | tok/s 513,303 | elapsed 21.1min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 1801)\nstep 1850 | epoch 0.15 | loss 3.0419 | ce 3.0738 | lr 1.00e-03 | tok/s 513,070 | elapsed 21.7min\nstep 1900 | epoch 0.15 | loss 3.0397 | ce 3.0310 | lr 1.00e-03 | tok/s 513,479 | elapsed 22.2min\nstep 1950 | epoch 0.15 | loss 3.0311 | ce 3.0196 | lr 1.00e-03 | tok/s 513,245 | elapsed 22.7min\nstep 2000 | epoch 0.16 | loss 3.0267 | ce 3.0684 | lr 1.00e-03 | tok/s 513,926 | elapsed 23.3min\nstep 2000 | val_loss 2.9920 | val_acc 0.4184 | val_ppl 19.93\nstep 2050 | epoch 0.16 | loss 3.0062 | ce 2.9481 | lr 1.00e-03 | tok/s 455,770 | elapsed 23.9min\nstep 2100 | epoch 0.17 | loss 3.0144 | ce 2.9917 | lr 1.00e-03 | tok/s 512,501 | elapsed 24.4min\nstep 2150 | epoch 0.17 | loss 3.0063 | ce 2.9750 | lr 1.00e-03 | tok/s 512,627 | elapsed 24.9min\nstep 2200 | epoch 0.17 | loss 3.0003 | ce 3.0238 | lr 1.00e-03 | tok/s 512,296 | elapsed 25.5min\nstep 2250 | epoch 0.18 | loss 2.9955 | ce 3.0522 | lr 1.00e-03 | tok/s 513,161 | elapsed 26.0min\nstep 2300 | epoch 0.18 | loss 2.9891 | ce 2.9937 | lr 1.00e-03 | tok/s 512,946 | elapsed 26.5min\nstep 2350 | epoch 0.19 | loss 2.9868 | ce 3.0349 | lr 1.00e-03 | tok/s 513,538 | elapsed 27.1min\nstep 2400 | epoch 0.19 | loss 2.9853 | ce 2.9938 | lr 1.00e-03 | tok/s 514,073 | elapsed 27.6min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 2401)\nstep 2450 | epoch 0.19 | loss 2.9859 | ce 2.9862 | lr 1.00e-03 | tok/s 512,995 | elapsed 28.1min\nstep 2500 | epoch 0.20 | loss 2.9795 | ce 2.9690 | lr 1.00e-03 | tok/s 514,757 | elapsed 28.7min\nstep 2550 | epoch 0.20 | loss 2.9669 | ce 3.0139 | lr 1.00e-03 | tok/s 515,337 | elapsed 29.2min\nstep 2600 | epoch 0.21 | loss 2.9701 | ce 2.9526 | lr 1.00e-03 | tok/s 514,861 | elapsed 29.7min\nstep 2650 | epoch 0.21 | loss 2.9582 | ce 2.9649 | lr 1.00e-03 | tok/s 514,656 | elapsed 30.2min\nstep 2700 | epoch 0.21 | loss 2.9663 | ce 2.9320 | lr 1.00e-03 | tok/s 514,338 | elapsed 30.8min\nstep 2750 | epoch 0.22 | loss 2.9491 | ce 2.9213 | lr 1.00e-03 | tok/s 514,772 | elapsed 31.3min\nstep 2800 | epoch 0.22 | loss 2.9532 | ce 2.9117 | lr 1.00e-03 | tok/s 514,600 | elapsed 31.8min\nstep 2850 | epoch 0.23 | loss 2.9520 | ce 2.9820 | lr 1.00e-03 | tok/s 514,801 | elapsed 32.4min\nstep 2900 | epoch 0.23 | loss 2.9492 | ce 2.9720 | lr 1.00e-03 | tok/s 514,757 | elapsed 32.9min\nstep 2950 | epoch 0.23 | loss 2.9453 | ce 2.9705 | lr 1.00e-03 | tok/s 515,305 | elapsed 33.4min\nstep 3000 | epoch 0.24 | loss 2.9349 | ce 2.9573 | lr 1.00e-03 | tok/s 514,309 | elapsed 34.0min\nstep 3000 | val_loss 2.9078 | val_acc 0.4289 | val_ppl 18.32\n[Export] step 3001: 194 tensors (+1 aliased) -> ./checkpoints/model_step3001.safetensors (114.1 MB, 3.28s)\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 3001)\nstep 3050 | epoch 0.24 | loss 2.9406 | ce 2.9532 | lr 1.00e-03 | tok/s 457,204 | elapsed 34.6min\nstep 3100 | epoch 0.25 | loss 2.9372 | ce 2.8889 | lr 1.00e-03 | tok/s 514,810 | elapsed 35.1min\nstep 3150 | epoch 0.25 | loss 2.9347 | ce 2.9377 | lr 1.00e-03 | tok/s 515,096 | elapsed 35.6min\nstep 3200 | epoch 0.25 | loss 2.9202 | ce 2.9010 | lr 1.00e-03 | tok/s 513,595 | elapsed 36.2min\nstep 3250 | epoch 0.26 | loss 2.9277 | ce 2.9291 | lr 1.00e-03 | tok/s 513,924 | elapsed 36.7min\nstep 3300 | epoch 0.26 | loss 2.9218 | ce 2.9293 | lr 1.00e-03 | tok/s 513,359 | elapsed 37.2min\nstep 3350 | epoch 0.27 | loss 2.9181 | ce 2.9201 | lr 1.00e-03 | tok/s 513,195 | elapsed 37.7min\nstep 3400 | epoch 0.27 | loss 2.9123 | ce 2.9581 | lr 1.00e-03 | tok/s 513,384 | elapsed 38.3min\nstep 3450 | epoch 0.27 | loss 2.9138 | ce 2.9242 | lr 1.00e-03 | tok/s 513,341 | elapsed 38.8min\nstep 3500 | epoch 0.28 | loss 2.9156 | ce 2.8908 | lr 1.00e-03 | tok/s 513,644 | elapsed 39.3min\nstep 3550 | epoch 0.28 | loss 2.9005 | ce 2.9377 | lr 1.00e-03 | tok/s 514,124 | elapsed 39.9min\nstep 3600 | epoch 0.29 | loss 2.8988 | ce 2.9116 | lr 1.00e-03 | tok/s 514,246 | elapsed 40.4min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 3601)\nstep 3650 | epoch 0.29 | loss 2.9020 | ce 2.8569 | lr 1.00e-03 | tok/s 514,059 | elapsed 40.9min\nstep 3700 | epoch 0.29 | loss 2.8989 | ce 2.8479 | lr 1.00e-03 | tok/s 514,305 | elapsed 41.5min\nstep 3750 | epoch 0.30 | loss 2.8980 | ce 2.9182 | lr 1.00e-03 | tok/s 514,668 | elapsed 42.0min\nstep 3800 | epoch 0.30 | loss 2.8944 | ce 2.8939 | lr 1.00e-03 | tok/s 513,631 | elapsed 42.5min\nstep 3850 | epoch 0.30 | loss 2.8902 | ce 2.8974 | lr 1.00e-03 | tok/s 513,858 | elapsed 43.1min\nstep 3900 | epoch 0.31 | loss 2.8897 | ce 2.8891 | lr 1.00e-03 | tok/s 514,406 | elapsed 43.6min\nstep 3950 | epoch 0.31 | loss 2.9010 | ce 2.8211 | lr 1.00e-03 | tok/s 514,972 | elapsed 44.1min\nstep 4000 | epoch 0.32 | loss 2.8922 | ce 2.9309 | lr 1.00e-03 | tok/s 514,200 | elapsed 44.7min\nstep 4000 | val_loss 2.8607 | val_acc 0.4348 | val_ppl 17.47\nstep 4050 | epoch 0.32 | loss 2.8933 | ce 2.8909 | lr 1.00e-03 | tok/s 504,264 | elapsed 45.2min\nstep 4100 | epoch 0.32 | loss 2.8866 | ce 2.8892 | lr 1.00e-03 | tok/s 513,817 | elapsed 45.7min\nstep 4150 | epoch 0.33 | loss 2.8814 | ce 2.8858 | lr 1.00e-03 | tok/s 512,576 | elapsed 46.3min\nstep 4200 | epoch 0.33 | loss 2.8874 | ce 2.8429 | lr 1.00e-03 | tok/s 513,298 | elapsed 46.8min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 4201)\nstep 4250 | epoch 0.34 | loss 2.8844 | ce 2.8381 | lr 1.00e-03 | tok/s 513,454 | elapsed 47.3min\nstep 4300 | epoch 0.34 | loss 2.8810 | ce 2.8731 | lr 1.00e-03 | tok/s 514,234 | elapsed 47.9min\nstep 4350 | epoch 0.34 | loss 2.8781 | ce 2.8870 | lr 1.00e-03 | tok/s 514,587 | elapsed 48.4min\nstep 4400 | epoch 0.35 | loss 2.8729 | ce 2.8459 | lr 1.00e-03 | tok/s 514,566 | elapsed 48.9min\nstep 4450 | epoch 0.35 | loss 2.8794 | ce 2.8447 | lr 1.00e-03 | tok/s 514,941 | elapsed 49.4min\nstep 4500 | epoch 0.36 | loss 2.8737 | ce 2.8227 | lr 1.00e-03 | tok/s 515,259 | elapsed 50.0min\nstep 4550 | epoch 0.36 | loss 2.8658 | ce 2.8771 | lr 1.00e-03 | tok/s 515,573 | elapsed 50.5min\nstep 4600 | epoch 0.36 | loss 2.8642 | ce 2.8583 | lr 1.00e-03 | tok/s 515,175 | elapsed 51.0min\nstep 4650 | epoch 0.37 | loss 2.8741 | ce 2.8572 | lr 1.00e-03 | tok/s 514,435 | elapsed 51.6min\nstep 4700 | epoch 0.37 | loss 2.8684 | ce 2.8927 | lr 1.00e-03 | tok/s 514,137 | elapsed 52.1min\nstep 4750 | epoch 0.38 | loss 2.8647 | ce 2.9059 | lr 1.00e-03 | tok/s 512,987 | elapsed 52.6min\nstep 4800 | epoch 0.38 | loss 2.8578 | ce 2.7831 | lr 1.00e-03 | tok/s 513,432 | elapsed 53.2min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 4801)\nstep 4850 | epoch 0.38 | loss 2.8573 | ce 2.8763 | lr 1.00e-03 | tok/s 513,139 | elapsed 53.7min\nstep 4900 | epoch 0.39 | loss 2.8544 | ce 2.8666 | lr 1.00e-03 | tok/s 513,979 | elapsed 54.2min\nstep 4950 | epoch 0.39 | loss 2.8611 | ce 2.8346 | lr 1.00e-03 | tok/s 514,630 | elapsed 54.8min\nstep 5000 | epoch 0.40 | loss 2.8557 | ce 2.7888 | lr 1.00e-03 | tok/s 514,745 | elapsed 55.3min\nstep 5000 | val_loss 2.8249 | val_acc 0.4398 | val_ppl 16.86\nstep 5050 | epoch 0.40 | loss 2.8517 | ce 2.8615 | lr 1.00e-03 | tok/s 505,257 | elapsed 55.8min\nstep 5100 | epoch 0.40 | loss 2.8497 | ce 2.8364 | lr 1.00e-03 | tok/s 514,402 | elapsed 56.4min\nstep 5150 | epoch 0.41 | loss 2.8654 | ce 2.8043 | lr 1.00e-03 | tok/s 514,333 | elapsed 56.9min\nstep 5200 | epoch 0.41 | loss 2.8476 | ce 2.8571 | lr 1.00e-03 | tok/s 513,244 | elapsed 57.4min\nstep 5250 | epoch 0.42 | loss 2.8487 | ce 2.8798 | lr 1.00e-03 | tok/s 513,812 | elapsed 58.0min\nstep 5300 | epoch 0.42 | loss 2.8485 | ce 2.8765 | lr 1.00e-03 | tok/s 513,331 | elapsed 58.5min\nstep 5350 | epoch 0.42 | loss 2.8459 | ce 2.8483 | lr 1.00e-03 | tok/s 513,239 | elapsed 59.0min\nstep 5400 | epoch 0.43 | loss 2.8492 | ce 2.8641 | lr 1.00e-03 | tok/s 513,034 | elapsed 59.5min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 5401)\nstep 5450 | epoch 0.43 | loss 2.8458 | ce 2.8819 | lr 1.00e-03 | tok/s 512,973 | elapsed 60.1min\nstep 5500 | epoch 0.44 | loss 2.8520 | ce 2.8339 | lr 1.00e-03 | tok/s 512,998 | elapsed 60.6min\nstep 5550 | epoch 0.44 | loss 2.8306 | ce 2.7871 | lr 1.00e-03 | tok/s 513,057 | elapsed 61.1min\nstep 5600 | epoch 0.44 | loss 2.8373 | ce 2.8232 | lr 1.00e-03 | tok/s 513,432 | elapsed 61.7min\nstep 5650 | epoch 0.45 | loss 2.8383 | ce 2.8451 | lr 1.00e-03 | tok/s 513,824 | elapsed 62.2min\nstep 5700 | epoch 0.45 | loss 2.8464 | ce 2.8253 | lr 1.00e-03 | tok/s 514,010 | elapsed 62.7min\nstep 5750 | epoch 0.46 | loss 2.8388 | ce 2.7875 | lr 1.00e-03 | tok/s 514,465 | elapsed 63.3min\nstep 5800 | epoch 0.46 | loss 2.8363 | ce 2.8168 | lr 1.00e-03 | tok/s 513,322 | elapsed 63.8min\nstep 5850 | epoch 0.46 | loss 2.8344 | ce 2.8476 | lr 1.00e-03 | tok/s 513,463 | elapsed 64.3min\nstep 5900 | epoch 0.47 | loss 2.8436 | ce 2.8406 | lr 1.00e-03 | tok/s 513,127 | elapsed 64.9min\nstep 5950 | epoch 0.47 | loss 2.8325 | ce 2.8527 | lr 1.00e-03 | tok/s 512,820 | elapsed 65.4min\nstep 6000 | epoch 0.48 | loss 2.8375 | ce 2.8413 | lr 1.00e-03 | tok/s 513,382 | elapsed 65.9min\nstep 6000 | val_loss 2.8020 | val_acc 0.4433 | val_ppl 16.48\n[Export] step 6001: 194 tensors (+1 aliased) -> ./checkpoints/model_step6001.safetensors (114.1 MB, 0.05s)\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 6001)\nstep 6050 | epoch 0.48 | loss 2.8403 | ce 2.8863 | lr 1.00e-03 | tok/s 501,429 | elapsed 66.5min\nstep 6100 | epoch 0.48 | loss 2.8272 | ce 2.8477 | lr 1.00e-03 | tok/s 513,534 | elapsed 67.0min\nstep 6150 | epoch 0.49 | loss 2.8255 | ce 2.8429 | lr 1.00e-03 | tok/s 512,617 | elapsed 67.5min\nstep 6200 | epoch 0.49 | loss 2.8352 | ce 2.8033 | lr 1.00e-03 | tok/s 512,301 | elapsed 68.1min\nstep 6250 | epoch 0.49 | loss 2.8309 | ce 2.8363 | lr 1.00e-03 | tok/s 512,582 | elapsed 68.6min\nstep 6300 | epoch 0.50 | loss 2.8245 | ce 2.7591 | lr 1.00e-03 | tok/s 513,094 | elapsed 69.1min\nstep 6350 | epoch 0.50 | loss 2.8247 | ce 2.8357 | lr 1.00e-03 | tok/s 512,194 | elapsed 69.7min\nstep 6400 | epoch 0.51 | loss 2.8226 | ce 2.7979 | lr 1.00e-03 | tok/s 512,077 | elapsed 70.2min\nstep 6450 | epoch 0.51 | loss 2.8313 | ce 2.8595 | lr 1.00e-03 | tok/s 512,230 | elapsed 70.7min\nstep 6500 | epoch 0.51 | loss 2.8217 | ce 2.7978 | lr 1.00e-03 | tok/s 512,269 | elapsed 71.3min\nstep 6550 | epoch 0.52 | loss 2.8229 | ce 2.8457 | lr 1.00e-03 | tok/s 512,354 | elapsed 71.8min\nstep 6600 | epoch 0.52 | loss 2.8237 | ce 2.8690 | lr 1.00e-03 | tok/s 512,522 | elapsed 72.3min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 6601)\nstep 6650 | epoch 0.53 | loss 2.8218 | ce 2.8206 | lr 1.00e-03 | tok/s 512,178 | elapsed 72.9min\nstep 6700 | epoch 0.53 | loss 2.8174 | ce 2.8573 | lr 1.00e-03 | tok/s 512,334 | elapsed 73.4min\nstep 6750 | epoch 0.53 | loss 2.8150 | ce 2.8370 | lr 1.00e-03 | tok/s 512,728 | elapsed 73.9min\nstep 6800 | epoch 0.54 | loss 2.8180 | ce 2.8580 | lr 1.00e-03 | tok/s 513,022 | elapsed 74.5min\nstep 6850 | epoch 0.54 | loss 2.8281 | ce 2.8132 | lr 1.00e-03 | tok/s 512,968 | elapsed 75.0min\nstep 6900 | epoch 0.55 | loss 2.8178 | ce 2.8154 | lr 1.00e-03 | tok/s 512,240 | elapsed 75.5min\nstep 6950 | epoch 0.55 | loss 2.8207 | ce 2.8321 | lr 1.00e-03 | tok/s 513,304 | elapsed 76.1min\nstep 7000 | epoch 0.55 | loss 2.8173 | ce 2.8385 | lr 1.00e-03 | tok/s 513,304 | elapsed 76.6min\nstep 7000 | val_loss 2.7840 | val_acc 0.4452 | val_ppl 16.18\nstep 7050 | epoch 0.56 | loss 2.8097 | ce 2.8206 | lr 1.00e-03 | tok/s 504,197 | elapsed 77.1min\nstep 7100 | epoch 0.56 | loss 2.8195 | ce 2.7813 | lr 1.00e-03 | tok/s 513,330 | elapsed 77.7min\nstep 7150 | epoch 0.57 | loss 2.8144 | ce 2.8892 | lr 1.00e-03 | tok/s 512,068 | elapsed 78.2min\nstep 7200 | epoch 0.57 | loss 2.8069 | ce 2.7879 | lr 1.00e-03 | tok/s 512,898 | elapsed 78.7min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 7201)\nstep 7250 | epoch 0.57 | loss 2.8141 | ce 2.7636 | lr 1.00e-03 | tok/s 512,311 | elapsed 79.3min\nstep 7300 | epoch 0.58 | loss 2.8097 | ce 2.8011 | lr 1.00e-03 | tok/s 512,491 | elapsed 79.8min\nstep 7350 | epoch 0.58 | loss 2.8209 | ce 2.8291 | lr 1.00e-03 | tok/s 512,155 | elapsed 80.3min\nstep 7400 | epoch 0.59 | loss 2.8133 | ce 2.7726 | lr 1.00e-03 | tok/s 512,242 | elapsed 80.9min\nstep 7450 | epoch 0.59 | loss 2.8112 | ce 2.8194 | lr 1.00e-03 | tok/s 512,583 | elapsed 81.4min\nstep 7500 | epoch 0.59 | loss 2.8085 | ce 2.7988 | lr 1.00e-03 | tok/s 512,421 | elapsed 81.9min\nstep 7550 | epoch 0.60 | loss 2.8048 | ce 2.7982 | lr 1.00e-03 | tok/s 512,031 | elapsed 82.5min\nstep 7600 | epoch 0.60 | loss 2.8135 | ce 2.8221 | lr 1.00e-03 | tok/s 512,657 | elapsed 83.0min\nstep 7650 | epoch 0.61 | loss 2.8040 | ce 2.7481 | lr 1.00e-03 | tok/s 512,617 | elapsed 83.5min\nstep 7700 | epoch 0.61 | loss 2.8124 | ce 2.7929 | lr 1.00e-03 | tok/s 512,612 | elapsed 84.1min\nstep 7750 | epoch 0.61 | loss 2.8025 | ce 2.7220 | lr 1.00e-03 | tok/s 512,839 | elapsed 84.6min\nstep 7800 | epoch 0.62 | loss 2.8077 | ce 2.8605 | lr 1.00e-03 | tok/s 513,316 | elapsed 85.1min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 7801)\nstep 7850 | epoch 0.62 | loss 2.8094 | ce 2.8137 | lr 1.00e-03 | tok/s 513,446 | elapsed 85.7min\nstep 7900 | epoch 0.63 | loss 2.8070 | ce 2.7868 | lr 1.00e-03 | tok/s 512,876 | elapsed 86.2min\nstep 7950 | epoch 0.63 | loss 2.8051 | ce 2.8221 | lr 1.00e-03 | tok/s 512,277 | elapsed 86.7min\nstep 8000 | epoch 0.63 | loss 2.7930 | ce 2.7904 | lr 1.00e-03 | tok/s 512,547 | elapsed 87.3min\nstep 8000 | val_loss 2.7712 | val_acc 0.4467 | val_ppl 15.98\nstep 8050 | epoch 0.64 | loss 2.8000 | ce 2.8436 | lr 1.00e-03 | tok/s 503,450 | elapsed 87.8min\nstep 8100 | epoch 0.64 | loss 2.8027 | ce 2.7912 | lr 1.00e-03 | tok/s 512,381 | elapsed 88.3min\nstep 8150 | epoch 0.65 | loss 2.7961 | ce 2.8081 | lr 1.00e-03 | tok/s 512,735 | elapsed 88.9min\nstep 8200 | epoch 0.65 | loss 2.7932 | ce 2.8361 | lr 1.00e-03 | tok/s 513,529 | elapsed 89.4min\nstep 8250 | epoch 0.65 | loss 2.8066 | ce 2.8162 | lr 1.00e-03 | tok/s 513,161 | elapsed 89.9min\nstep 8300 | epoch 0.66 | loss 2.8007 | ce 2.8113 | lr 1.00e-03 | tok/s 512,837 | elapsed 90.5min\nstep 8350 | epoch 0.66 | loss 2.7934 | ce 2.7807 | lr 1.00e-03 | tok/s 512,514 | elapsed 91.0min\nstep 8400 | epoch 0.67 | loss 2.7868 | ce 2.8165 | lr 1.00e-03 | tok/s 512,058 | elapsed 91.5min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 8401)\nstep 8450 | epoch 0.67 | loss 2.7972 | ce 2.7548 | lr 1.00e-03 | tok/s 511,639 | elapsed 92.1min\nstep 8500 | epoch 0.67 | loss 2.7987 | ce 2.7755 | lr 1.00e-03 | tok/s 512,384 | elapsed 92.6min\nstep 8550 | epoch 0.68 | loss 2.7951 | ce 2.8559 | lr 1.00e-03 | tok/s 512,326 | elapsed 93.1min\nstep 8600 | epoch 0.68 | loss 2.7941 | ce 2.8313 | lr 1.00e-03 | tok/s 512,363 | elapsed 93.7min\nstep 8650 | epoch 0.69 | loss 2.7946 | ce 2.7491 | lr 1.00e-03 | tok/s 512,147 | elapsed 94.2min\nstep 8700 | epoch 0.69 | loss 2.7889 | ce 2.7933 | lr 1.00e-03 | tok/s 512,377 | elapsed 94.7min\nstep 8750 | epoch 0.69 | loss 2.7919 | ce 2.8031 | lr 1.00e-03 | tok/s 512,210 | elapsed 95.3min\nstep 8800 | epoch 0.70 | loss 2.7856 | ce 2.7430 | lr 1.00e-03 | tok/s 512,300 | elapsed 95.8min\nstep 8850 | epoch 0.70 | loss 2.7933 | ce 2.8493 | lr 1.00e-03 | tok/s 511,910 | elapsed 96.3min\nstep 8900 | epoch 0.70 | loss 2.7949 | ce 2.7333 | lr 1.00e-03 | tok/s 511,901 | elapsed 96.9min\nstep 8950 | epoch 0.71 | loss 2.7891 | ce 2.7714 | lr 1.00e-03 | tok/s 511,873 | elapsed 97.4min\nstep 9000 | epoch 0.71 | loss 2.7847 | ce 2.8276 | lr 1.00e-03 | tok/s 512,114 | elapsed 97.9min\nstep 9000 | val_loss 2.7594 | val_acc 0.4480 | val_ppl 15.79\n[Export] step 9001: 194 tensors (+1 aliased) -> ./checkpoints/model_step9001.safetensors (114.1 MB, 0.05s)\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 9001)\nstep 9050 | epoch 0.72 | loss 2.7870 | ce 2.7389 | lr 1.00e-03 | tok/s 501,110 | elapsed 98.5min\nstep 9100 | epoch 0.72 | loss 2.7830 | ce 2.7636 | lr 1.00e-03 | tok/s 512,924 | elapsed 99.0min\nstep 9150 | epoch 0.72 | loss 2.7895 | ce 2.7919 | lr 1.00e-03 | tok/s 512,033 | elapsed 99.5min\nstep 9200 | epoch 0.73 | loss 2.7879 | ce 2.7666 | lr 1.00e-03 | tok/s 512,017 | elapsed 100.1min\nstep 9250 | epoch 0.73 | loss 2.7849 | ce 2.7981 | lr 1.00e-03 | tok/s 512,493 | elapsed 100.6min\nstep 9300 | epoch 0.74 | loss 2.7799 | ce 2.7809 | lr 1.00e-03 | tok/s 512,508 | elapsed 101.1min\nstep 9350 | epoch 0.74 | loss 2.7860 | ce 2.8281 | lr 1.00e-03 | tok/s 513,398 | elapsed 101.7min\nstep 9400 | epoch 0.74 | loss 2.7851 | ce 2.7223 | lr 1.00e-03 | tok/s 512,876 | elapsed 102.2min\nstep 9450 | epoch 0.75 | loss 2.7802 | ce 2.7982 | lr 1.00e-03 | tok/s 512,654 | elapsed 102.7min\nstep 9500 | epoch 0.75 | loss 2.7765 | ce 2.7723 | lr 1.00e-03 | tok/s 512,408 | elapsed 103.3min\nstep 9550 | epoch 0.76 | loss 2.7888 | ce 2.8476 | lr 1.00e-03 | tok/s 512,101 | elapsed 103.8min\nstep 9600 | epoch 0.76 | loss 2.7729 | ce 2.7354 | lr 1.00e-03 | tok/s 512,446 | elapsed 104.3min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 9601)\nstep 9650 | epoch 0.76 | loss 2.7809 | ce 2.7435 | lr 1.00e-03 | tok/s 511,627 | elapsed 104.9min\nstep 9700 | epoch 0.77 | loss 2.7787 | ce 2.7347 | lr 1.00e-03 | tok/s 512,293 | elapsed 105.4min\nstep 9750 | epoch 0.77 | loss 2.7749 | ce 2.7735 | lr 1.00e-03 | tok/s 512,514 | elapsed 105.9min\nstep 9800 | epoch 0.78 | loss 2.7844 | ce 2.7364 | lr 1.00e-03 | tok/s 512,273 | elapsed 106.5min\nstep 9850 | epoch 0.78 | loss 2.7794 | ce 2.7825 | lr 1.00e-03 | tok/s 511,964 | elapsed 107.0min\nstep 9900 | epoch 0.78 | loss 2.7787 | ce 2.7640 | lr 1.00e-03 | tok/s 511,972 | elapsed 107.5min\nstep 9950 | epoch 0.79 | loss 2.7847 | ce 2.7653 | lr 1.00e-03 | tok/s 511,957 | elapsed 108.1min\nstep 10000 | epoch 0.79 | loss 2.7711 | ce 2.7791 | lr 1.00e-03 | tok/s 512,104 | elapsed 108.6min\nstep 10000 | val_loss 2.7461 | val_acc 0.4496 | val_ppl 15.58\nstep 10050 | epoch 0.80 | loss 2.7797 | ce 2.7785 | lr 1.00e-03 | tok/s 503,344 | elapsed 109.1min\nstep 10100 | epoch 0.80 | loss 2.7765 | ce 2.7785 | lr 1.00e-03 | tok/s 512,530 | elapsed 109.7min\nstep 10150 | epoch 0.80 | loss 2.7769 | ce 2.7748 | lr 1.00e-03 | tok/s 512,271 | elapsed 110.2min\nstep 10200 | epoch 0.81 | loss 2.7781 | ce 2.7824 | lr 1.00e-03 | tok/s 512,601 | elapsed 110.7min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 10201)\nstep 10250 | epoch 0.81 | loss 2.7756 | ce 2.7887 | lr 1.00e-03 | tok/s 512,135 | elapsed 111.3min\nstep 10300 | epoch 0.82 | loss 2.7802 | ce 2.7619 | lr 1.00e-03 | tok/s 512,656 | elapsed 111.8min\nstep 10350 | epoch 0.82 | loss 2.7725 | ce 2.7414 | lr 1.00e-03 | tok/s 512,186 | elapsed 112.3min\nstep 10400 | epoch 0.82 | loss 2.7813 | ce 2.7434 | lr 1.00e-03 | tok/s 512,206 | elapsed 112.9min\nstep 10450 | epoch 0.83 | loss 2.7692 | ce 2.7980 | lr 1.00e-03 | tok/s 512,437 | elapsed 113.4min\nstep 10500 | epoch 0.83 | loss 2.7651 | ce 2.7766 | lr 1.00e-03 | tok/s 512,724 | elapsed 113.9min\nstep 10550 | epoch 0.84 | loss 2.7609 | ce 2.7090 | lr 1.00e-03 | tok/s 512,487 | elapsed 114.5min\nstep 10600 | epoch 0.84 | loss 2.7776 | ce 2.7781 | lr 1.00e-03 | tok/s 512,543 | elapsed 115.0min\nstep 10650 | epoch 0.84 | loss 2.7652 | ce 2.7837 | lr 1.00e-03 | tok/s 512,709 | elapsed 115.5min\nstep 10700 | epoch 0.85 | loss 2.7711 | ce 2.7502 | lr 1.00e-03 | tok/s 512,311 | elapsed 116.1min\nstep 10750 | epoch 0.85 | loss 2.7714 | ce 2.7753 | lr 1.00e-03 | tok/s 512,107 | elapsed 116.6min\nstep 10800 | epoch 0.86 | loss 2.7703 | ce 2.7363 | lr 1.00e-03 | tok/s 512,264 | elapsed 117.1min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 10801)\nstep 10850 | epoch 0.86 | loss 2.7697 | ce 2.7669 | lr 1.00e-03 | tok/s 512,363 | elapsed 117.7min\nstep 10900 | epoch 0.86 | loss 2.7707 | ce 2.7214 | lr 1.00e-03 | tok/s 512,390 | elapsed 118.2min\nstep 10950 | epoch 0.87 | loss 2.7722 | ce 2.7706 | lr 1.00e-03 | tok/s 512,513 | elapsed 118.7min\nstep 11000 | epoch 0.87 | loss 2.7740 | ce 2.7819 | lr 1.00e-03 | tok/s 513,479 | elapsed 119.3min\nstep 11000 | val_loss 2.7396 | val_acc 0.4508 | val_ppl 15.48\nstep 11050 | epoch 0.88 | loss 2.7677 | ce 2.7745 | lr 1.00e-03 | tok/s 503,878 | elapsed 119.8min\nstep 11100 | epoch 0.88 | loss 2.7680 | ce 2.7165 | lr 1.00e-03 | tok/s 513,670 | elapsed 120.3min\nstep 11150 | epoch 0.88 | loss 2.7733 | ce 2.7884 | lr 1.00e-03 | tok/s 515,070 | elapsed 120.9min\nstep 11200 | epoch 0.89 | loss 2.7656 | ce 2.7828 | lr 1.00e-03 | tok/s 514,548 | elapsed 121.4min\nstep 11250 | epoch 0.89 | loss 2.7656 | ce 2.7785 | lr 1.00e-03 | tok/s 514,693 | elapsed 121.9min\nstep 11300 | epoch 0.89 | loss 2.7698 | ce 2.8118 | lr 1.00e-03 | tok/s 515,118 | elapsed 122.5min\nstep 11350 | epoch 0.90 | loss 2.7641 | ce 2.7871 | lr 1.00e-03 | tok/s 515,698 | elapsed 123.0min\nstep 11400 | epoch 0.90 | loss 2.7647 | ce 2.7230 | lr 1.00e-03 | tok/s 515,337 | elapsed 123.5min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 11401)\nstep 11450 | epoch 0.91 | loss 2.7665 | ce 2.7360 | lr 1.00e-03 | tok/s 514,922 | elapsed 124.0min\nstep 11500 | epoch 0.91 | loss 2.7649 | ce 2.6975 | lr 1.00e-03 | tok/s 514,601 | elapsed 124.6min\nstep 11550 | epoch 0.91 | loss 2.7574 | ce 2.7868 | lr 1.00e-03 | tok/s 515,090 | elapsed 125.1min\nstep 11600 | epoch 0.92 | loss 2.7619 | ce 2.8172 | lr 1.00e-03 | tok/s 515,199 | elapsed 125.6min\nstep 11650 | epoch 0.92 | loss 2.7655 | ce 2.7617 | lr 1.00e-03 | tok/s 515,270 | elapsed 126.2min\nstep 11700 | epoch 0.93 | loss 2.7593 | ce 2.7297 | lr 1.00e-03 | tok/s 514,894 | elapsed 126.7min\nstep 11750 | epoch 0.93 | loss 2.7649 | ce 2.7866 | lr 1.00e-03 | tok/s 515,130 | elapsed 127.2min\nstep 11800 | epoch 0.93 | loss 2.7605 | ce 2.8025 | lr 1.00e-03 | tok/s 515,079 | elapsed 127.8min\nstep 11850 | epoch 0.94 | loss 2.7676 | ce 2.8043 | lr 1.00e-03 | tok/s 515,966 | elapsed 128.3min\nstep 11900 | epoch 0.94 | loss 2.7642 | ce 2.7443 | lr 1.00e-03 | tok/s 515,651 | elapsed 128.8min\nstep 11950 | epoch 0.95 | loss 2.7670 | ce 2.7430 | lr 1.00e-03 | tok/s 515,633 | elapsed 129.3min\nstep 12000 | epoch 0.95 | loss 2.7615 | ce 2.7911 | lr 1.00e-03 | tok/s 515,206 | elapsed 129.9min\nstep 12000 | val_loss 2.7324 | val_acc 0.4516 | val_ppl 15.37\n[Export] step 12001: 194 tensors (+1 aliased) -> ./checkpoints/model_step12001.safetensors (114.1 MB, 0.05s)\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 12001)\nstep 12050 | epoch 0.95 | loss 2.7615 | ce 2.7475 | lr 1.00e-03 | tok/s 504,613 | elapsed 130.4min\nstep 12100 | epoch 0.96 | loss 2.7576 | ce 2.7657 | lr 1.00e-03 | tok/s 515,245 | elapsed 130.9min\nstep 12150 | epoch 0.96 | loss 2.7540 | ce 2.7406 | lr 1.00e-03 | tok/s 515,251 | elapsed 131.5min\nstep 12200 | epoch 0.97 | loss 2.7652 | ce 2.7572 | lr 1.00e-03 | tok/s 514,915 | elapsed 132.0min\nstep 12250 | epoch 0.97 | loss 2.7549 | ce 2.7503 | lr 1.00e-03 | tok/s 515,172 | elapsed 132.5min\nstep 12300 | epoch 0.97 | loss 2.7691 | ce 2.7684 | lr 1.00e-03 | tok/s 515,038 | elapsed 133.1min\nstep 12350 | epoch 0.98 | loss 2.7582 | ce 2.7499 | lr 1.00e-03 | tok/s 514,919 | elapsed 133.6min\nstep 12400 | epoch 0.98 | loss 2.7596 | ce 2.7423 | lr 1.00e-03 | tok/s 515,212 | elapsed 134.1min\nstep 12450 | epoch 0.99 | loss 2.7578 | ce 2.7880 | lr 1.00e-03 | tok/s 514,558 | elapsed 134.7min\nstep 12500 | epoch 0.99 | loss 2.7557 | ce 2.7271 | lr 1.00e-03 | tok/s 514,918 | elapsed 135.2min\nstep 12550 | epoch 0.99 | loss 2.7533 | ce 2.7727 | lr 1.00e-03 | tok/s 513,856 | elapsed 135.7min\nstep 12600 | epoch 1.00 | loss 2.7600 | ce 2.7319 | lr 1.00e-03 | tok/s 513,420 | elapsed 136.3min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 12601)\nstep 12650 | epoch 1.00 | loss 2.7587 | ce 2.7420 | lr 1.00e-03 | tok/s 513,520 | elapsed 136.8min\nstep 12700 | epoch 1.01 | loss 2.7610 | ce 2.7465 | lr 1.00e-03 | tok/s 513,411 | elapsed 137.3min\nstep 12750 | epoch 1.01 | loss 2.7583 | ce 2.7505 | lr 1.00e-03 | tok/s 514,368 | elapsed 137.8min\nstep 12800 | epoch 1.01 | loss 2.7545 | ce 2.7579 | lr 1.00e-03 | tok/s 515,224 | elapsed 138.4min\nstep 12850 | epoch 1.02 | loss 2.7569 | ce 2.7840 | lr 1.00e-03 | tok/s 514,578 | elapsed 138.9min\nstep 12900 | epoch 1.02 | loss 2.7524 | ce 2.7476 | lr 1.00e-03 | tok/s 515,105 | elapsed 139.4min\nstep 12950 | epoch 1.03 | loss 2.7610 | ce 2.7625 | lr 1.00e-03 | tok/s 514,637 | elapsed 140.0min\nstep 13000 | epoch 1.03 | loss 2.7473 | ce 2.7262 | lr 1.00e-03 | tok/s 514,614 | elapsed 140.5min\nstep 13000 | val_loss 2.7208 | val_acc 0.4531 | val_ppl 15.19\nstep 13050 | epoch 1.03 | loss 2.7548 | ce 2.7918 | lr 1.00e-03 | tok/s 505,607 | elapsed 141.0min\nstep 13100 | epoch 1.04 | loss 2.7565 | ce 2.7792 | lr 1.00e-03 | tok/s 514,981 | elapsed 141.6min\nstep 13150 | epoch 1.04 | loss 2.7531 | ce 2.7616 | lr 1.00e-03 | tok/s 513,859 | elapsed 142.1min\nstep 13200 | epoch 1.05 | loss 2.7436 | ce 2.7340 | lr 1.00e-03 | tok/s 514,633 | elapsed 142.6min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 13201)\nstep 13250 | epoch 1.05 | loss 2.7505 | ce 2.7427 | lr 1.00e-03 | tok/s 514,643 | elapsed 143.2min\nstep 13300 | epoch 1.05 | loss 2.7479 | ce 2.8008 | lr 1.00e-03 | tok/s 515,426 | elapsed 143.7min\nstep 13350 | epoch 1.06 | loss 2.7482 | ce 2.7341 | lr 1.00e-03 | tok/s 515,529 | elapsed 144.2min\nstep 13400 | epoch 1.06 | loss 2.7453 | ce 2.7200 | lr 1.00e-03 | tok/s 515,370 | elapsed 144.8min\nstep 13450 | epoch 1.07 | loss 2.7443 | ce 2.7120 | lr 1.00e-03 | tok/s 514,186 | elapsed 145.3min\nstep 13500 | epoch 1.07 | loss 2.7569 | ce 2.7525 | lr 1.00e-03 | tok/s 513,151 | elapsed 145.8min\nstep 13550 | epoch 1.07 | loss 2.7513 | ce 2.7541 | lr 1.00e-03 | tok/s 512,427 | elapsed 146.3min\nstep 13600 | epoch 1.08 | loss 2.7440 | ce 2.7200 | lr 1.00e-03 | tok/s 512,657 | elapsed 146.9min\nstep 13650 | epoch 1.08 | loss 2.7515 | ce 2.7841 | lr 1.00e-03 | tok/s 512,487 | elapsed 147.4min\nstep 13700 | epoch 1.08 | loss 2.7528 | ce 2.7871 | lr 1.00e-03 | tok/s 512,515 | elapsed 147.9min\nstep 13750 | epoch 1.09 | loss 2.7514 | ce 2.7618 | lr 1.00e-03 | tok/s 512,926 | elapsed 148.5min\nstep 13800 | epoch 1.09 | loss 2.7550 | ce 2.7390 | lr 1.00e-03 | tok/s 513,930 | elapsed 149.0min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 13801)\nstep 13850 | epoch 1.10 | loss 2.7548 | ce 2.7525 | lr 1.00e-03 | tok/s 513,151 | elapsed 149.5min\nstep 13900 | epoch 1.10 | loss 2.7468 | ce 2.7441 | lr 1.00e-03 | tok/s 513,204 | elapsed 150.1min\nstep 13950 | epoch 1.10 | loss 2.7479 | ce 2.7633 | lr 1.00e-03 | tok/s 512,780 | elapsed 150.6min\nstep 14000 | epoch 1.11 | loss 2.7505 | ce 2.7267 | lr 1.00e-03 | tok/s 513,072 | elapsed 151.1min\nstep 14000 | val_loss 2.7143 | val_acc 0.4539 | val_ppl 15.09\nstep 14050 | epoch 1.11 | loss 2.7441 | ce 2.7086 | lr 1.00e-03 | tok/s 503,876 | elapsed 151.7min\nstep 14100 | epoch 1.12 | loss 2.7506 | ce 2.7142 | lr 1.00e-03 | tok/s 513,550 | elapsed 152.2min\nstep 14150 | epoch 1.12 | loss 2.7489 | ce 2.7291 | lr 1.00e-03 | tok/s 513,225 | elapsed 152.7min\nstep 14200 | epoch 1.12 | loss 2.7492 | ce 2.7220 | lr 1.00e-03 | tok/s 514,341 | elapsed 153.3min\nstep 14250 | epoch 1.13 | loss 2.7471 | ce 2.7319 | lr 1.00e-03 | tok/s 513,288 | elapsed 153.8min\nstep 14300 | epoch 1.13 | loss 2.7458 | ce 2.7343 | lr 1.00e-03 | tok/s 513,868 | elapsed 154.3min\nstep 14350 | epoch 1.14 | loss 2.7426 | ce 2.7405 | lr 1.00e-03 | tok/s 514,408 | elapsed 154.9min\nstep 14400 | epoch 1.14 | loss 2.7456 | ce 2.7218 | lr 1.00e-03 | tok/s 513,900 | elapsed 155.4min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 14401)\nstep 14450 | epoch 1.14 | loss 2.7418 | ce 2.7637 | lr 1.00e-03 | tok/s 514,689 | elapsed 155.9min\nstep 14500 | epoch 1.15 | loss 2.7459 | ce 2.7480 | lr 1.00e-03 | tok/s 514,962 | elapsed 156.5min\nstep 14550 | epoch 1.15 | loss 2.7490 | ce 2.6903 | lr 1.00e-03 | tok/s 515,629 | elapsed 157.0min\nstep 14600 | epoch 1.16 | loss 2.7405 | ce 2.6805 | lr 1.00e-03 | tok/s 515,096 | elapsed 157.5min\nstep 14650 | epoch 1.16 | loss 2.7413 | ce 2.7664 | lr 1.00e-03 | tok/s 514,640 | elapsed 158.1min\nstep 14700 | epoch 1.16 | loss 2.7351 | ce 2.7796 | lr 1.00e-03 | tok/s 514,257 | elapsed 158.6min\nstep 14750 | epoch 1.17 | loss 2.7431 | ce 2.6591 | lr 1.00e-03 | tok/s 513,382 | elapsed 159.1min\nstep 14800 | epoch 1.17 | loss 2.7408 | ce 2.6871 | lr 1.00e-03 | tok/s 513,657 | elapsed 159.6min\nstep 14850 | epoch 1.18 | loss 2.7368 | ce 2.7414 | lr 1.00e-03 | tok/s 514,657 | elapsed 160.2min\nstep 14900 | epoch 1.18 | loss 2.7376 | ce 2.7495 | lr 1.00e-03 | tok/s 513,222 | elapsed 160.7min\nstep 14950 | epoch 1.18 | loss 2.7380 | ce 2.7502 | lr 1.00e-03 | tok/s 512,429 | elapsed 161.2min\nstep 15000 | epoch 1.19 | loss 2.7365 | ce 2.7581 | lr 1.00e-03 | tok/s 513,469 | elapsed 161.8min\nstep 15000 | val_loss 2.7118 | val_acc 0.4545 | val_ppl 15.06\n[Export] step 15001: 194 tensors (+1 aliased) -> ./checkpoints/model_step15001.safetensors (114.1 MB, 0.12s)\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 15001)\nstep 15050 | epoch 1.19 | loss 2.7387 | ce 2.7593 | lr 1.00e-03 | tok/s 502,687 | elapsed 162.3min\nstep 15100 | epoch 1.20 | loss 2.7403 | ce 2.7501 | lr 1.00e-03 | tok/s 514,191 | elapsed 162.8min\nstep 15150 | epoch 1.20 | loss 2.7435 | ce 2.7639 | lr 1.00e-03 | tok/s 514,297 | elapsed 163.4min\nstep 15200 | epoch 1.20 | loss 2.7343 | ce 2.7208 | lr 1.00e-03 | tok/s 513,829 | elapsed 163.9min\nstep 15250 | epoch 1.21 | loss 2.7374 | ce 2.7332 | lr 1.00e-03 | tok/s 515,081 | elapsed 164.4min\nstep 15300 | epoch 1.21 | loss 2.7290 | ce 2.7063 | lr 1.00e-03 | tok/s 514,676 | elapsed 165.0min\nstep 15350 | epoch 1.22 | loss 2.7316 | ce 2.7706 | lr 1.00e-03 | tok/s 515,277 | elapsed 165.5min\nstep 15400 | epoch 1.22 | loss 2.7450 | ce 2.7784 | lr 1.00e-03 | tok/s 514,054 | elapsed 166.0min\nstep 15450 | epoch 1.22 | loss 2.7439 | ce 2.7328 | lr 1.00e-03 | tok/s 514,629 | elapsed 166.6min\nstep 15500 | epoch 1.23 | loss 2.7457 | ce 2.7414 | lr 1.00e-03 | tok/s 514,167 | elapsed 167.1min\nstep 15550 | epoch 1.23 | loss 2.7330 | ce 2.7136 | lr 1.00e-03 | tok/s 514,762 | elapsed 167.6min\nstep 15600 | epoch 1.24 | loss 2.7330 | ce 2.7358 | lr 9.98e-04 | tok/s 514,155 | elapsed 168.2min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 15601)\nstep 15650 | epoch 1.24 | loss 2.7381 | ce 2.7639 | lr 9.96e-04 | tok/s 512,461 | elapsed 168.7min\nstep 15700 | epoch 1.24 | loss 2.7323 | ce 2.7383 | lr 9.93e-04 | tok/s 512,674 | elapsed 169.2min\nstep 15750 | epoch 1.25 | loss 2.7361 | ce 2.7349 | lr 9.89e-04 | tok/s 512,500 | elapsed 169.8min\nstep 15800 | epoch 1.25 | loss 2.7301 | ce 2.7006 | lr 9.84e-04 | tok/s 513,152 | elapsed 170.3min\nstep 15850 | epoch 1.26 | loss 2.7297 | ce 2.6967 | lr 9.78e-04 | tok/s 511,984 | elapsed 170.8min\nstep 15900 | epoch 1.26 | loss 2.7259 | ce 2.7234 | lr 9.71e-04 | tok/s 512,646 | elapsed 171.4min\nstep 15950 | epoch 1.26 | loss 2.7405 | ce 2.7727 | lr 9.64e-04 | tok/s 512,255 | elapsed 171.9min\nstep 16000 | epoch 1.27 | loss 2.7321 | ce 2.7162 | lr 9.55e-04 | tok/s 512,920 | elapsed 172.4min\nstep 16000 | val_loss 2.7008 | val_acc 0.4565 | val_ppl 14.89\nstep 16050 | epoch 1.27 | loss 2.7336 | ce 2.7485 | lr 9.46e-04 | tok/s 504,260 | elapsed 173.0min\nstep 16100 | epoch 1.28 | loss 2.7175 | ce 2.6817 | lr 9.36e-04 | tok/s 514,228 | elapsed 173.5min\nstep 16150 | epoch 1.28 | loss 2.7284 | ce 2.7200 | lr 9.25e-04 | tok/s 514,331 | elapsed 174.0min\nstep 16200 | epoch 1.28 | loss 2.7223 | ce 2.7064 | lr 9.14e-04 | tok/s 514,748 | elapsed 174.6min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 16201)\nstep 16250 | epoch 1.29 | loss 2.7209 | ce 2.6876 | lr 9.02e-04 | tok/s 513,680 | elapsed 175.1min\nstep 16300 | epoch 1.29 | loss 2.7134 | ce 2.7119 | lr 8.89e-04 | tok/s 513,213 | elapsed 175.6min\nstep 16350 | epoch 1.29 | loss 2.7222 | ce 2.6854 | lr 8.75e-04 | tok/s 512,564 | elapsed 176.1min\nstep 16400 | epoch 1.30 | loss 2.7207 | ce 2.6801 | lr 8.61e-04 | tok/s 512,666 | elapsed 176.7min\nstep 16450 | epoch 1.30 | loss 2.7223 | ce 2.7212 | lr 8.46e-04 | tok/s 512,555 | elapsed 177.2min\nstep 16500 | epoch 1.31 | loss 2.7116 | ce 2.6817 | lr 8.30e-04 | tok/s 512,707 | elapsed 177.7min\nstep 16550 | epoch 1.31 | loss 2.7153 | ce 2.7655 | lr 8.14e-04 | tok/s 512,445 | elapsed 178.3min\nstep 16600 | epoch 1.31 | loss 2.7122 | ce 2.7218 | lr 7.98e-04 | tok/s 512,537 | elapsed 178.8min\nstep 16650 | epoch 1.32 | loss 2.7107 | ce 2.7351 | lr 7.80e-04 | tok/s 512,635 | elapsed 179.3min\nstep 16700 | epoch 1.32 | loss 2.7096 | ce 2.7676 | lr 7.63e-04 | tok/s 512,089 | elapsed 179.9min\nstep 16750 | epoch 1.33 | loss 2.7089 | ce 2.7320 | lr 7.45e-04 | tok/s 512,638 | elapsed 180.4min\nstep 16800 | epoch 1.33 | loss 2.7072 | ce 2.7475 | lr 7.26e-04 | tok/s 512,722 | elapsed 180.9min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 16801)\nstep 16850 | epoch 1.33 | loss 2.7071 | ce 2.7147 | lr 7.08e-04 | tok/s 512,364 | elapsed 181.5min\nstep 16900 | epoch 1.34 | loss 2.6960 | ce 2.6424 | lr 6.89e-04 | tok/s 512,906 | elapsed 182.0min\nstep 16950 | epoch 1.34 | loss 2.6958 | ce 2.7162 | lr 6.69e-04 | tok/s 512,554 | elapsed 182.5min\nstep 17000 | epoch 1.35 | loss 2.7011 | ce 2.7590 | lr 6.50e-04 | tok/s 513,349 | elapsed 183.1min\nstep 17000 | val_loss 2.6624 | val_acc 0.4620 | val_ppl 14.33\nstep 17050 | epoch 1.35 | loss 2.7022 | ce 2.6949 | lr 6.30e-04 | tok/s 504,889 | elapsed 183.6min\nstep 17100 | epoch 1.35 | loss 2.6943 | ce 2.6557 | lr 6.10e-04 | tok/s 514,698 | elapsed 184.1min\nstep 17150 | epoch 1.36 | loss 2.6849 | ce 2.6454 | lr 5.90e-04 | tok/s 514,605 | elapsed 184.7min\nstep 17200 | epoch 1.36 | loss 2.6777 | ce 2.6970 | lr 5.70e-04 | tok/s 513,805 | elapsed 185.2min\nstep 17250 | epoch 1.37 | loss 2.6788 | ce 2.6647 | lr 5.50e-04 | tok/s 512,651 | elapsed 185.7min\nstep 17300 | epoch 1.37 | loss 2.6785 | ce 2.6216 | lr 5.29e-04 | tok/s 513,081 | elapsed 186.3min\nstep 17350 | epoch 1.37 | loss 2.6750 | ce 2.6945 | lr 5.09e-04 | tok/s 513,108 | elapsed 186.8min\nstep 17400 | epoch 1.38 | loss 2.6736 | ce 2.7298 | lr 4.89e-04 | tok/s 512,315 | elapsed 187.3min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 17401)\nstep 17450 | epoch 1.38 | loss 2.6697 | ce 2.6768 | lr 4.69e-04 | tok/s 512,865 | elapsed 187.9min\nstep 17500 | epoch 1.39 | loss 2.6748 | ce 2.6293 | lr 4.49e-04 | tok/s 512,578 | elapsed 188.4min\nstep 17550 | epoch 1.39 | loss 2.6665 | ce 2.6493 | lr 4.30e-04 | tok/s 511,862 | elapsed 188.9min\nstep 17600 | epoch 1.39 | loss 2.6621 | ce 2.7009 | lr 4.11e-04 | tok/s 512,216 | elapsed 189.5min\nstep 17650 | epoch 1.40 | loss 2.6547 | ce 2.6845 | lr 3.92e-04 | tok/s 512,483 | elapsed 190.0min\nstep 17700 | epoch 1.40 | loss 2.6569 | ce 2.7060 | lr 3.73e-04 | tok/s 512,807 | elapsed 190.5min\nstep 17750 | epoch 1.41 | loss 2.6578 | ce 2.6618 | lr 3.54e-04 | tok/s 512,279 | elapsed 191.1min\nstep 17800 | epoch 1.41 | loss 2.6527 | ce 2.6804 | lr 3.36e-04 | tok/s 511,948 | elapsed 191.6min\nstep 17850 | epoch 1.41 | loss 2.6580 | ce 2.6613 | lr 3.19e-04 | tok/s 512,451 | elapsed 192.1min\nstep 17900 | epoch 1.42 | loss 2.6472 | ce 2.5631 | lr 3.02e-04 | tok/s 511,944 | elapsed 192.7min\nstep 17950 | epoch 1.42 | loss 2.6446 | ce 2.6326 | lr 2.85e-04 | tok/s 512,091 | elapsed 193.2min\nstep 18000 | epoch 1.43 | loss 2.6422 | ce 2.6589 | lr 2.69e-04 | tok/s 512,671 | elapsed 193.7min\nstep 18000 | val_loss 2.6095 | val_acc 0.4698 | val_ppl 13.59\n[Export] step 18001: 194 tensors (+1 aliased) -> ./checkpoints/model_step18001.safetensors (114.1 MB, 0.05s)\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 18001)\nstep 18050 | epoch 1.43 | loss 2.6409 | ce 2.6603 | lr 2.54e-04 | tok/s 480,005 | elapsed 194.3min\nstep 18100 | epoch 1.43 | loss 2.6389 | ce 2.6796 | lr 2.39e-04 | tok/s 512,580 | elapsed 194.8min\nstep 18150 | epoch 1.44 | loss 2.6395 | ce 2.6795 | lr 2.24e-04 | tok/s 512,586 | elapsed 195.4min\nstep 18200 | epoch 1.44 | loss 2.6381 | ce 2.6149 | lr 2.11e-04 | tok/s 513,235 | elapsed 195.9min\nstep 18250 | epoch 1.45 | loss 2.6254 | ce 2.6491 | lr 1.98e-04 | tok/s 512,619 | elapsed 196.4min\nstep 18300 | epoch 1.45 | loss 2.6296 | ce 2.5857 | lr 1.86e-04 | tok/s 512,285 | elapsed 197.0min\nstep 18350 | epoch 1.45 | loss 2.6312 | ce 2.6688 | lr 1.74e-04 | tok/s 512,393 | elapsed 197.5min\nstep 18400 | epoch 1.46 | loss 2.6256 | ce 2.6221 | lr 1.63e-04 | tok/s 512,969 | elapsed 198.0min\nstep 18450 | epoch 1.46 | loss 2.6208 | ce 2.6073 | lr 1.54e-04 | tok/s 513,204 | elapsed 198.6min\nstep 18500 | epoch 1.47 | loss 2.6218 | ce 2.6544 | lr 1.44e-04 | tok/s 512,415 | elapsed 199.1min\nstep 18550 | epoch 1.47 | loss 2.6166 | ce 2.6012 | lr 1.36e-04 | tok/s 512,436 | elapsed 199.6min\nstep 18600 | epoch 1.47 | loss 2.6213 | ce 2.6207 | lr 1.29e-04 | tok/s 512,987 | elapsed 200.2min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 18601)\nstep 18650 | epoch 1.48 | loss 2.6192 | ce 2.5554 | lr 1.22e-04 | tok/s 512,288 | elapsed 200.7min\nstep 18700 | epoch 1.48 | loss 2.6205 | ce 2.6088 | lr 1.16e-04 | tok/s 511,698 | elapsed 201.2min\nstep 18750 | epoch 1.48 | loss 2.6232 | ce 2.6280 | lr 1.11e-04 | tok/s 512,009 | elapsed 201.8min\nstep 18800 | epoch 1.49 | loss 2.6128 | ce 2.6425 | lr 1.07e-04 | tok/s 512,967 | elapsed 202.3min\nstep 18850 | epoch 1.49 | loss 2.6144 | ce 2.6497 | lr 1.04e-04 | tok/s 512,090 | elapsed 202.8min\nstep 18900 | epoch 1.50 | loss 2.6191 | ce 2.5731 | lr 1.02e-04 | tok/s 512,580 | elapsed 203.4min\nstep 18950 | epoch 1.50 | loss 2.6170 | ce 2.6598 | lr 1.00e-04 | tok/s 512,755 | elapsed 203.9min\n[Save] state_dict(): 0.00s | 195 tensors | 0.13 GB\n[Save] optimizer.state_dict(): 0.00s | 300 tensors | 0.14 GB\n[Save] _cpu walk: 0.12s\n[Save] torch.save -> /tmp: 0.18s | file 275 MB\n[Save] copyfile -> ./checkpoints/arya_latest.pt: 0.08s\n[Export] step 19000: 194 tensors (+1 aliased) -> ./checkpoints/model_step19000.safetensors (114.1 MB, 0.03s)\n[Training] Complete\n",
"mimetype": "text/plain"
}
]
},
{
"id": "Hstk",
"code_hash": "491df3d6836c4a6956d9cdf3e6e8d89f",
"outputs": [],
"console": []
},
{
"id": "nWHF",
"code_hash": "d108748a7f1e39fb39644a9755a27b91",
"outputs": [],
"console": []
},
{
"id": "iLit",
"code_hash": "a1daefd9105ef45d64a60fc09e306b69",
"outputs": [],
"console": []
},
{
"id": "ZHCJ",
"code_hash": "9aa4360427f7a6b988b89521f957855c",
"outputs": [],
"console": []
},
{
"id": "ROlb",
"code_hash": null,
"outputs": [],
"console": []
},
{
"id": "qnkX",
"code_hash": "26bf08358a5fd5faeadfb725840c6654",
"outputs": [],
"console": []
},
{
"id": "TqIu",
"code_hash": "8cf0ab5f2f0947afe55db20860f35990",
"outputs": [
{
"type": "data",
"data": {
"text/html": "<pre class='text-xs'>0</pre>"
}
}
],
"console": [
{
"type": "stream",
"name": "stderr",
"text": "/usr/local/lib/python3.13/site-packages/torch/profiler/profiler.py:224: UserWarning: Warning: Profiler clears events at the end of each cycle.Only events from the current cycle will be reported.To keep events across cycles, set acc_events=True.\n _warn_once(\n",
"mimetype": "text/plain"
},
{
"type": "stream",
"name": "stdout",
"text": "[CLI] Overriding config.py: {'use_liger': False}\n[Setup] device=cuda (cuda)\n\n=== Speed diagnosis: can Splash Attention even run here? ===\n device: cuda\n config pattern: 'dense'\n [BLOCKED] device gate: Splash Attention is TPU-only (kernel.py's call_jax bridge) -- not on TPU, so this alone rules it out here.\n [OK] architecture gate: kernel.fused_attention (Splash's only entry point) is reached by DenseMHA, used in patterns ('dense', 'pyramid_swa', 'mod') -- all three patterns this codebase currently implements. Current pattern reaches DenseMHA, so this gate passes.\n Architecture gate passes (all supported patterns reach DenseMHA), but not on TPU -- Splash Attention structurally cannot run on this device regardless of pattern. xFormers (T4-class) or flash_attn (A100+) handle GPU instead; check backend stats below to see which one actually engaged.\n[Tokenizer] Loaded, vocab_size=12288\n[Dataset] Found 6 files in ./training_data (curriculum OFF)\n[Dataset] Cache hit -> ./training_data/.token_cache/tokens_83a770868891c636.npy (4,179,431,312 tokens)\n[Engram] layers=[1, 8] max_ngram=3 compressed_vocab=9807 (raw 12288)\n[Model] 28.50M params, pattern=dense\n[Engram] 2,532,736 engram params\n[Precision] autocast dtype=torch.bfloat16 (device_kind=cuda)\n[Compile] Note: hardest_file_check uses batch=1 (different shape from learnability_check/gpu_profile_check's batch=8) -- expect one additional recompilation when that section runs, not a bug.\n[Compile] WARNING: reduce-overhead requested via alpha_config.ALPHA_COMPILE_MODE -- this model's tied embedding is a known CUDA-graphs hazard (see train.py). Proceeding anyway since you asked explicitly.\n[Compile] mode=reduce-overhead\n\n=== Learnability check: SKIPPED (--skip-learnability) ===\n Model is at random initialization -- hardest_file_check is also skipped (its loss ranking would be meaningless on untrained weights). Only gpu_profile_check's speed/roofline numbers run, since those reflect architecture shape only, not learned behavior.\n\n=== Hardest-file check: SKIPPED (--skip-learnability) ===\n Model is at random initialization -- the loss ranking would be meaningless.\n\n=== GPU profiling (CUDA) ===\n device: NVIDIA RTX PRO 6000 Blackwell Server Edition\n SMs: 188 | total VRAM: 102.0 GB\n\n --- torch.profiler: 3 warmup + 8 recorded steps ---\n\n Top operators by self_cuda_time_total (8-step window):\n------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ \n Name Self CPU % Self CPU CPU total % CPU total CPU time avg Self CUDA Self CUDA % CUDA total CUDA time avg CPU Mem Self CPU Mem CUDA Mem Self CUDA Mem # of Calls Total MFLOPs \n------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ \n## Call CompiledFxGraph felvw6fyfphsgtviq7567hgcflv5... 0.00% 0.000us 0.00% 0.000us 0.000us 83.263ms 59.63% 83.263ms 10.408ms 0 B 0 B 0 B 0 B 8 -- \n CompiledFunctionBackward 1.92% 3.378ms 11.51% 20.255ms 1.266ms 81.821ms 58.60% 81.838ms 5.115ms 0 B 0 B 0 B 0 B 16 -- \n## Call CompiledFxGraph f2papzyhk3eoy65khzkyflsjbzrx... 0.00% 0.000us 0.00% 0.000us 0.000us 51.195ms 36.67% 51.195ms 6.399ms 0 B 0 B 0 B 0 B 8 -- \n CompiledFunction 1.94% 3.423ms 7.09% 12.477ms 779.838us 50.828ms 36.40% 50.919ms 3.182ms 0 B 0 B 0 B 0 B 16 -- \ntriton_tem_fused__unsafe_view_flex_attention_backwar... 0.00% 0.000us 0.00% 0.000us 0.000us 21.908ms 15.69% 21.908ms 158.753us 0 B 0 B 0 B 0 B 138 -- \nvoid cutlass::Kernel2<cutlass_80_tensorop_bf16_s1681... 0.00% 0.000us 0.00% 0.000us 0.000us 15.735ms 11.27% 15.735ms 26.715us 0 B 0 B 0 B 0 B 589 -- \nvoid cutlass::Kernel2<cutlass_80_tensorop_bf16_s1681... 0.00% 0.000us 0.00% 0.000us 0.000us 15.098ms 10.81% 15.098ms 34.948us 0 B 0 B 0 B 0 B 432 -- \nvoid cutlass::Kernel2<cutlass_80_tensorop_bf16_s1681... 0.00% 0.000us 0.00% 0.000us 0.000us 14.937ms 10.70% 14.937ms 34.656us 0 B 0 B 0 B 0 B 431 -- \ntriton_tem_fused__unsafe_view_flex_attention_split_w... 0.00% 0.000us 0.00% 0.000us 0.000us 5.449ms 3.90% 5.449ms 42.566us 0 B 0 B 0 B 0 B 128 -- \nvoid cutlass::Kernel2<cutlass_80_tensorop_bf16_s1681... 0.00% 0.000us 0.00% 0.000us 0.000us 4.532ms 3.25% 4.532ms 26.047us 0 B 0 B 0 B 0 B 174 -- \ntriton_poi_fused__unsafe_view_cat_mul_silu_silu_back... 0.00% 0.000us 0.00% 0.000us 0.000us 4.388ms 3.14% 4.388ms 31.798us 0 B 0 B 0 B 0 B 138 -- \nvoid cutlass::Kernel2<cutlass_80_tensorop_bf16_s1681... 0.00% 0.000us 0.00% 0.000us 0.000us 3.829ms 2.74% 3.829ms 29.911us 0 B 0 B 0 B 0 B 128 -- \ntriton_red_fused__to_copy_exp_logsumexp_ne_neg_scatt... 0.00% 0.000us 0.00% 0.000us 0.000us 3.274ms 2.34% 3.274ms 204.596us 0 B 0 B 0 B 0 B 16 -- \ntriton_per_fused__fused_rms_norm_backward__to_copy_a... 0.00% 0.000us 0.00% 0.000us 0.000us 2.714ms 1.94% 2.714ms 24.231us 0 B 0 B 0 B 0 B 112 -- \ntriton_per_fused__fused_rms_norm_backward__to_copy_a... 0.00% 0.000us 0.00% 0.000us 0.000us 2.660ms 1.90% 2.660ms 21.980us 0 B 0 B 0 B 0 B 121 -- \n------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ \nSelf CPU time total: 176.020ms\nSelf CUDA time total: 139.630ms\n\n Chrome trace exported to ./profile_trace.json (open at chrome://tracing or https://ui.perfetto.dev)\n\n --- torch.utils.benchmark: forward-only vs forward+backward ---\n forward-only: 4.33 ms/iter (median 4.33 ms, 20 iters)\n forward+backward: 16.84 ms/iter (median 16.84 ms, 20 iters)\n tokens/step: 8,192 -> 486,582 tok/s (forward+backward, this batch/seq shape)\n\n --- torch.cuda.Event: raw GPU-side step time ---\n mean GPU-side step time: 16.79 ms (20 events)\n vs torch.utils.benchmark forward+backward: 16.84 ms (gap = Python/CPU dispatch overhead)\n\n --- Memory ---\n peak reserved: 2.79 GB / 95.0 GB (2.9%)\n peak allocated: 2.51 GB\n\n --- Model FLOPs Utilization (MFU) ---\n total params: 28.50M | non-embedding: 23.78M\n [SKIPPED] 'NVIDIA RTX PRO 6000 Blackwell Server Edition' not in the known-GPU table (T4/L4/A100/H100) -- add its dense FP16 Tensor Core TFLOPS to GPU_PEAK_FP16_TFLOPS at the top of this file to enable this calculation.\n\n[Roofline] Unavailable -- gpu_profile_check didn't return usable data (likely non-CUDA device or an unrecognized GPU -- see the MFU section above).\n\n=== Summary ===\nLearnability: SKIPPED (--skip-learnability)\n\n=== kernel.py backend usage during the learnability check ===\n N/A -- learnability check was skipped (--skip-learnability), so no forward passes ran during that phase. See gpu_profile_check's own backend stats above (if run) for kernel dispatch info instead.\n",
"mimetype": "text/plain"
}
]
},
{
"id": "Vxnm",
"code_hash": "472b180a72ef3d7ee4b56780749ad24e",
"outputs": [],
"console": []
},
{
"id": "DnEU",
"code_hash": "d6b0911a49796272ca86fe8f1cfba105",
"outputs": [],
"console": []
},
{
"id": "ulZA",
"code_hash": "42312cfb889e4c7d1d95b40183870910",
"outputs": [],
"console": []
},
{
"id": "ecfG",
"code_hash": "8c8d6ed6bb295c0981866d2d9b707d6f",
"outputs": [
{
"type": "data",
"data": {
"text/html": "<pre class='text-xs'>0</pre>"
}
}
],
"console": [
{
"type": "stream",
"name": "stderr",
"text": "WARNING: The directory '/home/marimo/.cache/pip' or its parent directory is not owned or is not writable by the current user. The cache has been disabled. Check the permissions and owner of that directory. If executing pip with sudo, you should use sudo's -H flag.\n",
"mimetype": "text/plain"
},
{
"type": "stream",
"name": "stdout",
"text": "Requirement already satisfied: liger-kernel in /tmp/uv-venv/lib/python3.13/site-packages (0.8.4)\nRequirement already satisfied: rustbpe in /tmp/uv-venv/lib/python3.13/site-packages (0.1.0)\nRequirement already satisfied: torchao in /tmp/uv-venv/lib/python3.13/site-packages (0.18.0)\nRequirement already satisfied: triton in /tmp/uv-venv/lib/python3.13/site-packages (3.8.0)\nCollecting accelerate\n Downloading accelerate-1.15.0-py3-none-any.whl.metadata (19 kB)\nRequirement already satisfied: safetensors in /tmp/uv-venv/lib/python3.13/site-packages (0.8.0)\nCollecting helion\n Downloading helion-1.5.1-py3-none-any.whl.metadata (23 kB)\nRequirement already satisfied: torch>=2.1.2 in /usr/local/lib/python3.13/site-packages (from liger-kernel) (2.11.0)\nRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.13/site-packages (from accelerate) (2.4.6)\nRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.13/site-packages (from accelerate) (26.2)\nRequirement already satisfied: psutil in /usr/local/lib/python3.13/site-packages (from accelerate) (7.2.2)\nRequirement already satisfied: pyyaml in /usr/local/lib/python3.13/site-packages (from accelerate) (6.0.3)\nRequirement already satisfied: huggingface_hub>=0.21.0 in /tmp/uv-venv/lib/python3.13/site-packages (from accelerate) (1.33.0)\nCollecting filecheck (from helion)\n Downloading filecheck-1.0.6-py3-none-any.whl.metadata (5.2 kB)\nRequirement already satisfied: rich in /tmp/uv-venv/lib/python3.13/site-packages (from helion) (15.0.0)\nRequirement already satisfied: scikit-learn>=1.3.0 in /usr/local/lib/python3.13/site-packages (from helion) (1.9.0)\nRequirement already satisfied: typing-extensions>=4.0.0 in /usr/local/lib/python3.13/site-packages (from helion) (4.16.0)\nRequirement already satisfied: click<9.0.0,>=8.4.2 in /usr/local/lib/python3.13/site-packages (from huggingface_hub>=0.21.0->accelerate) (8.4.2)\nRequirement already satisfied: filelock>=3.10.0 in /tmp/uv-venv/lib/python3.13/site-packages (from huggingface_hub>=0.21.0->accelerate) (4.2.0)\nRequirement already satisfied: fsspec>=2023.5.0 in /tmp/uv-venv/lib/python3.13/site-packages (from huggingface_hub>=0.21.0->accelerate) (2026.9.0)\nRequirement already satisfied: hf-xet<2.0.0,>=1.6.0 in /tmp/uv-venv/lib/python3.13/site-packages (from huggingface_hub>=0.21.0->accelerate) (1.7.0)\nRequirement already satisfied: httpx<1,>=0.23.0 in /tmp/uv-venv/lib/python3.13/site-packages (from huggingface_hub>=0.21.0->accelerate) (0.28.1)\nRequirement already satisfied: tqdm>=4.42.1 in /tmp/uv-venv/lib/python3.13/site-packages (from huggingface_hub>=0.21.0->accelerate) (4.70.1)\nRequirement already satisfied: anyio in /tmp/uv-venv/lib/python3.13/site-packages (from httpx<1,>=0.23.0->huggingface_hub>=0.21.0->accelerate) (4.15.1)\nRequirement already satisfied: certifi in /tmp/uv-venv/lib/python3.13/site-packages (from httpx<1,>=0.23.0->huggingface_hub>=0.21.0->accelerate) (2026.7.22)\nRequirement already satisfied: httpcore==1.* in /tmp/uv-venv/lib/python3.13/site-packages (from httpx<1,>=0.23.0->huggingface_hub>=0.21.0->accelerate) (1.0.9)\nRequirement already satisfied: idna in /tmp/uv-venv/lib/python3.13/site-packages (from httpx<1,>=0.23.0->huggingface_hub>=0.21.0->accelerate) (3.20)\nRequirement already satisfied: h11>=0.16 in /tmp/uv-venv/lib/python3.13/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->huggingface_hub>=0.21.0->accelerate) (0.16.0)\nRequirement already satisfied: scipy>=1.10.0 in /usr/local/lib/python3.13/site-packages (from scikit-learn>=1.3.0->helion) (1.18.0)\nRequirement already satisfied: joblib>=1.4.0 in /usr/local/lib/python3.13/site-packages (from scikit-learn>=1.3.0->helion) (1.5.3)\nRequirement already satisfied: narwhals>=2.0.1 in /usr/local/lib/python3.13/site-packages (from scikit-learn>=1.3.0->helion) (2.24.0)\nRequirement already satisfied: threadpoolctl>=3.5.0 in /usr/local/lib/python3.13/site-packages (from scikit-learn>=1.3.0->helion) (3.6.0)\nCollecting setuptools<82 (from torch>=2.1.2->liger-kernel)\n Downloading setuptools-81.0.0-py3-none-any.whl.metadata (6.6 kB)\nRequirement already satisfied: sympy>=1.13.3 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (1.14.0)\nRequirement already satisfied: networkx>=2.5.1 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (3.6.1)\nRequirement already satisfied: jinja2 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (3.1.6)\nRequirement already satisfied: cuda-toolkit==13.0.2 in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (13.0.2)\nRequirement already satisfied: cuda-bindings<14,>=13.0.3 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (13.3.1)\nRequirement already satisfied: nvidia-cudnn-cu13==9.19.0.56 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (9.19.0.56)\nRequirement already satisfied: nvidia-cusparselt-cu13==0.8.0 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (0.8.0)\nRequirement already satisfied: nvidia-nccl-cu13==2.28.9 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (2.28.9)\nRequirement already satisfied: nvidia-nvshmem-cu13==3.4.5 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (3.4.5)\nCollecting triton\n Downloading triton-3.6.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl.metadata (1.7 kB)\nRequirement already satisfied: nvidia-cublas==13.1.0.3.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (13.1.0.3)\nRequirement already satisfied: nvidia-cuda-runtime==13.0.96.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (13.0.96)\nRequirement already satisfied: nvidia-cufft==12.0.0.61.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (12.0.0.61)\nRequirement already satisfied: nvidia-cufile==1.15.1.6.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (1.15.1.6)\nRequirement already satisfied: nvidia-cuda-cupti==13.0.85.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (13.0.85)\nRequirement already satisfied: nvidia-curand==10.4.0.35.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (10.4.0.35)\nRequirement already satisfied: nvidia-cusolver==12.0.4.66.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (12.0.4.66)\nRequirement already satisfied: nvidia-cusparse==12.6.3.3.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (12.6.3.3)\nRequirement already satisfied: nvidia-nvjitlink==13.0.88.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (13.0.88)\nRequirement already satisfied: nvidia-cuda-nvrtc==13.0.88.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (13.0.88)\nRequirement already satisfied: nvidia-nvtx==13.0.85.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (13.0.85)\nRequirement already satisfied: cuda-pathfinder>=1.4.2 in /usr/local/lib/python3.13/site-packages (from cuda-bindings<14,>=13.0.3->torch>=2.1.2->liger-kernel) (1.5.6)\nRequirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.13/site-packages (from sympy>=1.13.3->torch>=2.1.2->liger-kernel) (1.3.0)\nRequirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.13/site-packages (from jinja2->torch>=2.1.2->liger-kernel) (3.0.3)\nRequirement already satisfied: markdown-it-py>=2.2.0 in /tmp/uv-venv/lib/python3.13/site-packages (from rich->helion) (4.2.0)\nRequirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.13/site-packages (from rich->helion) (2.20.0)\nRequirement already satisfied: mdurl~=0.1 in /tmp/uv-venv/lib/python3.13/site-packages (from markdown-it-py>=2.2.0->rich->helion) (0.1.2)\nDownloading accelerate-1.15.0-py3-none-any.whl (394 kB)\nDownloading helion-1.5.1-py3-none-any.whl (4.7 MB)\n \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 4.7/4.7 MB 29.2 MB/s 0:00:00\nDownloading triton-3.6.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (188.3 MB)\n \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 188.3/188.3 MB 42.2 MB/s 0:00:04\nDownloading setuptools-81.0.0-py3-none-any.whl (1.1 MB)\n \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 1.1/1.1 MB 220.5 MB/s 0:00:00\nDownloading filecheck-1.0.6-py3-none-any.whl (24 kB)\nInstalling collected packages: triton, setuptools, filecheck, helion, accelerate\n Attempting uninstall: triton\n Found existing installation: triton 3.8.0\n Uninstalling triton-3.8.0:\n Successfully uninstalled triton-3.8.0\n Attempting uninstall: setuptools\n Found existing installation: setuptools 84.0.0\n Uninstalling setuptools-84.0.0:\n Successfully uninstalled setuptools-84.0.0\n\nSuccessfully installed accelerate-1.15.0 filecheck-1.0.6 helion-1.5.1 setuptools-81.0.0 triton-3.6.0\n",
"mimetype": "text/plain"
},
{
"type": "stream",
"name": "stderr",
"text": "WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\n\n[notice] A new release of pip is available: 25.3 -> 26.2.1\n[notice] To update, run: pip install --upgrade pip\n",
"mimetype": "text/plain"
}
]
},
{
"id": "Pvdt",
"code_hash": "b23d6205e0dff43a7a9b1cab8bd8e97a",
"outputs": [],
"console": []
},
{
"id": "ZBYS",
"code_hash": "f103139918c160f7b8011d902c51c124",
"outputs": [],
"console": []
},
{
"id": "aLJB",
"code_hash": "4e534c092104c4c90c1278ab93710faf",
"outputs": [
{
"type": "data",
"data": {
"text/html": "<pre class='text-xs'>0</pre>"
}
}
],
"console": [
{
"type": "stream",
"name": "stderr",
"text": "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n",
"mimetype": "text/plain"
},
{
"type": "stream",
"name": "stdout",
"text": "\u26a0 ftfy not installed \u2014 mojibake repair skipped. Run `pip install ftfy` for full encoding cleanup.\n\u26a0 No HF_TOKEN found. Proceeding unauthenticated \u2014 downloads may be slow/throttled. Add HF_TOKEN to Colab Secrets (or set the env var) for faster, more reliable downloads. Note: bigcode/the-stack-v2 REQUIRES authentication + data-access agreement at https://huggingface.co/bigcode/the-stack-v2\nDownloading LittleCurriculum, target 5000.0 MB -> ./training_data/littlecurriculum_5000mb.txt\n\u2713 ./training_data/littlecurriculum_5000mb.txt | 5000.0/5000.0 MB (100.0%) | 3,861,797 lines | 175,867 filtered | 2,768,361 dupes dropped\n \u2713 verified: 200,000 lines sampled, 0 artifacts\nDownloading SmolLM-Corpus / cosmopedia-v2, target 5000.0 MB -> ./training_data/smollm_cosmopedia_v2_5000mb.txt\n\u2713 ./training_data/smollm_cosmopedia_v2_5000mb.txt | 5000.0/5000.0 MB (100.0%) | 1,409,211 lines | 16,022 filtered | 7,891 dupes dropped\n \u2713 verified: 200,000 lines sampled, 0 artifacts\nTrying TinyStories source: roneneldan/TinyStories\nDownloading TinyStories (roneneldan/TinyStories), target 1000.0 MB -> ./training_data/tinystories_1000mb.txt\n\u2713 ./training_data/tinystories_1000mb.txt | 1000.0/1000.0 MB (100.0%) | 1,175,331 lines | 109 filtered | 114,855 dupes dropped\n \u2713 verified: 200,000 lines sampled, 0 artifacts\n",
"mimetype": "text/plain"
}
]
},
{
"id": "nHfw",
"code_hash": "950061ed21b5034668672f8f1537fc20",
"outputs": [
{
"type": "data",
"data": {
"text/markdown": "<span class=\"markdown prose dark:prose-invert contents\"><h2 id=\"inference-script\">Inference Script</h2>\n<span class=\"paragraph\">This script loads the trained model from <code>arya_latest.pt</code> and demonstrates how to generate text using a given prompt.</span></span>"
}
}
],
"console": []
},
{
"id": "xXTn",
"code_hash": "ccc1222007395c4c23a2ca9294992615",
"outputs": [
{
"type": "data",
"data": {
"text/html": "<pre class='text-xs'>0</pre>"
}
}
],
"console": [
{
"type": "stream",
"name": "stderr",
"text": "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n",
"mimetype": "text/plain"
},
{
"type": "stream",
"name": "stdout",
"text": "\u26a0 ftfy not installed \u2014 mojibake repair (e.g. '\u00ce\u00bcC' -> '\u03bcC', '\u00c3\u2014' -> '\u00d7') will be skipped. Run `pip install ftfy` for full text-encoding cleanup \u2014 this matters especially for NCERT, which has verified mojibake in the source data.\n\u26a0 No HF_TOKEN found (Colab Secrets key icon -> add secret named HF_TOKEN -> your HF write-access token -> toggle notebook access). Proceeding unauthenticated \u2014 downloads may be slow or appear to hang due to rate limiting, not actually broken.\nDownloading FineWeb-Edu (sample-10BT), target 4000.0 MB -> ./training_data/fineweb_edu_4000mb.txt\n\u2713 ./training_data/fineweb_edu_4000mb.txt | 4000.0 MB / 4000.0 MB (100.0%) | 899,945 lines kept | 16,690 filtered | 2,736 duplicates dropped\n \u2713 verified: 200,000 lines sampled, 0 markup/encoding artifacts\nDownloading Cosmopedia (auto_math_text), target 2000.0 MB -> ./training_data/cosmopedia_2000mb.txt\n\u2713 ./training_data/cosmopedia_2000mb.txt | 2000.0 MB / 2000.0 MB (100.0%) | 808,583 lines kept | 176,271 filtered | 0 duplicates dropped\n \u2713 verified: 200,000 lines sampled, 0 markup/encoding artifacts\n",
"mimetype": "text/plain"
}
]
},
{
"id": "AjVT",
"code_hash": "9421950de89c80b932668244f9cb90bf",
"outputs": [
{
"type": "data",
"data": {
"text/html": "<pre class='text-xs'>0</pre>"
}
}
],
"console": [
{
"type": "stream",
"name": "stderr",
"text": "[Inference] Loaded checkpoint (step 19000, 23.78M params) onto cuda\n",
"mimetype": "text/plain"
},
{
"type": "stream",
"name": "stdout",
"text": "Internet is a network of virtual computers that can be accessed and shared by 3rd parties. These networks are typically large, but some common examples include the Internet of Things (IoT), mobile phones, IoFLs (International Intelligent Directory) or Microsoft Applications Service U.S.A. A company called SATIN has been creating a network for thousands of people all over the world using its website to connect them with other businesses operating inside it \u2013 and we have just created one very well-connected city! The City on Mars is an incredible place where you could visit from anywhere in space anytime. One great thing about exploring this fantastic planet is taking out your smartphone at night! This means watching video clips as astronauts get off their telescopes searching for life before getting too close to anything else they see making sense because NASA\u2019s satellite isn\u2019t ready yet; also if there were a radio ferry system going around with light pollution coming into our eyes so\n",
"mimetype": "text/plain"
}
]
},
{
"id": "pHFh",
"code_hash": "e181efcf155eef21719f37511de1f32f",
"outputs": [
{
"type": "data",
"data": {
"text/markdown": "<span class=\"markdown prose dark:prose-invert contents\"></span>"
}
}
],
"console": []
},
{
"id": "NCOB",
"code_hash": "9573a4817336aebce40126886c37e8aa",
"outputs": [
{
"type": "data",
"data": {
"text/markdown": "<span class=\"markdown prose dark:prose-invert contents\"><span class=\"paragraph\">Now, let's define a function to generate text and then test it with an example prompt.</span></span>"
}
}
],
"console": []
},
{
"id": "aqbW",
"code_hash": "9541eaa3a371cb54992795cf9ea79a97",
"outputs": [],
"console": []
},
{
"id": "TRpd",
"code_hash": "e2e709938b0cb769e6643c21951a1aa7",
"outputs": [],
"console": []
},
{
"id": "TXez",
"code_hash": "9efe733301c0f8f6d66eccbd830a0937",
"outputs": [],
"console": []
},
{
"id": "dNNg",
"code_hash": "08809d03fe76b2df1122d25d14fe74d5",
"outputs": [],
"console": []
},
{
"id": "yCnT",
"code_hash": "13122d375dfe64b0638eed09336e688c",
"outputs": [],
"console": []
},
{
"id": "wlCL",
"code_hash": "c8fc96583ecaa55247aa6caf3e855d22",
"outputs": [],
"console": []
},
{
"id": "kqZH",
"code_hash": "d6e07408db81621dabaca73882dcacb1",
"outputs": [],
"console": []
},
{
"id": "wAgl",
"code_hash": "9ea3e14a9bb177fa422b4038a8daaca4",
"outputs": [
{
"type": "data",
"data": {
"text/html": "<pre class='text-xs'>0</pre>"
}
}
],
"console": [
{
"type": "stream",
"name": "stdout",
"text": "[token_train] 6 files, 17.89 GB corpus, vocab_size=12288\n[token_train] Trained. mergeable ranks = 12287\n[token_train] tiktoken.Encoding built: n_vocab=12288, <|bos|> id=12287\n[token_train] Saved tokenizer/tokenizer.pkl\n",
"mimetype": "text/plain"
}
]
},
{
"id": "rEll",
"code_hash": "c72b1f6f545da1a69cbaa110db3ed3da",
"outputs": [],
"console": []
},
{
"id": "dGlV",
"code_hash": "b5cb5e47bc909cfe489c2c8e8038ad4f",
"outputs": [],
"console": []
},
{
"id": "SdmI",
"code_hash": "d10c338ecd83ffd0ca2127d594ec829b",
"outputs": [],
"console": []
},
{
"id": "lgWD",
"code_hash": "dc7ae64fa54ee4cc7080f818cda4726c",
"outputs": [
{
"type": "data",
"data": {
"text/plain": ""
}
}
],
"console": []
},
{
"id": "yOPj",
"code_hash": "2037cfc3a9f1a07e5812e64c25fcd5d7",
"outputs": [
{
"type": "data",
"data": {
"text/markdown": "<span class=\"markdown prose dark:prose-invert contents\"><span class=\"paragraph\">\u2705 Pushed to <a href=\"https://huggingface.co/RuVI-AI/pragya-slm\" rel=\"noopener noreferrer\" target=\"_blank\">https://huggingface.co/RuVI-AI/pragya-slm</a> \u2014 28 files (1 checkpoint)</span></span>"
}
}
],
"console": [
{
"type": "stream",
"name": "stdout",
"text": "[auth] ArushBuilds (type=user)\n[repo] ready: https://huggingface.co/RuVI-AI/pragya-e2\n[preflight] 27 local file(s) match ALLOW:\n __marimo__/session/notebook.py.json\n __pycache__/tokenizer.cpython-313.pyc\n checkpoints/arya_latest.pt\n checkpoints/model_step12001.safetensors\n checkpoints/model_step15001.safetensors\n checkpoints/model_step18001.safetensors\n checkpoints/model_step19000.safetensors\n checkpoints/model_step3001.safetensors\n checkpoints/model_step6001.safetensors\n checkpoints/model_step9001.safetensors\n lock.txt\n profile_trace.json\n tokenizer.model\n tokenizer.py\n tokenizer.vocab\n tokenizer/tokenizer.pkl\n tokenizer/tokenizer_config.json\n tokenizer_1.model\n tokenizer_1.py\n tokenizer_1.vocab\n training_data/.token_cache/tokens_83a770868891c636.ranges.json\n training_data/cosmopedia_1000mb.txt\n training_data/cosmopedia_2000mb.txt\n training_data/fineweb_edu_4000mb.txt\n training_data/littlecurriculum_5000mb.txt\n training_data/smollm_cosmopedia_v2_5000mb.txt\n training_data/tinystories_1000mb.txt\n[upload] starting (whitelist: ['*.safetensors', '*.json', '*.model', '*.txt', '*.md', 'tokenizer*', 'tokenizer_files/**', '*.pkl', '*.pt', 'checkpoints/**'])\n",
"mimetype": "text/plain"
}
]
},
{
"id": "fwwy",
"code_hash": "35717b83aa01ab740b9698a3184aeb99",
"outputs": [],
"console": []
},
{
"id": "LJZf",
"code_hash": "8ba9c69f5c18b12e4f905eb4e412f6e8",
"outputs": [],
"console": []
}
]
} |