File size: 60,283 Bytes
4e74b6a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 | {
"version": "1",
"metadata": {
"marimo_version": "0.25.1",
"script_metadata_hash": "ff000607a504b65002f0c642ea5a638f"
},
"cells": [
{
"id": "Hbol",
"code_hash": "1d0db38904205bec4d6f6f6a1f6cec3e",
"outputs": [
{
"type": "data",
"data": {
"text/html": "<div style='display: flex;flex: 1;flex-direction: column;justify-content: flex-start;align-items: normal;flex-wrap: nowrap;gap: 0.5rem'><marimo-ui-element object-id='Hbol-0' random-id='a53f583d-7b3d-0ba6-be9e-1b9cc0292fa3'><marimo-anywidget data-initial-value='{"model_id":"98e1714f8f124240bab453a3b3fba95e"}' data-label='null' data-model-id='"98e1714f8f124240bab453a3b3fba95e"'></marimo-anywidget></marimo-ui-element></div>"
}
}
],
"console": []
},
{
"id": "MJUe",
"code_hash": "8c3add7f504aee86f92a5fd2a080e930",
"outputs": [
{
"type": "error",
"ename": "multiple-defs",
"evalue": "The variable 'os' was defined by another cell",
"traceback": []
}
],
"console": []
},
{
"id": "vblA",
"code_hash": "905d09ca55d5b894841a2106100436a4",
"outputs": [
{
"type": "error",
"ename": "interruption",
"evalue": "This cell was interrupted and needs to be re-run",
"traceback": []
}
],
"console": [
{
"type": "stream",
"name": "stderr",
"text": "<span class=\"codehilite\"><div class=\"highlight\"><pre><span></span><span class=\"gt\">Traceback (most recent call last):</span>\n File <span class=\"nb\">"/tmp/marimo_464/__marimo__cell_vblA_.py"</span>, line <span class=\"m\">3</span>, in <span class=\"n\"><module></span>\n<span class=\"w\"> </span><span class=\"n\">result</span> <span class=\"o\">=</span> <span class=\"n\">subprocess</span><span class=\"o\">.</span><span class=\"n\">run</span><span class=\"p\">(</span>\n<span class=\"w\"> </span> <span class=\"p\">[</span><span class=\"s1\">'python'</span><span class=\"p\">,</span> <span class=\"s1\">'wikitext.py'</span><span class=\"p\">,</span> <span class=\"s1\">'--sources'</span><span class=\"p\">,</span> <span class=\"s1\">'cosmopedia'</span><span class=\"p\">,</span> <span class=\"s1\">'--cosmopedia-mb'</span><span class=\"p\">,</span> <span class=\"s1\">'1000'</span><span class=\"p\">],</span>\n<span class=\"w\"> </span> <span class=\"n\">capture_output</span><span class=\"o\">=</span><span class=\"kc\">True</span><span class=\"p\">,</span> <span class=\"n\">text</span><span class=\"o\">=</span><span class=\"kc\">True</span>\n<span class=\"w\"> </span><span class=\"p\">)</span>\n File <span class=\"nb\">"/usr/local/lib/python3.13/subprocess.py"</span>, line <span class=\"m\">556</span>, in <span class=\"n\">run</span>\n<span class=\"w\"> </span><span class=\"n\">stdout</span><span class=\"p\">,</span> <span class=\"n\">stderr</span> <span class=\"o\">=</span> <span class=\"n\">process</span><span class=\"o\">.</span><span class=\"n\">communicate</span><span class=\"p\">(</span><span class=\"nb\">input</span><span class=\"p\">,</span> <span class=\"n\">timeout</span><span class=\"o\">=</span><span class=\"n\">timeout</span><span class=\"p\">)</span>\n<span class=\"w\"> </span><span class=\"pm\">~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^</span>\n File <span class=\"nb\">"/usr/local/lib/python3.13/subprocess.py"</span>, line <span class=\"m\">1222</span>, in <span class=\"n\">communicate</span>\n<span class=\"w\"> </span><span class=\"n\">stdout</span><span class=\"p\">,</span> <span class=\"n\">stderr</span> <span class=\"o\">=</span> <span class=\"bp\">self</span><span class=\"o\">.</span><span class=\"n\">_communicate</span><span class=\"p\">(</span><span class=\"nb\">input</span><span class=\"p\">,</span> <span class=\"n\">endtime</span><span class=\"p\">,</span> <span class=\"n\">timeout</span><span class=\"p\">)</span>\n<span class=\"w\"> </span><span class=\"pm\">~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^</span>\n File <span class=\"nb\">"/usr/local/lib/python3.13/subprocess.py"</span>, line <span class=\"m\">2154</span>, in <span class=\"n\">_communicate</span>\n<span class=\"w\"> </span><span class=\"n\">ready</span> <span class=\"o\">=</span> <span class=\"n\">selector</span><span class=\"o\">.</span><span class=\"n\">select</span><span class=\"p\">(</span><span class=\"n\">timeout</span><span class=\"p\">)</span>\n File <span class=\"nb\">"/usr/local/lib/python3.13/selectors.py"</span>, line <span class=\"m\">398</span>, in <span class=\"n\">select</span>\n<span class=\"w\"> </span><span class=\"n\">fd_event_list</span> <span class=\"o\">=</span> <span class=\"bp\">self</span><span class=\"o\">.</span><span class=\"n\">_selector</span><span class=\"o\">.</span><span class=\"n\">poll</span><span class=\"p\">(</span><span class=\"n\">timeout</span><span class=\"p\">)</span>\n File <span class=\"nb\">"/usr/local/lib/python3.13/site-packages/marimo/_utils/signals.py"</span>, line <span class=\"m\">33</span>, in <span class=\"n\">__call__</span>\n<span class=\"w\"> </span><span class=\"bp\">self</span><span class=\"o\">.</span><span class=\"n\">_handler</span><span class=\"p\">(</span><span class=\"n\">signum</span><span class=\"p\">,</span> <span class=\"n\">frame</span><span class=\"p\">)</span>\n<span class=\"w\"> </span><span class=\"pm\">~~~~~~~~~~~~~^^^^^^^^^^^^^^^</span>\n File <span class=\"nb\">"/usr/local/lib/python3.13/site-packages/marimo/_runtime/handlers.py"</span>, line <span class=\"m\">74</span>, in <span class=\"n\">interrupt_handler</span>\n<span class=\"w\"> </span><span class=\"k\">raise</span> <span class=\"n\">MarimoInterrupt</span>\n<span class=\"gr\">KeyboardInterrupt</span>\n</pre></div>\n</span>",
"mimetype": "application/vnd.marimo+traceback"
}
]
},
{
"id": "bkHC",
"code_hash": "3e9c348c42c7ba5ad7449c3567bcf13f",
"outputs": [
{
"type": "data",
"data": {
"text/plain": ""
}
}
],
"console": [
{
"type": "stream",
"name": "stdout",
"text": "============================================================\nGPU / NVIDIA DRIVER DIAGNOSTICS\n============================================================\n\n1. Searching for nvidia-smi in common locations:\n /usr/bin/nvidia-smi: \u2705 FOUND\n /usr/local/bin/nvidia-smi: \u274c not found\n /usr/sbin/nvidia-smi: \u274c not found\n /bin/nvidia-smi: \u2705 FOUND\n /opt/nvidia/bin/nvidia-smi: \u274c not found\n\n2. Checking /proc/driver/nvidia (kernel module):\n \u2705 /proc/driver/nvidia EXISTS \u2014 NVIDIA kernel module is loaded\n\n3. Checking /dev/nvidia* device files:\n \u2705 /dev/nvidia-modeset\n \u2705 /dev/nvidia-uvm\n \u2705 /dev/nvidia-uvm-tools\n \u2705 /dev/nvidia5\n \u2705 /dev/nvidiactl\n\n4. Container/VM check:\n Not in a Docker container (no /.dockerenv)\n cgroup info: 7:pids:/08732f114d396919360979af82a55f523c27d3644f584f3cbb76ddf2909f86c0\n6:memory:/08732f114d396919360979af82a55f523c27d3644f584f3cbb76ddf2909f86c0\n5:job:/08732f114d396919360979af82a55f523c27d3644f584\n\n5. Current PATH:\n /tmp/uv-venv/bin:/tmp/uv-venv/lib/python3.13/site-packages/nvidia/cu13/bin:/usr/local/lib/python3.13/site-packages/nvidia/cu13/bin:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\n\n6. LD_LIBRARY_PATH:\n /tmp/uv-venv/lib/python3.13/site-packages/nvidia/cu13/lib:/usr/local/lib/python3.13/site-packages/nvidia/cu13/lib:\n\n7. Searching for CUDA installations:\n \u274c /usr/local/cuda not found\n \u274c /usr/local/cuda-12 not found\n \u274c /usr/local/cuda-13 not found\n \u274c /opt/cuda not found\n\n8. Checking lspci for NVIDIA hardware:\n \u274c lspci command not available\n\n9. PyTorch CUDA details:\n PyTorch version: 2.11.0+cu130\n CUDA available: True\n CUDA version (PyTorch built with): 13.0\n \u2705 libcuda.so is loadable\n\n============================================================\n",
"mimetype": "text/plain"
}
]
},
{
"id": "lEQa",
"code_hash": "8ee678e3af92e02ec6c3e6be3ca5bd98",
"outputs": [
{
"type": "data",
"data": {
"text/markdown": "<span class=\"markdown prose dark:prose-invert contents\"><h2 id=\"gpu-diagnosis-summary-molab-rtx-6000-pro-blackwell\">\ud83d\udd0d GPU Diagnosis Summary \u2014 Molab RTX 6000 Pro Blackwell</h2>\n<span class=\"paragraph\">Based on the diagnostic cells in this notebook, here is what I found:</span>\n<h3 id=\"root-cause-the-gpu-is-not-being-exposed-to-this-molab-containerruntime\">Root Cause: <strong>The GPU is not being exposed to this Molab container/runtime</strong></h3>\n<table>\n<thead>\n<tr>\n<th>Check</th>\n<th>Result</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td><code>nvidia-smi</code> binary</td>\n<td>\u274c Not found anywhere on the system</td>\n</tr>\n<tr>\n<td><code>torch.cuda.is_available()</code></td>\n<td>\u274c Returns <code>False</code></td>\n</tr>\n<tr>\n<td><code>/dev/nvidia*</code> devices</td>\n<td>\u274c Not present</td>\n</tr>\n<tr>\n<td><code>/proc/driver/nvidia</code></td>\n<td>\u274c Not present (no kernel module loaded)</td>\n</tr>\n<tr>\n<td><code>future.py</code> benchmark output</td>\n<td>\u274c Ran on <code>device=cpu</code> with <code>peak_mem=0.00 MB</code></td>\n</tr>\n<tr>\n<td><code>nvidia-smi -pm 1</code> command</td>\n<td>\u274c <code>FileNotFoundError: No such file or directory: 'nvidia-smi'</code></td>\n</tr>\n</tbody>\n</table>\n<h3 id=\"key-observations\">Key Observations</h3>\n<ol>\n<li><strong>PyTorch 2.11.0 with CUDA 13.0.2 is installed</strong> \u2014 all CUDA libraries (cuDNN, cuBLAS, NCCL, etc.) are present via pip, but they are <strong>useless without the GPU being exposed to the container</strong>.</li>\n<li><strong>This is a Molab environment</strong> \u2014 the GPU should be available, but the container/runtime was not started with GPU passthrough properly configured.</li>\n<li><strong>No <code>nvidia-smi</code>, no <code>/dev/nvidia*</code>, no <code>/proc/driver/nvidia</code></strong> \u2014 this means the NVIDIA driver is not accessible inside the container at all. The GPU hardware may exist on the host, but the container cannot reach it.</li>\n<li><strong>JAX also has a CUDA plugin mismatch</strong>: <code>jax_cuda13_plugin v0.11.0</code> is incompatible with <code>jaxlib v0.11.2</code>.</li>\n</ol>\n<h3 id=\"what-needs-to-happen\">What Needs to Happen</h3>\n<span class=\"paragraph\">The problem is <strong>not</strong> with your Python scripts or PyTorch installation \u2014 it's that the <strong>Molab container does not have GPU access configured</strong>. See the fix steps below.</span></span>"
}
}
],
"console": []
},
{
"id": "PKri",
"code_hash": "cbb92b645fbce36bb05be35b0050b993",
"outputs": [],
"console": []
},
{
"id": "Xref",
"code_hash": "742418f0b2f877a29bf7d87e80d61aa7",
"outputs": [],
"console": []
},
{
"id": "SFPL",
"code_hash": "3cf5501ecb9fdc92a089911cf68a8f5f",
"outputs": [],
"console": []
},
{
"id": "BYtC",
"code_hash": "66ea21b2bc2ce38743edd9613eac2d87",
"outputs": [],
"console": []
},
{
"id": "RGSE",
"code_hash": "6470f3209ebb8c510b3f5b2f231bfc62",
"outputs": [
{
"type": "data",
"data": {
"text/html": "<pre class='text-xs'>0</pre>"
}
}
],
"console": [
{
"type": "stream",
"name": "stdout",
"text": "[Tok] Found 5 file(s) in ./training_data\n[Tok] Tokenizing with 5 worker(s) ...\n[Tok] 1/5 files done | 194,327,554 tokens | 88.8s\n[Tok] 2/5 files done | 444,656,091 tokens | 102.5s\n[Tok] 3/5 files done | 697,810,294 tokens | 104.7s\n[Tok] 4/5 files done | 1,183,347,616 tokens | 183.6s\n[Tok] 5/5 files done | 1,705,892,346 tokens | 207.8s\n[Tok] Assembling 1,705,892,346 tokens into ./training_data/.token_cache/tokens_8e3ffbd7fdd3687a.tmp.npy ...\n[Tok] Done in 226.7s | 1,705,892,346 tokens | 6.82 GB\n[Tok] Saved -> ./training_data/.token_cache/tokens_8e3ffbd7fdd3687a.npy\n./training_data/.token_cache/tokens_8e3ffbd7fdd3687a.npy\n",
"mimetype": "text/plain"
}
]
},
{
"id": "emfo",
"code_hash": "5a157233f59154540805237fc3f714af",
"outputs": [],
"console": []
},
{
"id": "Kclp",
"code_hash": "68abf4e051a86628218ef4bfde2bf07b",
"outputs": [
{
"type": "data",
"data": {
"text/html": "<pre class='text-xs'>0</pre>"
}
}
],
"console": [
{
"type": "stream",
"name": "stderr",
"text": "/tmp/uv-venv/lib/python3.13/site-packages/jaxlib/plugin_support.py:99: RuntimeWarning: JAX plugin jax_cuda13_plugin version 0.11.0 is installed, but it is not compatible with the installed jaxlib version 0.11.2, so it will not be used.\n warnings.warn(\n/marimo/model.py:407: UserWarning: config module has no attribute 'use_engram' -- falling back to default False. If this is unexpected, check for a casing mismatch or rename in your config.py.\n warnings.warn(\n/marimo/model.py:407: UserWarning: config module has no attribute 'engram_layer_ids' -- falling back to default (1,). If this is unexpected, check for a casing mismatch or rename in your config.py.\n warnings.warn(\n/marimo/model.py:407: UserWarning: config module has no attribute 'engram_max_ngram' -- falling back to default 3. If this is unexpected, check for a casing mismatch or rename in your config.py.\n warnings.warn(\n/marimo/model.py:407: UserWarning: config module has no attribute 'engram_vocab_size' -- falling back to default (8192, 8192). If this is unexpected, check for a casing mismatch or rename in your config.py.\n warnings.warn(\n/marimo/model.py:407: UserWarning: config module has no attribute 'engram_embed_per_ngram' -- falling back to default 128. If this is unexpected, check for a casing mismatch or rename in your config.py.\n warnings.warn(\n/marimo/model.py:407: UserWarning: config module has no attribute 'engram_n_heads' -- falling back to default 4. If this is unexpected, check for a casing mismatch or rename in your config.py.\n warnings.warn(\n/marimo/model.py:407: UserWarning: config module has no attribute 'engram_kernel_size' -- falling back to default 4. If this is unexpected, check for a casing mismatch or rename in your config.py.\n warnings.warn(\n/marimo/model.py:407: UserWarning: config module has no attribute 'engram_seed' -- falling back to default 0. If this is unexpected, check for a casing mismatch or rename in your config.py.\n warnings.warn(\n/marimo/model.py:407: UserWarning: config module has no attribute 'engram_compressed_vocab' -- falling back to default 0. If this is unexpected, check for a casing mismatch or rename in your config.py.\n warnings.warn(\n[RANK 0] Detected kernel version 4.19.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n",
"mimetype": "text/plain"
},
{
"type": "stream",
"name": "stdout",
"text": "[Setup] GPU compute capability 12.x (Ampere+) -- TF32 enabled for fp32 matmuls/convs.\n[Setup] Parallelism: plain data-parallel (DDP) only -- no FSDP/tensor-parallel/pipeline-parallel. Appropriate for small device counts (e.g. dual T4); set use_fsdp=True in config.py when sharding is actually needed.\n[Setup] Using device: cuda\n[Setup] Num processes (GPUs): 1\n[Tokenizer] Loaded, vocab_size=16384\n[Dataset] Found 5 files in ./training_data (curriculum OFF)\n[Dataset] Cache hit -> ./training_data/.token_cache/tokens_e02f67284dbe9bc8.npy (1,705,892,330 tokens)\n[Dataset] train=1,688,833,377 tokens, val=17,058,921 tokens\n[Val] fixed validation set: 33 batches x 8 seqs across 5 file(s) (cosmopedia_800mb.txt:4, fineweb_edu_2000mb.txt:9, littlecurriculum_1000mb.txt:5, smollm_cosmopedia_v2_2500mb.txt:10, tinystories_1000mb.txt:5)\n",
"mimetype": "text/plain"
},
{
"type": "stream",
"name": "stderr",
"text": "WARNING[XFORMERS]: xFormers can't load C++/CUDA extensions. xFormers was built for:\n PyTorch 2.10.0+cu128 with CUDA 1208 (you have 2.11.0+cu130)\n Python 3.10.19 (you have 3.13.11)\n Please reinstall xformers (see https://github.com/facebookresearch/xformers#installing-xformers)\n Memory-efficient attention, SwiGLU, sparse and more won't be available.\n Set XFORMERS_MORE_DETAILS=1 for more details\n",
"mimetype": "text/plain"
},
{
"type": "stream",
"name": "stdout",
"text": "[Setup] GPU-side batching=True, num_workers=2 (from config.py)\n[Mem] 4.06/101.98 GB used after corpus+batcher on-device (96% free).\n[kernel] attention backend device=cuda:0\n gpu NVIDIA RTX PRO 6000 Blackwell Server Edition (sm_120)\n turing-flash not eligible (sm != 7.5)\n flex-attention ACTIVE (use_flexattention=True in config.py)\n fa4 not eligible (sm_120 lacks FA4's required warp-specialized MMA -- uses FlashAttention-2 instead)\n flash_attn available (Ampere+ path, backend='flash')\n xformers not installed\n >>> active <<< flex-attention (config-enabled, torch.compile'd)\n[TritonAudit] triton_kernel.py not found -- eager path.\n[Compile] Compiling model for maximum speed...\n[Compile] reduce-overhead requested via alpha_config.ALPHA_COMPILE_MODE. model.py's lm_head now reads the tied weight via F.linear(x, self.lm_head.weight) instead of the module call, specifically to remove the aliasing ambiguity that caused the earlier 'overwritten by a subsequent run' crash at wte(idx). This is a plausible fix, NOT a verified one -- it hasn't been run against the actual crash on real hardware. VERIFY before trusting a real run: run a few hundred steps, and if val_loss tracks normally (compare against a mode=default run on the same data/seed) and there's no CUDAGraphs RuntimeError, it's confirmed safe. If loss silently diverges or looks wrong without an explicit crash, that's worse than a crash -- stop and revert to mode=default immediately.\n[Compile/diag] reduce-overhead active. reserved_bytes.all.current=3376.0 MiB, num_alloc_retries=0 -- watch this grow abnormally across the first few steps as a sign of multiple cudagraph pools being captured (one per accumulate()/sync branch).\n[Model] Total params: 21.40M\n[Checkpoint] Resumed from step 4201\nstep 4250 | epoch 0.82 | loss 3.2497 | ce 2.7845 | mtp 4.7671 | lr 1.00e-03 | tok/s 272,809 | elapsed 1.0min\nstep 4300 | epoch 0.83 | loss 3.2486 | ce 2.7882 | mtp 4.7806 | lr 1.00e-03 | tok/s 382,466 | elapsed 1.7min\nstep 4350 | epoch 0.84 | loss 3.2540 | ce 2.7770 | mtp 4.7296 | lr 1.00e-03 | tok/s 382,523 | elapsed 2.4min\nstep 4400 | epoch 0.85 | loss 3.2446 | ce 2.7229 | mtp 4.6862 | lr 1.00e-03 | tok/s 382,623 | elapsed 3.1min\nstep 4450 | epoch 0.86 | loss 3.2412 | ce 2.7880 | mtp 4.7893 | lr 1.00e-03 | tok/s 382,591 | elapsed 3.8min\nstep 4500 | epoch 0.87 | loss 3.2548 | ce 2.7694 | mtp 4.7516 | lr 1.00e-03 | tok/s 382,453 | elapsed 4.6min\nstep 4550 | epoch 0.88 | loss 3.2409 | ce 2.7962 | mtp 4.7767 | lr 1.00e-03 | tok/s 382,622 | elapsed 5.3min\nstep 4600 | epoch 0.89 | loss 3.2531 | ce 2.7872 | mtp 4.7626 | lr 1.00e-03 | tok/s 382,553 | elapsed 6.0min\nstep 4650 | epoch 0.90 | loss 3.2403 | ce 2.7664 | mtp 4.7491 | lr 1.00e-03 | tok/s 382,398 | elapsed 6.7min\nstep 4700 | epoch 0.91 | loss 3.2396 | ce 2.7420 | mtp 4.7319 | lr 1.00e-03 | tok/s 382,365 | elapsed 7.4min\nstep 4750 | epoch 0.92 | loss 3.2410 | ce 2.8252 | mtp 4.8113 | lr 1.00e-03 | tok/s 382,496 | elapsed 8.1min\nstep 4800 | epoch 0.93 | loss 3.2443 | ce 2.7749 | mtp 4.7580 | lr 1.00e-03 | tok/s 382,385 | elapsed 8.8min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 4801)\n",
"mimetype": "text/plain"
},
{
"type": "stream",
"name": "stderr",
"text": "/usr/local/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.\n\nSOLUTION: Use torch.compile(flex_attention)(...)\n\nIf you want to debug your score_mod/mask_mod, you can set:\ntorch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True\n\nThis will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.\n _warn_once(\n",
"mimetype": "text/plain"
},
{
"type": "stream",
"name": "stdout",
"text": "step 4850 | epoch 0.94 | loss 3.2659 | ce 2.7077 | mtp 4.6465 | lr 1.00e-03 | tok/s 380,742 | elapsed 9.6min\nstep 4900 | epoch 0.95 | loss 3.2798 | ce 2.8341 | mtp 4.8258 | lr 1.00e-03 | tok/s 382,575 | elapsed 10.3min\nstep 4950 | epoch 0.96 | loss 3.2609 | ce 2.8397 | mtp 4.8178 | lr 1.00e-03 | tok/s 382,502 | elapsed 11.0min\nstep 5000 | epoch 0.97 | loss 3.2684 | ce 2.7936 | mtp 4.7488 | lr 1.00e-03 | tok/s 382,634 | elapsed 11.7min\nstep 5000 | val_loss 2.7831 | val_acc 0.4447 | val_ppl 16.17 | val_mtp 4.7238\nstep 5050 | epoch 0.98 | loss 3.2597 | ce 2.7843 | mtp 4.7560 | lr 1.00e-03 | tok/s 348,386 | elapsed 12.5min\nstep 5100 | epoch 0.99 | loss 3.2641 | ce 2.7968 | mtp 4.7500 | lr 1.00e-03 | tok/s 382,641 | elapsed 13.2min\nstep 5150 | epoch 1.00 | loss 3.2572 | ce 2.7979 | mtp 4.7651 | lr 1.00e-03 | tok/s 382,574 | elapsed 13.9min\nstep 5200 | epoch 1.01 | loss 3.2515 | ce 2.7947 | mtp 4.7557 | lr 1.00e-03 | tok/s 382,578 | elapsed 14.6min\nstep 5250 | epoch 1.02 | loss 3.2621 | ce 2.7278 | mtp 4.6914 | lr 1.00e-03 | tok/s 382,645 | elapsed 15.3min\nstep 5300 | epoch 1.03 | loss 3.2546 | ce 2.7793 | mtp 4.7285 | lr 1.00e-03 | tok/s 382,480 | elapsed 16.0min\nstep 5350 | epoch 1.04 | loss 3.2455 | ce 2.6583 | mtp 4.5840 | lr 1.00e-03 | tok/s 382,576 | elapsed 16.8min\nstep 5400 | epoch 1.05 | loss 3.2523 | ce 2.7776 | mtp 4.7328 | lr 1.00e-03 | tok/s 382,543 | elapsed 17.5min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 5401)\nstep 5450 | epoch 1.06 | loss 3.2423 | ce 2.7498 | mtp 4.6815 | lr 1.00e-03 | tok/s 382,251 | elapsed 18.2min\nstep 5500 | epoch 1.07 | loss 3.2510 | ce 2.6684 | mtp 4.6032 | lr 1.00e-03 | tok/s 382,487 | elapsed 18.9min\nstep 5550 | epoch 1.08 | loss 3.2436 | ce 2.7862 | mtp 4.7602 | lr 1.00e-03 | tok/s 382,474 | elapsed 19.6min\nstep 5600 | epoch 1.09 | loss 3.2381 | ce 2.7546 | mtp 4.7156 | lr 1.00e-03 | tok/s 382,658 | elapsed 20.3min\nstep 5650 | epoch 1.10 | loss 3.2513 | ce 2.7550 | mtp 4.6968 | lr 1.00e-03 | tok/s 382,566 | elapsed 21.0min\nstep 5700 | epoch 1.11 | loss 3.2449 | ce 2.7134 | mtp 4.6536 | lr 1.00e-03 | tok/s 382,552 | elapsed 21.8min\nstep 5750 | epoch 1.12 | loss 3.2471 | ce 2.8041 | mtp 4.7418 | lr 1.00e-03 | tok/s 382,589 | elapsed 22.5min\nstep 5800 | epoch 1.13 | loss 3.2334 | ce 2.6755 | mtp 4.6193 | lr 1.00e-03 | tok/s 382,541 | elapsed 23.2min\nstep 5850 | epoch 1.14 | loss 3.2424 | ce 2.7036 | mtp 4.6521 | lr 1.00e-03 | tok/s 382,645 | elapsed 23.9min\nstep 5900 | epoch 1.14 | loss 3.2408 | ce 2.8372 | mtp 4.8067 | lr 1.00e-03 | tok/s 382,578 | elapsed 24.6min\nstep 5950 | epoch 1.15 | loss 3.2185 | ce 2.7701 | mtp 4.7291 | lr 1.00e-03 | tok/s 382,491 | elapsed 25.3min\nstep 6000 | epoch 1.16 | loss 3.2294 | ce 2.8179 | mtp 4.7804 | lr 1.00e-03 | tok/s 382,481 | elapsed 26.0min\nstep 6000 | val_loss 2.7617 | val_acc 0.4469 | val_ppl 15.83 | val_mtp 4.6964\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 6001)\nstep 6050 | epoch 1.17 | loss 3.2329 | ce 2.7653 | mtp 4.7358 | lr 1.00e-03 | tok/s 351,542 | elapsed 26.8min\nstep 6100 | epoch 1.18 | loss 3.2337 | ce 2.7917 | mtp 4.7440 | lr 1.00e-03 | tok/s 382,571 | elapsed 27.5min\nstep 6150 | epoch 1.19 | loss 3.2299 | ce 2.7774 | mtp 4.7530 | lr 1.00e-03 | tok/s 382,382 | elapsed 28.2min\nstep 6200 | epoch 1.20 | loss 3.2335 | ce 2.8303 | mtp 4.8014 | lr 1.00e-03 | tok/s 382,514 | elapsed 29.0min\nstep 6250 | epoch 1.21 | loss 3.2377 | ce 2.7719 | mtp 4.7240 | lr 1.00e-03 | tok/s 382,644 | elapsed 29.7min\nstep 6300 | epoch 1.22 | loss 3.2381 | ce 2.8040 | mtp 4.7803 | lr 1.00e-03 | tok/s 382,635 | elapsed 30.4min\nstep 6350 | epoch 1.23 | loss 3.2210 | ce 2.7596 | mtp 4.7317 | lr 1.00e-03 | tok/s 382,586 | elapsed 31.1min\nstep 6400 | epoch 1.24 | loss 3.2329 | ce 2.8232 | mtp 4.7856 | lr 1.00e-03 | tok/s 382,626 | elapsed 31.8min\nstep 6450 | epoch 1.25 | loss 3.2189 | ce 2.7387 | mtp 4.7013 | lr 1.00e-03 | tok/s 382,641 | elapsed 32.5min\nstep 6500 | epoch 1.26 | loss 3.2280 | ce 2.7446 | mtp 4.6822 | lr 1.00e-03 | tok/s 382,626 | elapsed 33.2min\nstep 6550 | epoch 1.27 | loss 3.2220 | ce 2.7189 | mtp 4.6619 | lr 1.00e-03 | tok/s 382,569 | elapsed 34.0min\nstep 6600 | epoch 1.28 | loss 3.2266 | ce 2.7489 | mtp 4.6926 | lr 1.00e-03 | tok/s 382,428 | elapsed 34.7min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 6601)\nstep 6650 | epoch 1.29 | loss 3.2219 | ce 2.7380 | mtp 4.6877 | lr 1.00e-03 | tok/s 382,422 | elapsed 35.4min\nstep 6700 | epoch 1.30 | loss 3.2319 | ce 2.7639 | mtp 4.7341 | lr 1.00e-03 | tok/s 382,620 | elapsed 36.1min\nstep 6750 | epoch 1.31 | loss 3.2213 | ce 2.7892 | mtp 4.7617 | lr 1.00e-03 | tok/s 382,500 | elapsed 36.8min\nstep 6800 | epoch 1.32 | loss 3.2230 | ce 2.8104 | mtp 4.7653 | lr 1.00e-03 | tok/s 382,452 | elapsed 37.5min\nstep 6850 | epoch 1.33 | loss 3.2214 | ce 2.7859 | mtp 4.7450 | lr 1.00e-03 | tok/s 382,326 | elapsed 38.2min\nstep 6900 | epoch 1.34 | loss 3.2172 | ce 2.7358 | mtp 4.6660 | lr 1.00e-03 | tok/s 382,567 | elapsed 39.0min\nstep 6950 | epoch 1.35 | loss 3.2195 | ce 2.7229 | mtp 4.6856 | lr 1.00e-03 | tok/s 382,392 | elapsed 39.7min\nstep 7000 | epoch 1.36 | loss 3.2114 | ce 2.6931 | mtp 4.6144 | lr 1.00e-03 | tok/s 382,575 | elapsed 40.4min\nstep 7000 | val_loss 2.7433 | val_acc 0.4497 | val_ppl 15.54 | val_mtp 4.6769\nstep 7050 | epoch 1.37 | loss 3.2335 | ce 2.8189 | mtp 4.7878 | lr 1.00e-03 | tok/s 351,393 | elapsed 41.2min\nstep 7100 | epoch 1.38 | loss 3.2065 | ce 2.7366 | mtp 4.7006 | lr 1.00e-03 | tok/s 382,353 | elapsed 41.9min\nstep 7150 | epoch 1.39 | loss 3.2068 | ce 2.6479 | mtp 4.5821 | lr 1.00e-03 | tok/s 382,496 | elapsed 42.6min\nstep 7200 | epoch 1.40 | loss 3.2185 | ce 2.7355 | mtp 4.7042 | lr 1.00e-03 | tok/s 382,516 | elapsed 43.3min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 7201)\nstep 7250 | epoch 1.41 | loss 3.2064 | ce 2.7026 | mtp 4.6376 | lr 1.00e-03 | tok/s 382,381 | elapsed 44.0min\nstep 7300 | epoch 1.42 | loss 3.2029 | ce 2.6976 | mtp 4.6647 | lr 1.00e-03 | tok/s 382,482 | elapsed 44.7min\nstep 7350 | epoch 1.43 | loss 3.2211 | ce 2.7250 | mtp 4.6739 | lr 1.00e-03 | tok/s 382,499 | elapsed 45.4min\nstep 7400 | epoch 1.44 | loss 3.2053 | ce 2.7086 | mtp 4.6259 | lr 1.00e-03 | tok/s 382,423 | elapsed 46.2min\nstep 7450 | epoch 1.45 | loss 3.2124 | ce 2.7437 | mtp 4.7088 | lr 1.00e-03 | tok/s 382,420 | elapsed 46.9min\nstep 7500 | epoch 1.46 | loss 3.2008 | ce 2.7938 | mtp 4.7554 | lr 1.00e-03 | tok/s 382,444 | elapsed 47.6min\nstep 7550 | epoch 1.46 | loss 3.2253 | ce 2.8095 | mtp 4.7720 | lr 1.00e-03 | tok/s 382,515 | elapsed 48.3min\nstep 7600 | epoch 1.47 | loss 3.2146 | ce 2.6771 | mtp 4.5960 | lr 1.00e-03 | tok/s 382,387 | elapsed 49.0min\nstep 7650 | epoch 1.48 | loss 3.2117 | ce 2.7279 | mtp 4.6717 | lr 1.00e-03 | tok/s 382,512 | elapsed 49.7min\nstep 7700 | epoch 1.49 | loss 3.2166 | ce 2.8195 | mtp 4.7949 | lr 1.00e-03 | tok/s 382,432 | elapsed 50.4min\nstep 7750 | epoch 1.50 | loss 3.2071 | ce 2.8050 | mtp 4.7955 | lr 1.00e-03 | tok/s 382,485 | elapsed 51.2min\nstep 7800 | epoch 1.51 | loss 3.1955 | ce 2.6457 | mtp 4.5646 | lr 1.00e-03 | tok/s 382,362 | elapsed 51.9min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 7801)\nstep 7850 | epoch 1.52 | loss 3.1984 | ce 2.7339 | mtp 4.6823 | lr 1.00e-03 | tok/s 382,259 | elapsed 52.6min\nstep 7900 | epoch 1.53 | loss 3.2054 | ce 2.7006 | mtp 4.6352 | lr 1.00e-03 | tok/s 382,384 | elapsed 53.3min\nstep 7950 | epoch 1.54 | loss 3.1984 | ce 2.7458 | mtp 4.7036 | lr 1.00e-03 | tok/s 382,419 | elapsed 54.0min\nstep 8000 | epoch 1.55 | loss 3.2196 | ce 2.6623 | mtp 4.5975 | lr 1.00e-03 | tok/s 382,336 | elapsed 54.7min\nstep 8000 | val_loss 2.7312 | val_acc 0.4509 | val_ppl 15.35 | val_mtp 4.6579\nstep 8050 | epoch 1.56 | loss 3.2058 | ce 2.7301 | mtp 4.6740 | lr 1.00e-03 | tok/s 351,525 | elapsed 55.5min\nstep 8100 | epoch 1.57 | loss 3.2071 | ce 2.7264 | mtp 4.6461 | lr 1.00e-03 | tok/s 382,513 | elapsed 56.2min\nstep 8150 | epoch 1.58 | loss 3.2008 | ce 2.6983 | mtp 4.6641 | lr 1.00e-03 | tok/s 382,416 | elapsed 56.9min\nstep 8200 | epoch 1.59 | loss 3.2116 | ce 2.8162 | mtp 4.7914 | lr 1.00e-03 | tok/s 382,331 | elapsed 57.6min\nstep 8250 | epoch 1.60 | loss 3.2034 | ce 2.7395 | mtp 4.6785 | lr 1.00e-03 | tok/s 382,449 | elapsed 58.4min\nstep 8300 | epoch 1.61 | loss 3.1981 | ce 2.6764 | mtp 4.5919 | lr 1.00e-03 | tok/s 382,575 | elapsed 59.1min\nstep 8350 | epoch 1.62 | loss 3.2048 | ce 2.6708 | mtp 4.5898 | lr 1.00e-03 | tok/s 382,416 | elapsed 59.8min\nstep 8400 | epoch 1.63 | loss 3.1999 | ce 2.7175 | mtp 4.6515 | lr 1.00e-03 | tok/s 382,356 | elapsed 60.5min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 8401)\nstep 8450 | epoch 1.64 | loss 3.1904 | ce 2.6759 | mtp 4.6114 | lr 1.00e-03 | tok/s 382,350 | elapsed 61.2min\nstep 8500 | epoch 1.65 | loss 3.2006 | ce 2.7439 | mtp 4.7034 | lr 1.00e-03 | tok/s 382,496 | elapsed 61.9min\nstep 8550 | epoch 1.66 | loss 3.1973 | ce 2.7408 | mtp 4.6826 | lr 9.99e-04 | tok/s 382,616 | elapsed 62.6min\nstep 8600 | epoch 1.67 | loss 3.2094 | ce 2.7908 | mtp 4.7327 | lr 9.96e-04 | tok/s 382,496 | elapsed 63.4min\nstep 8650 | epoch 1.68 | loss 3.1980 | ce 2.7243 | mtp 4.6695 | lr 9.92e-04 | tok/s 382,591 | elapsed 64.1min\nstep 8700 | epoch 1.69 | loss 3.1873 | ce 2.6993 | mtp 4.6424 | lr 9.86e-04 | tok/s 382,541 | elapsed 64.8min\nstep 8750 | epoch 1.70 | loss 3.1929 | ce 2.7614 | mtp 4.7247 | lr 9.78e-04 | tok/s 382,477 | elapsed 65.5min\nstep 8800 | epoch 1.71 | loss 3.1957 | ce 2.7114 | mtp 4.6565 | lr 9.68e-04 | tok/s 382,398 | elapsed 66.2min\nstep 8850 | epoch 1.72 | loss 3.1805 | ce 2.7804 | mtp 4.7508 | lr 9.57e-04 | tok/s 382,362 | elapsed 66.9min\nstep 8900 | epoch 1.73 | loss 3.1945 | ce 2.6758 | mtp 4.6215 | lr 9.44e-04 | tok/s 382,470 | elapsed 67.6min\nstep 8950 | epoch 1.74 | loss 3.1968 | ce 2.7593 | mtp 4.7063 | lr 9.30e-04 | tok/s 382,393 | elapsed 68.4min\nstep 9000 | epoch 1.75 | loss 3.1973 | ce 2.7034 | mtp 4.6453 | lr 9.14e-04 | tok/s 382,458 | elapsed 69.1min\nstep 9000 | val_loss 2.7127 | val_acc 0.4529 | val_ppl 15.07 | val_mtp 4.6364\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 9001)\nstep 9050 | epoch 1.76 | loss 3.1857 | ce 2.6662 | mtp 4.6081 | lr 8.96e-04 | tok/s 351,321 | elapsed 69.8min\nstep 9100 | epoch 1.77 | loss 3.1846 | ce 2.7206 | mtp 4.6902 | lr 8.78e-04 | tok/s 382,386 | elapsed 70.6min\nstep 9150 | epoch 1.78 | loss 3.1873 | ce 2.7426 | mtp 4.6904 | lr 8.58e-04 | tok/s 382,469 | elapsed 71.3min\nstep 9200 | epoch 1.79 | loss 3.1690 | ce 2.7144 | mtp 4.6480 | lr 8.36e-04 | tok/s 382,399 | elapsed 72.0min\nstep 9250 | epoch 1.79 | loss 3.1679 | ce 2.6436 | mtp 4.5934 | lr 8.14e-04 | tok/s 382,521 | elapsed 72.7min\nstep 9300 | epoch 1.80 | loss 3.1727 | ce 2.7025 | mtp 4.6337 | lr 7.91e-04 | tok/s 382,364 | elapsed 73.4min\nstep 9350 | epoch 1.81 | loss 3.1601 | ce 2.6701 | mtp 4.6017 | lr 7.66e-04 | tok/s 382,370 | elapsed 74.1min\nstep 9400 | epoch 1.82 | loss 3.1583 | ce 2.6817 | mtp 4.6235 | lr 7.41e-04 | tok/s 382,369 | elapsed 74.8min\nstep 9450 | epoch 1.83 | loss 3.1644 | ce 2.7060 | mtp 4.6738 | lr 7.15e-04 | tok/s 382,465 | elapsed 75.6min\nstep 9500 | epoch 1.84 | loss 3.1556 | ce 2.7239 | mtp 4.6831 | lr 6.89e-04 | tok/s 382,457 | elapsed 76.3min\nstep 9550 | epoch 1.85 | loss 3.1447 | ce 2.7006 | mtp 4.6298 | lr 6.61e-04 | tok/s 382,287 | elapsed 77.0min\nstep 9600 | epoch 1.86 | loss 3.1491 | ce 2.7040 | mtp 4.6683 | lr 6.34e-04 | tok/s 382,500 | elapsed 77.7min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 9601)\nstep 9650 | epoch 1.87 | loss 3.1417 | ce 2.6781 | mtp 4.6272 | lr 6.06e-04 | tok/s 382,285 | elapsed 78.4min\nstep 9700 | epoch 1.88 | loss 3.1306 | ce 2.6439 | mtp 4.6024 | lr 5.78e-04 | tok/s 382,489 | elapsed 79.1min\nstep 9750 | epoch 1.89 | loss 3.1398 | ce 2.7172 | mtp 4.6728 | lr 5.49e-04 | tok/s 382,429 | elapsed 79.8min\nstep 9800 | epoch 1.90 | loss 3.1197 | ce 2.6194 | mtp 4.5320 | lr 5.21e-04 | tok/s 382,525 | elapsed 80.6min\nstep 9850 | epoch 1.91 | loss 3.1295 | ce 2.6028 | mtp 4.5196 | lr 4.93e-04 | tok/s 382,406 | elapsed 81.3min\nstep 9900 | epoch 1.92 | loss 3.1206 | ce 2.6325 | mtp 4.5692 | lr 4.65e-04 | tok/s 382,472 | elapsed 82.0min\nstep 9950 | epoch 1.93 | loss 3.1129 | ce 2.6952 | mtp 4.6722 | lr 4.38e-04 | tok/s 382,446 | elapsed 82.7min\nstep 10000 | epoch 1.94 | loss 3.1154 | ce 2.6692 | mtp 4.6086 | lr 4.10e-04 | tok/s 382,520 | elapsed 83.4min\nstep 10000 | val_loss 2.6455 | val_acc 0.4628 | val_ppl 14.09 | val_mtp 4.5700\nstep 10050 | epoch 1.95 | loss 3.1152 | ce 2.7101 | mtp 4.6998 | lr 3.84e-04 | tok/s 351,479 | elapsed 84.2min\nstep 10100 | epoch 1.96 | loss 3.0981 | ce 2.7101 | mtp 4.6818 | lr 3.58e-04 | tok/s 382,386 | elapsed 84.9min\nstep 10150 | epoch 1.97 | loss 3.1073 | ce 2.6429 | mtp 4.5895 | lr 3.33e-04 | tok/s 382,338 | elapsed 85.6min\nstep 10200 | epoch 1.98 | loss 3.0987 | ce 2.6008 | mtp 4.5463 | lr 3.08e-04 | tok/s 382,221 | elapsed 86.3min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 10201)\nstep 10250 | epoch 1.99 | loss 3.0922 | ce 2.5542 | mtp 4.4953 | lr 2.85e-04 | tok/s 382,100 | elapsed 87.0min\nstep 10300 | epoch 2.00 | loss 3.1006 | ce 2.5690 | mtp 4.5043 | lr 2.63e-04 | tok/s 382,503 | elapsed 87.8min\nstep 10350 | epoch 2.01 | loss 3.0934 | ce 2.6455 | mtp 4.5685 | lr 2.42e-04 | tok/s 382,499 | elapsed 88.5min\nstep 10400 | epoch 2.02 | loss 3.0852 | ce 2.6516 | mtp 4.5939 | lr 2.22e-04 | tok/s 382,421 | elapsed 89.2min\nstep 10450 | epoch 2.03 | loss 3.0919 | ce 2.6383 | mtp 4.5758 | lr 2.03e-04 | tok/s 382,527 | elapsed 89.9min\nstep 10500 | epoch 2.04 | loss 3.0712 | ce 2.5706 | mtp 4.4839 | lr 1.86e-04 | tok/s 382,507 | elapsed 90.6min\nstep 10550 | epoch 2.05 | loss 3.0722 | ce 2.5038 | mtp 4.4426 | lr 1.70e-04 | tok/s 382,466 | elapsed 91.3min\nstep 10600 | epoch 2.06 | loss 3.0809 | ce 2.7021 | mtp 4.6678 | lr 1.55e-04 | tok/s 382,419 | elapsed 92.0min\nstep 10650 | epoch 2.07 | loss 3.0762 | ce 2.5851 | mtp 4.5454 | lr 1.43e-04 | tok/s 382,504 | elapsed 92.8min\nstep 10700 | epoch 2.08 | loss 3.0696 | ce 2.6424 | mtp 4.5866 | lr 1.31e-04 | tok/s 382,402 | elapsed 93.5min\nstep 10750 | epoch 2.09 | loss 3.0728 | ce 2.6090 | mtp 4.5445 | lr 1.22e-04 | tok/s 382,446 | elapsed 94.2min\nstep 10800 | epoch 2.10 | loss 3.0575 | ce 2.6173 | mtp 4.5710 | lr 1.14e-04 | tok/s 382,458 | elapsed 94.9min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 10801)\nstep 10850 | epoch 2.11 | loss 3.0712 | ce 2.6774 | mtp 4.6211 | lr 1.08e-04 | tok/s 382,147 | elapsed 95.6min\nstep 10900 | epoch 2.11 | loss 3.0621 | ce 2.5669 | mtp 4.4923 | lr 1.03e-04 | tok/s 382,404 | elapsed 96.3min\nstep 10950 | epoch 2.12 | loss 3.0637 | ce 2.6406 | mtp 4.5938 | lr 1.01e-04 | tok/s 382,365 | elapsed 97.0min\n[Save] state_dict(): 0.00s | 181 tensors | 0.14 GB\n[Save] optimizer.state_dict(): 0.00s | 279 tensors | 0.14 GB\n[Save] _cpu walk: 0.18s\n[Save] torch.save -> /tmp: 0.17s | file 272 MB\n[Save] copyfile -> ./checkpoints/arya_latest.pt: 1.24s\n[Training] Complete\n",
"mimetype": "text/plain"
}
]
},
{
"id": "Hstk",
"code_hash": "491df3d6836c4a6956d9cdf3e6e8d89f",
"outputs": [
{
"type": "error",
"ename": "multiple-defs",
"evalue": "The variable 'os' was defined by another cell",
"traceback": []
}
],
"console": []
},
{
"id": "nWHF",
"code_hash": "d108748a7f1e39fb39644a9755a27b91",
"outputs": [],
"console": []
},
{
"id": "iLit",
"code_hash": "a1daefd9105ef45d64a60fc09e306b69",
"outputs": [],
"console": []
},
{
"id": "ZHCJ",
"code_hash": "9aa4360427f7a6b988b89521f957855c",
"outputs": [],
"console": []
},
{
"id": "ROlb",
"code_hash": null,
"outputs": [],
"console": []
},
{
"id": "qnkX",
"code_hash": "26bf08358a5fd5faeadfb725840c6654",
"outputs": [],
"console": []
},
{
"id": "TqIu",
"code_hash": "8cf0ab5f2f0947afe55db20860f35990",
"outputs": [],
"console": []
},
{
"id": "Vxnm",
"code_hash": "472b180a72ef3d7ee4b56780749ad24e",
"outputs": [],
"console": []
},
{
"id": "DnEU",
"code_hash": "d6b0911a49796272ca86fe8f1cfba105",
"outputs": [],
"console": []
},
{
"id": "ulZA",
"code_hash": "42312cfb889e4c7d1d95b40183870910",
"outputs": [],
"console": []
},
{
"id": "ecfG",
"code_hash": "8a966b97e40cefcac634802092c90442",
"outputs": [
{
"type": "data",
"data": {
"text/html": "<pre class='text-xs'>0</pre>"
}
}
],
"console": [
{
"type": "stream",
"name": "stderr",
"text": "WARNING: The directory '/home/marimo/.cache/pip' or its parent directory is not owned or is not writable by the current user. The cache has been disabled. Check the permissions and owner of that directory. If executing pip with sudo, you should use sudo's -H flag.\n",
"mimetype": "text/plain"
},
{
"type": "stream",
"name": "stdout",
"text": "Requirement already satisfied: liger-kernel in /tmp/uv-venv/lib/python3.13/site-packages (0.8.4)\nRequirement already satisfied: rustbpe in /tmp/uv-venv/lib/python3.13/site-packages (0.1.0)\nRequirement already satisfied: torchao in /tmp/uv-venv/lib/python3.13/site-packages (0.18.0)\nRequirement already satisfied: triton in /tmp/uv-venv/lib/python3.13/site-packages (3.8.0)\nCollecting accelerate\n Downloading accelerate-1.15.0-py3-none-any.whl.metadata (19 kB)\nRequirement already satisfied: torch>=2.1.2 in /usr/local/lib/python3.13/site-packages (from liger-kernel) (2.11.0)\nRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.13/site-packages (from accelerate) (2.4.6)\nRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.13/site-packages (from accelerate) (26.2)\nRequirement already satisfied: psutil in /usr/local/lib/python3.13/site-packages (from accelerate) (7.2.2)\nRequirement already satisfied: pyyaml in /usr/local/lib/python3.13/site-packages (from accelerate) (6.0.3)\nRequirement already satisfied: huggingface_hub>=0.21.0 in /tmp/uv-venv/lib/python3.13/site-packages (from accelerate) (1.33.0)\nRequirement already satisfied: safetensors>=0.4.3 in /tmp/uv-venv/lib/python3.13/site-packages (from accelerate) (0.8.0)\nRequirement already satisfied: click<9.0.0,>=8.4.2 in /usr/local/lib/python3.13/site-packages (from huggingface_hub>=0.21.0->accelerate) (8.4.2)\nRequirement already satisfied: filelock>=3.10.0 in /tmp/uv-venv/lib/python3.13/site-packages (from huggingface_hub>=0.21.0->accelerate) (4.1.0)\nRequirement already satisfied: fsspec>=2023.5.0 in /tmp/uv-venv/lib/python3.13/site-packages (from huggingface_hub>=0.21.0->accelerate) (2026.9.0)\nRequirement already satisfied: hf-xet<2.0.0,>=1.6.0 in /tmp/uv-venv/lib/python3.13/site-packages (from huggingface_hub>=0.21.0->accelerate) (1.7.0)\nRequirement already satisfied: httpx<1,>=0.23.0 in /tmp/uv-venv/lib/python3.13/site-packages (from huggingface_hub>=0.21.0->accelerate) (0.28.1)\nRequirement already satisfied: tqdm>=4.42.1 in /tmp/uv-venv/lib/python3.13/site-packages (from huggingface_hub>=0.21.0->accelerate) (4.70.1)\nRequirement already satisfied: typing-extensions>=4.1.0 in /usr/local/lib/python3.13/site-packages (from huggingface_hub>=0.21.0->accelerate) (4.16.0)\nRequirement already satisfied: anyio in /tmp/uv-venv/lib/python3.13/site-packages (from httpx<1,>=0.23.0->huggingface_hub>=0.21.0->accelerate) (4.15.1)\nRequirement already satisfied: certifi in /tmp/uv-venv/lib/python3.13/site-packages (from httpx<1,>=0.23.0->huggingface_hub>=0.21.0->accelerate) (2026.7.22)\nRequirement already satisfied: httpcore==1.* in /tmp/uv-venv/lib/python3.13/site-packages (from httpx<1,>=0.23.0->huggingface_hub>=0.21.0->accelerate) (1.0.9)\nRequirement already satisfied: idna in /tmp/uv-venv/lib/python3.13/site-packages (from httpx<1,>=0.23.0->huggingface_hub>=0.21.0->accelerate) (3.20)\nRequirement already satisfied: h11>=0.16 in /tmp/uv-venv/lib/python3.13/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->huggingface_hub>=0.21.0->accelerate) (0.16.0)\nCollecting setuptools<82 (from torch>=2.1.2->liger-kernel)\n Downloading setuptools-81.0.0-py3-none-any.whl.metadata (6.6 kB)\nRequirement already satisfied: sympy>=1.13.3 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (1.14.0)\nRequirement already satisfied: networkx>=2.5.1 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (3.6.1)\nRequirement already satisfied: jinja2 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (3.1.6)\nRequirement already satisfied: cuda-toolkit==13.0.2 in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (13.0.2)\nRequirement already satisfied: cuda-bindings<14,>=13.0.3 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (13.3.1)\nRequirement already satisfied: nvidia-cudnn-cu13==9.19.0.56 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (9.19.0.56)\nRequirement already satisfied: nvidia-cusparselt-cu13==0.8.0 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (0.8.0)\nRequirement already satisfied: nvidia-nccl-cu13==2.28.9 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (2.28.9)\nRequirement already satisfied: nvidia-nvshmem-cu13==3.4.5 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (3.4.5)\nCollecting triton\n Downloading triton-3.6.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl.metadata (1.7 kB)\nRequirement already satisfied: nvidia-cublas==13.1.0.3.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (13.1.0.3)\nRequirement already satisfied: nvidia-cuda-runtime==13.0.96.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (13.0.96)\nRequirement already satisfied: nvidia-cufft==12.0.0.61.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (12.0.0.61)\nRequirement already satisfied: nvidia-cufile==1.15.1.6.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (1.15.1.6)\nRequirement already satisfied: nvidia-cuda-cupti==13.0.85.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (13.0.85)\nRequirement already satisfied: nvidia-curand==10.4.0.35.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (10.4.0.35)\nRequirement already satisfied: nvidia-cusolver==12.0.4.66.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (12.0.4.66)\nRequirement already satisfied: nvidia-cusparse==12.6.3.3.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (12.6.3.3)\nRequirement already satisfied: nvidia-nvjitlink==13.0.88.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (13.0.88)\nRequirement already satisfied: nvidia-cuda-nvrtc==13.0.88.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (13.0.88)\nRequirement already satisfied: nvidia-nvtx==13.0.85.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (13.0.85)\nRequirement already satisfied: cuda-pathfinder>=1.4.2 in /usr/local/lib/python3.13/site-packages (from cuda-bindings<14,>=13.0.3->torch>=2.1.2->liger-kernel) (1.5.6)\nRequirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.13/site-packages (from sympy>=1.13.3->torch>=2.1.2->liger-kernel) (1.3.0)\nRequirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.13/site-packages (from jinja2->torch>=2.1.2->liger-kernel) (3.0.3)\nDownloading accelerate-1.15.0-py3-none-any.whl (394 kB)\nDownloading triton-3.6.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (188.3 MB)\n \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 188.3/188.3 MB 41.1 MB/s 0:00:04\nDownloading setuptools-81.0.0-py3-none-any.whl (1.1 MB)\n \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 1.1/1.1 MB 192.5 MB/s 0:00:00\nInstalling collected packages: triton, setuptools, accelerate\n Attempting uninstall: triton\n Found existing installation: triton 3.8.0\n Uninstalling triton-3.8.0:\n Successfully uninstalled triton-3.8.0\n Attempting uninstall: setuptools\n Found existing installation: setuptools 84.0.0\n Uninstalling setuptools-84.0.0:\n Successfully uninstalled setuptools-84.0.0\n\nSuccessfully installed accelerate-1.15.0 setuptools-81.0.0 triton-3.6.0\n",
"mimetype": "text/plain"
},
{
"type": "stream",
"name": "stderr",
"text": "WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\n\n[notice] A new release of pip is available: 25.3 -> 26.2.1\n[notice] To update, run: pip install --upgrade pip\n",
"mimetype": "text/plain"
}
]
},
{
"id": "Pvdt",
"code_hash": "b23d6205e0dff43a7a9b1cab8bd8e97a",
"outputs": [],
"console": []
},
{
"id": "ZBYS",
"code_hash": "f103139918c160f7b8011d902c51c124",
"outputs": [],
"console": []
},
{
"id": "aLJB",
"code_hash": "2cffa5c83d39aafb59ce2c1c668a56e4",
"outputs": [
{
"type": "data",
"data": {
"text/html": "<pre class='text-xs'>0</pre>"
}
}
],
"console": [
{
"type": "stream",
"name": "stderr",
"text": "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n",
"mimetype": "text/plain"
},
{
"type": "stream",
"name": "stdout",
"text": "\u26a0 ftfy not installed \u2014 mojibake repair skipped. Run `pip install ftfy` for full encoding cleanup.\n\u26a0 No HF_TOKEN found. Proceeding unauthenticated \u2014 downloads may be slow/throttled. Add HF_TOKEN to Colab Secrets (or set the env var) for faster, more reliable downloads. Note: bigcode/the-stack-v2 REQUIRES authentication + data-access agreement at https://huggingface.co/bigcode/the-stack-v2\nDownloading LittleCurriculum, target 1000.0 MB -> ./training_data/littlecurriculum_1000mb.txt\n\u2713 ./training_data/littlecurriculum_1000mb.txt | 1000.0/1000.0 MB (100.0%) | 766,481 lines | 22,235 filtered | 25 dupes dropped\n \u2713 verified: 200,000 lines sampled, 0 artifacts\nDownloading SmolLM-Corpus / cosmopedia-v2, target 2500.0 MB -> ./training_data/smollm_cosmopedia_v2_2500mb.txt\n\u2713 ./training_data/smollm_cosmopedia_v2_2500mb.txt | 2500.0/2500.0 MB (100.0%) | 704,490 lines | 8,003 filtered | 2,010 dupes dropped\n \u2713 verified: 200,000 lines sampled, 0 artifacts\nTrying TinyStories source: roneneldan/TinyStories\nDownloading TinyStories (roneneldan/TinyStories), target 1000.0 MB -> ./training_data/tinystories_1000mb.txt\n\u2713 ./training_data/tinystories_1000mb.txt | 1000.0/1000.0 MB (100.0%) | 1,175,331 lines | 109 filtered | 114,855 dupes dropped\n \u2713 verified: 200,000 lines sampled, 0 artifacts\n",
"mimetype": "text/plain"
}
]
},
{
"id": "nHfw",
"code_hash": "950061ed21b5034668672f8f1537fc20",
"outputs": [
{
"type": "data",
"data": {
"text/markdown": "<span class=\"markdown prose dark:prose-invert contents\"><h2 id=\"inference-script\">Inference Script</h2>\n<span class=\"paragraph\">This script loads the trained model from <code>arya_latest.pt</code> and demonstrates how to generate text using a given prompt.</span></span>"
}
}
],
"console": []
},
{
"id": "AjVT",
"code_hash": "40a0fe8617850dcbc35c1f4c092c5058",
"outputs": [
{
"type": "data",
"data": {
"text/html": "<pre class='text-xs'>0</pre>"
}
}
],
"console": [
{
"type": "stream",
"name": "stderr",
"text": "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n",
"mimetype": "text/plain"
},
{
"type": "stream",
"name": "stdout",
"text": "\u26a0 ftfy not installed \u2014 mojibake repair (e.g. '\u00ce\u00bcC' -> '\u03bcC', '\u00c3\u2014' -> '\u00d7') will be skipped. Run `pip install ftfy` for full text-encoding cleanup \u2014 this matters especially for NCERT, which has verified mojibake in the source data.\n\u26a0 No HF_TOKEN found (Colab Secrets key icon -> add secret named HF_TOKEN -> your HF write-access token -> toggle notebook access). Proceeding unauthenticated \u2014 downloads may be slow or appear to hang due to rate limiting, not actually broken.\nDownloading FineWeb-Edu (sample-10BT), target 2000.0 MB -> ./training_data/fineweb_edu_2000mb.txt\n\u2713 ./training_data/fineweb_edu_2000mb.txt | 2000.0 MB / 2000.0 MB (100.0%) | 449,095 lines kept | 8,544 filtered | 469 duplicates dropped\n \u2713 verified: 200,000 lines sampled, 0 markup/encoding artifacts\nDownloading Cosmopedia (auto_math_text), target 800.0 MB -> ./training_data/cosmopedia_800mb.txt\n\u2713 ./training_data/cosmopedia_800mb.txt | 800.0 MB / 800.0 MB (100.0%) | 323,453 lines kept | 70,585 filtered | 0 duplicates dropped\n \u2713 verified: 200,000 lines sampled, 0 markup/encoding artifacts\n",
"mimetype": "text/plain"
}
]
},
{
"id": "pHFh",
"code_hash": "2f5b1cffdecd0732a7efafa1a72d19e5",
"outputs": [],
"console": []
},
{
"id": "NCOB",
"code_hash": "e181efcf155eef21719f37511de1f32f",
"outputs": [
{
"type": "data",
"data": {
"text/markdown": "<span class=\"markdown prose dark:prose-invert contents\"></span>"
}
}
],
"console": []
},
{
"id": "aqbW",
"code_hash": "9573a4817336aebce40126886c37e8aa",
"outputs": [
{
"type": "data",
"data": {
"text/markdown": "<span class=\"markdown prose dark:prose-invert contents\"><span class=\"paragraph\">Now, let's define a function to generate text and then test it with an example prompt.</span></span>"
}
}
],
"console": []
},
{
"id": "TRpd",
"code_hash": "9541eaa3a371cb54992795cf9ea79a97",
"outputs": [],
"console": []
},
{
"id": "TXez",
"code_hash": "e2e709938b0cb769e6643c21951a1aa7",
"outputs": [],
"console": []
},
{
"id": "dNNg",
"code_hash": "9efe733301c0f8f6d66eccbd830a0937",
"outputs": [
{
"type": "data",
"data": {
"text/html": "<pre class='text-xs'>0</pre>"
}
}
],
"console": [
{
"type": "stream",
"name": "stderr",
"text": "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n",
"mimetype": "text/plain"
},
{
"type": "stream",
"name": "stdout",
"text": "\u26a0 ftfy not installed \u2014 mojibake repair (e.g. '\u00ce\u00bcC' -> '\u03bcC', '\u00c3\u2014' -> '\u00d7') will be skipped. Run `pip install ftfy` for full text-encoding cleanup \u2014 this matters especially for NCERT, which has verified mojibake in the source data.\n\u26a0 No HF_TOKEN found (Colab Secrets key icon -> add secret named HF_TOKEN -> your HF write-access token -> toggle notebook access). Proceeding unauthenticated \u2014 downloads may be slow or appear to hang due to rate limiting, not actually broken.\nDownloading FineWeb-Edu (sample-10BT), target 6000.0 MB -> ./training_data/fineweb_edu_6000mb.txt\n\u2713 ./training_data/fineweb_edu_6000mb.txt | 6000.0 MB / 6000.0 MB (100.0%) | 1,350,169 lines kept | 24,307 filtered | 6,840 duplicates dropped\n \u2713 verified: 200,000 lines sampled, 0 markup/encoding artifacts\n",
"mimetype": "text/plain"
}
]
},
{
"id": "yCnT",
"code_hash": "08809d03fe76b2df1122d25d14fe74d5",
"outputs": [],
"console": []
},
{
"id": "wlCL",
"code_hash": "13122d375dfe64b0638eed09336e688c",
"outputs": [],
"console": []
},
{
"id": "kqZH",
"code_hash": "c8fc96583ecaa55247aa6caf3e855d22",
"outputs": [],
"console": []
},
{
"id": "wAgl",
"code_hash": "d6e07408db81621dabaca73882dcacb1",
"outputs": [],
"console": []
},
{
"id": "rEll",
"code_hash": "9ea3e14a9bb177fa422b4038a8daaca4",
"outputs": [
{
"type": "data",
"data": {
"text/html": "<pre class='text-xs'>0</pre>"
}
}
],
"console": [
{
"type": "stream",
"name": "stdout",
"text": "[token_train] 5 files, 7.65 GB corpus, vocab_size=16384\n[token_train] Trained. mergeable ranks = 16383\n[token_train] tiktoken.Encoding built: n_vocab=16384, <|bos|> id=16383\n[token_train] Saved tokenizer/tokenizer.pkl\n",
"mimetype": "text/plain"
}
]
},
{
"id": "dGlV",
"code_hash": "c72b1f6f545da1a69cbaa110db3ed3da",
"outputs": [],
"console": []
},
{
"id": "SdmI",
"code_hash": "b5cb5e47bc909cfe489c2c8e8038ad4f",
"outputs": [],
"console": []
},
{
"id": "lgWD",
"code_hash": "d10c338ecd83ffd0ca2127d594ec829b",
"outputs": [],
"console": []
},
{
"id": "yOPj",
"code_hash": "dc7ae64fa54ee4cc7080f818cda4726c",
"outputs": [
{
"type": "data",
"data": {
"text/plain": ""
}
}
],
"console": []
},
{
"id": "fwwy",
"code_hash": "926909c3d75d0e4ed80b112909eecdc1",
"outputs": [
{
"type": "error",
"ename": "exception",
"evalue": "Name `HF_TOKEN` is not defined. It was expected to be defined in ",
"traceback": null
}
],
"console": [
{
"type": "stream",
"name": "stdout",
"text": "[auth] ArushBuilds (type=user)\n[repo] ready: https://huggingface.co/RuVI-AI/pragya-exp\n[preflight] 20 local file(s) match ALLOW:\n __marimo__/session/notebook.py.json\n __pycache__/tokenizer.cpython-313.pyc\n checkpoints/another_checkpoint_val_acc_15.pt\n checkpoints/arya_latest.pt\n checkpoints/simplysafer.pt\n lock.txt\n tokenizer.model\n tokenizer.py\n tokenizer.vocab\n tokenizer/tokenizer.pkl\n tokenizer/tokenizer_config.json\n tokenizer_1.model\n tokenizer_1.py\n tokenizer_1.vocab\n training_data/.token_cache/tokens_e02f67284dbe9bc8.ranges.json\n training_data/cosmopedia_800mb.txt\n training_data/fineweb_edu_2000mb.txt\n training_data/littlecurriculum_1000mb.txt\n training_data/smollm_cosmopedia_v2_2500mb.txt\n training_data/tinystories_1000mb.txt\n[upload] starting (whitelist: ['*.safetensors', '*.json', '*.model', '*.txt', '*.md', 'tokenizer*', 'tokenizer_files/**', '*.pkl', '*.pt', 'checkpoints/**'])\n",
"mimetype": "text/plain"
}
]
},
{
"id": "LJZf",
"code_hash": "35717b83aa01ab740b9698a3184aeb99",
"outputs": [
{
"type": "error",
"ename": "multiple-defs",
"evalue": "The variable 'os' was defined by another cell",
"traceback": []
}
],
"console": []
},
{
"id": "urSm",
"code_hash": "8ba9c69f5c18b12e4f905eb4e412f6e8",
"outputs": [],
"console": []
}
]
} |