Duplicate from RuVI-AI/pragya-e2

Co-authored-by: Arush Codes <ArushBuilds@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,41 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+training_data/cosmopedia_1000mb.txt filter=lfs diff=lfs merge=lfs -text
+training_data/cosmopedia_2000mb.txt filter=lfs diff=lfs merge=lfs -text
+training_data/fineweb_edu_4000mb.txt filter=lfs diff=lfs merge=lfs -text
+training_data/littlecurriculum_5000mb.txt filter=lfs diff=lfs merge=lfs -text
+training_data/smollm_cosmopedia_v2_5000mb.txt filter=lfs diff=lfs merge=lfs -text
+training_data/tinystories_1000mb.txt filter=lfs diff=lfs merge=lfs -text

__marimo__/session/notebook.py.json

@@ -0,0 +1,536 @@
+{
+  "version": "1",
+  "metadata": {
+    "marimo_version": "0.25.1",
+    "script_metadata_hash": "ff000607a504b65002f0c642ea5a638f"
+  },
+  "cells": [
+    {
+      "id": "Hbol",
+      "code_hash": "1d0db38904205bec4d6f6f6a1f6cec3e",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "MJUe",
+      "code_hash": "8c3add7f504aee86f92a5fd2a080e930",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "vblA",
+      "code_hash": "905d09ca55d5b894841a2106100436a4",
+      "outputs": [
+        {
+          "type": "error",
+          "ename": "interruption",
+          "evalue": "This cell was interrupted and needs to be re-run",
+          "traceback": []
+        }
+      ],
+      "console": [
+        {
+          "type": "stream",
+          "name": "stderr",
+          "text": "<span class=\"codehilite\"><div class=\"highlight\"><pre><span></span><span class=\"gt\">Traceback (most recent call last):</span>\n  File <span class=\"nb\">&quot;/tmp/marimo_730/__marimo__cell_vblA_.py&quot;</span>, line <span class=\"m\">3</span>, in <span class=\"n\">&lt;module&gt;</span>\n<span class=\"w\">    </span><span class=\"n\">result</span> <span class=\"o\">=</span> <span class=\"n\">subprocess</span><span class=\"o\">.</span><span class=\"n\">run</span><span class=\"p\">(</span>\n<span class=\"w\">    </span>    <span class=\"p\">[</span><span class=\"s1\">&#39;python&#39;</span><span class=\"p\">,</span> <span class=\"s1\">&#39;wikitext.py&#39;</span><span class=\"p\">,</span> <span class=\"s1\">&#39;--sources&#39;</span><span class=\"p\">,</span> <span class=\"s1\">&#39;cosmopedia&#39;</span><span class=\"p\">,</span> <span class=\"s1\">&#39;--cosmopedia-mb&#39;</span><span class=\"p\">,</span> <span class=\"s1\">&#39;1000&#39;</span><span class=\"p\">],</span>\n<span class=\"w\">    </span>    <span class=\"n\">capture_output</span><span class=\"o\">=</span><span class=\"kc\">True</span><span class=\"p\">,</span> <span class=\"n\">text</span><span class=\"o\">=</span><span class=\"kc\">True</span>\n<span class=\"w\">    </span><span class=\"p\">)</span>\n  File <span class=\"nb\">&quot;/usr/local/lib/python3.13/subprocess.py&quot;</span>, line <span class=\"m\">556</span>, in <span class=\"n\">run</span>\n<span class=\"w\">    </span><span class=\"n\">stdout</span><span class=\"p\">,</span> <span class=\"n\">stderr</span> <span class=\"o\">=</span> <span class=\"n\">process</span><span class=\"o\">.</span><span class=\"n\">communicate</span><span class=\"p\">(</span><span class=\"nb\">input</span><span class=\"p\">,</span> <span class=\"n\">timeout</span><span class=\"o\">=</span><span class=\"n\">timeout</span><span class=\"p\">)</span>\n<span class=\"w\">                     </span><span class=\"pm\">~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^</span>\n  File <span class=\"nb\">&quot;/usr/local/lib/python3.13/subprocess.py&quot;</span>, line <span class=\"m\">1222</span>, in <span class=\"n\">communicate</span>\n<span class=\"w\">    </span><span class=\"n\">stdout</span><span class=\"p\">,</span> <span class=\"n\">stderr</span> <span class=\"o\">=</span> <span class=\"bp\">self</span><span class=\"o\">.</span><span class=\"n\">_communicate</span><span class=\"p\">(</span><span class=\"nb\">input</span><span class=\"p\">,</span> <span class=\"n\">endtime</span><span class=\"p\">,</span> <span class=\"n\">timeout</span><span class=\"p\">)</span>\n<span class=\"w\">                     </span><span class=\"pm\">~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^</span>\n  File <span class=\"nb\">&quot;/usr/local/lib/python3.13/subprocess.py&quot;</span>, line <span class=\"m\">2154</span>, in <span class=\"n\">_communicate</span>\n<span class=\"w\">    </span><span class=\"n\">ready</span> <span class=\"o\">=</span> <span class=\"n\">selector</span><span class=\"o\">.</span><span class=\"n\">select</span><span class=\"p\">(</span><span class=\"n\">timeout</span><span class=\"p\">)</span>\n  File <span class=\"nb\">&quot;/usr/local/lib/python3.13/selectors.py&quot;</span>, line <span class=\"m\">398</span>, in <span class=\"n\">select</span>\n<span class=\"w\">    </span><span class=\"n\">fd_event_list</span> <span class=\"o\">=</span> <span class=\"bp\">self</span><span class=\"o\">.</span><span class=\"n\">_selector</span><span class=\"o\">.</span><span class=\"n\">poll</span><span class=\"p\">(</span><span class=\"n\">timeout</span><span class=\"p\">)</span>\n  File <span class=\"nb\">&quot;/usr/local/lib/python3.13/site-packages/marimo/_utils/signals.py&quot;</span>, line <span class=\"m\">33</span>, in <span class=\"n\">__call__</span>\n<span class=\"w\">    </span><span class=\"bp\">self</span><span class=\"o\">.</span><span class=\"n\">_handler</span><span class=\"p\">(</span><span class=\"n\">signum</span><span class=\"p\">,</span> <span class=\"n\">frame</span><span class=\"p\">)</span>\n<span class=\"w\">    </span><span class=\"pm\">~~~~~~~~~~~~~^^^^^^^^^^^^^^^</span>\n  File <span class=\"nb\">&quot;/usr/local/lib/python3.13/site-packages/marimo/_runtime/handlers.py&quot;</span>, line <span class=\"m\">74</span>, in <span class=\"n\">interrupt_handler</span>\n<span class=\"w\">    </span><span class=\"k\">raise</span> <span class=\"n\">MarimoInterrupt</span>\n<span class=\"gr\">KeyboardInterrupt</span>\n</pre></div>\n</span>",
+          "mimetype": "application/vnd.marimo+traceback"
+        }
+      ]
+    },
+    {
+      "id": "bkHC",
+      "code_hash": "3e9c348c42c7ba5ad7449c3567bcf13f",
+      "outputs": [
+        {
+          "type": "data",
+          "data": {
+            "text/plain": ""
+          }
+        }
+      ],
+      "console": [
+        {
+          "type": "stream",
+          "name": "stdout",
+          "text": "============================================================\nGPU / NVIDIA DRIVER DIAGNOSTICS\n============================================================\n\n1. Searching for nvidia-smi in common locations:\n   /usr/bin/nvidia-smi: \u2705 FOUND\n   /usr/local/bin/nvidia-smi: \u274c not found\n   /usr/sbin/nvidia-smi: \u274c not found\n   /bin/nvidia-smi: \u2705 FOUND\n   /opt/nvidia/bin/nvidia-smi: \u274c not found\n\n2. Checking /proc/driver/nvidia (kernel module):\n   \u2705 /proc/driver/nvidia EXISTS \u2014 NVIDIA kernel module is loaded\n\n3. Checking /dev/nvidia* device files:\n   \u2705 /dev/nvidia-modeset\n   \u2705 /dev/nvidia-uvm\n   \u2705 /dev/nvidia-uvm-tools\n   \u2705 /dev/nvidia1\n   \u2705 /dev/nvidiactl\n\n4. Container/VM check:\n   Not in a Docker container (no /.dockerenv)\n   cgroup info: 7:pids:/2eb5332fc32851d41d6e92fc384df745fdb0c6fa13f4fdf88ad07c56c10555a3\n6:memory:/2eb5332fc32851d41d6e92fc384df745fdb0c6fa13f4fdf88ad07c56c10555a3\n5:job:/2eb5332fc32851d41d6e92fc384df745fdb0c6fa13f4f\n\n5. Current PATH:\n   /tmp/uv-venv/bin:/tmp/uv-venv/lib/python3.13/site-packages/nvidia/cu13/bin:/usr/local/lib/python3.13/site-packages/nvidia/cu13/bin:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\n\n6. LD_LIBRARY_PATH:\n   /tmp/uv-venv/lib/python3.13/site-packages/nvidia/cu13/lib:/usr/local/lib/python3.13/site-packages/nvidia/cu13/lib:\n\n7. Searching for CUDA installations:\n   \u274c /usr/local/cuda not found\n   \u274c /usr/local/cuda-12 not found\n   \u274c /usr/local/cuda-13 not found\n   \u274c /opt/cuda not found\n\n8. Checking lspci for NVIDIA hardware:\n   \u274c lspci command not available\n\n9. PyTorch CUDA details:\n   PyTorch version: 2.11.0+cu130\n   CUDA available: True\n   CUDA version (PyTorch built with): 13.0\n   \u2705 libcuda.so is loadable\n\n============================================================\n",
+          "mimetype": "text/plain"
+        }
+      ]
+    },
+    {
+      "id": "lEQa",
+      "code_hash": "8ee678e3af92e02ec6c3e6be3ca5bd98",
+      "outputs": [
+        {
+          "type": "data",
+          "data": {
+            "text/markdown": "<span class=\"markdown prose dark:prose-invert contents\"><h2 id=\"gpu-diagnosis-summary-molab-rtx-6000-pro-blackwell\">\ud83d\udd0d GPU Diagnosis Summary \u2014 Molab RTX 6000 Pro Blackwell</h2>\n<span class=\"paragraph\">Based on the diagnostic cells in this notebook, here is what I found:</span>\n<h3 id=\"root-cause-the-gpu-is-not-being-exposed-to-this-molab-containerruntime\">Root Cause: <strong>The GPU is not being exposed to this Molab container/runtime</strong></h3>\n<table>\n<thead>\n<tr>\n<th>Check</th>\n<th>Result</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td><code>nvidia-smi</code> binary</td>\n<td>\u274c Not found anywhere on the system</td>\n</tr>\n<tr>\n<td><code>torch.cuda.is_available()</code></td>\n<td>\u274c Returns <code>False</code></td>\n</tr>\n<tr>\n<td><code>/dev/nvidia*</code> devices</td>\n<td>\u274c Not present</td>\n</tr>\n<tr>\n<td><code>/proc/driver/nvidia</code></td>\n<td>\u274c Not present (no kernel module loaded)</td>\n</tr>\n<tr>\n<td><code>future.py</code> benchmark output</td>\n<td>\u274c Ran on <code>device=cpu</code> with <code>peak_mem=0.00 MB</code></td>\n</tr>\n<tr>\n<td><code>nvidia-smi -pm 1</code> command</td>\n<td>\u274c <code>FileNotFoundError: No such file or directory: 'nvidia-smi'</code></td>\n</tr>\n</tbody>\n</table>\n<h3 id=\"key-observations\">Key Observations</h3>\n<ol>\n<li><strong>PyTorch 2.11.0 with CUDA 13.0.2 is installed</strong> \u2014 all CUDA libraries (cuDNN, cuBLAS, NCCL, etc.) are present via pip, but they are <strong>useless without the GPU being exposed to the container</strong>.</li>\n<li><strong>This is a Molab environment</strong> \u2014 the GPU should be available, but the container/runtime was not started with GPU passthrough properly configured.</li>\n<li><strong>No <code>nvidia-smi</code>, no <code>/dev/nvidia*</code>, no <code>/proc/driver/nvidia</code></strong> \u2014 this means the NVIDIA driver is not accessible inside the container at all. The GPU hardware may exist on the host, but the container cannot reach it.</li>\n<li><strong>JAX also has a CUDA plugin mismatch</strong>: <code>jax_cuda13_plugin v0.11.0</code> is incompatible with <code>jaxlib v0.11.2</code>.</li>\n</ol>\n<h3 id=\"what-needs-to-happen\">What Needs to Happen</h3>\n<span class=\"paragraph\">The problem is <strong>not</strong> with your Python scripts or PyTorch installation \u2014 it's that the <strong>Molab container does not have GPU access configured</strong>. See the fix steps below.</span></span>"
+          }
+        }
+      ],
+      "console": []
+    },
+    {
+      "id": "PKri",
+      "code_hash": "cbb92b645fbce36bb05be35b0050b993",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "Xref",
+      "code_hash": "e5c259217edc1ee4fb636163544d13f9",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "SFPL",
+      "code_hash": "759edb93690b45a7fb4fcbba25445967",
+      "outputs": [
+        {
+          "type": "data",
+          "data": {
+            "text/plain": ""
+          }
+        }
+      ],
+      "console": [
+        {
+          "type": "stream",
+          "name": "stdout",
+          "text": "python: 3.13.11 /tmp/uv-venv/bin/python\ntorch, triton: 2.11.0+cu130 3.6.0\nhelion: Version: 1.5.1\n",
+          "mimetype": "text/plain"
+        }
+      ]
+    },
+    {
+      "id": "BYtC",
+      "code_hash": "66ea21b2bc2ce38743edd9613eac2d87",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "RGSE",
+      "code_hash": "6470f3209ebb8c510b3f5b2f231bfc62",
+      "outputs": [
+        {
+          "type": "data",
+          "data": {
+            "text/html": "<pre class='text-xs'>0</pre>"
+          }
+        }
+      ],
+      "console": [
+        {
+          "type": "stream",
+          "name": "stderr",
+          "text": "/tmp/uv-venv/lib/python3.13/site-packages/torchao/quantization/quant_api.py:1558: SyntaxWarning: invalid escape sequence '\\.'\n  * regex for parameter names, must start with `re:`, e.g. `re:language\\.layers\\..+\\.q_proj.weight`.\n",
+          "mimetype": "text/plain"
+        },
+        {
+          "type": "stream",
+          "name": "stdout",
+          "text": "[Tok] Found 6 file(s) in ./training_data\n[Tok] Fingerprint: 83a770868891c636  (document_boundaries=True)\n[Tok] Tokenizing with 6 worker(s) ...\n[Tok] 1/6 files done | 13,452,009 tokens | 6.8s\n[Tok] 2/6 files done | 270,323,821 tokens | 99.7s\n[Tok] 3/6 files done | 770,143,654 tokens | 188.4s\n[Tok] 4/6 files done | 1,779,412,071 tokens | 371.8s\n[Tok] 5/6 files done | 2,873,479,424 tokens | 428.4s\n[Tok] 6/6 files done | 4,179,431,280 tokens | 481.8s\n[Tok] Assembling 4,179,431,280 tokens into ./training_data/.token_cache/tokens_83a770868891c636.tmp.npy ...\n[Tok] Ranges sidecar -> ./training_data/.token_cache/tokens_83a770868891c636.ranges.json  (6 entries)\n[Tok] Done in 536.6s | 4,179,431,280 tokens | 16.72 GB\n[Tok] Saved -> ./training_data/.token_cache/tokens_83a770868891c636.npy\n./training_data/.token_cache/tokens_83a770868891c636.npy\n",
+          "mimetype": "text/plain"
+        }
+      ]
+    },
+    {
+      "id": "Kclp",
+      "code_hash": "5a157233f59154540805237fc3f714af",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "emfo",
+      "code_hash": "16f7001224229d5030ea9ad7077bab8a",
+      "outputs": [
+        {
+          "type": "data",
+          "data": {
+            "text/html": "<pre class='text-xs'>0</pre>"
+          }
+        }
+      ],
+      "console": [
+        {
+          "type": "stream",
+          "name": "stderr",
+          "text": "[RANK 0] Detected kernel version 4.19.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n",
+          "mimetype": "text/plain"
+        },
+        {
+          "type": "stream",
+          "name": "stdout",
+          "text": "[Setup] GPU compute capability 12.x (Ampere+) -- TF32 enabled for fp32 matmuls/convs.\n[Setup] Parallelism: plain data-parallel (DDP) only -- no FSDP/tensor-parallel/pipeline-parallel. Appropriate for small device counts (e.g. dual T4); set use_fsdp=True in config.py when sharding is actually needed.\n[Setup] Using device: cuda\n[Setup] Num processes (GPUs): 1\n[Tokenizer] Loaded, vocab_size=12288\n[Dataset] Found 6 files in ./training_data (curriculum OFF)\n[Dataset] Cache hit -> ./training_data/.token_cache/tokens_83a770868891c636.npy  (4,179,431,312 tokens)\n[Dataset] train=4,137,636,969 tokens, val=41,794,311 tokens\n[Val] fixed validation set: 33 batches x 8 seqs across 6 file(s) (cosmopedia_1000mb.txt:1, cosmopedia_2000mb.txt:4, fineweb_edu_4000mb.txt:8, littlecurriculum_5000mb.txt:10, smollm_cosmopedia_v2_5000mb.txt:8, tinystories_1000mb.txt:2)\n[Setup] GPU-side batching=True, num_workers=2 (from config.py)\n[Mem] 8.96/101.98 GB used after corpus+batcher on-device (91% free).\n[Engram] layers=[1, 8] max_ngram=3 compressed_vocab=9807 (raw 12288)\n[DocMask] ON: documents split at BOS-after-BOS (bos_id=12287), attention windows [None]\n[Engram] 2,532,736 engram params\n[triton_kernel] loaded in pid=33709\n[TritonAudit] triton_kernel present but unavailable -- eager path.\n[Compile] Compiling model for maximum speed...\n[Compile] reduce-overhead requested via alpha_config.ALPHA_COMPILE_MODE. model.py's lm_head now reads the tied weight via F.linear(x, self.lm_head.weight) instead of the module call, specifically to remove the aliasing ambiguity that caused the earlier 'overwritten by a subsequent run' crash at wte(idx). This is a plausible fix, NOT a verified one -- it hasn't been run against the actual crash on real hardware. VERIFY before trusting a real run: run a few hundred steps, and if val_loss tracks normally (compare against a mode=default run on the same data/seed) and there's no CUDAGraphs RuntimeError, it's confirmed safe. If loss silently diverges or looks wrong without an explicit crash, that's worse than a crash -- stop and revert to mode=default immediately.\n[Compile/diag] reduce-overhead active. reserved_bytes.all.current=8104.0 MiB, num_alloc_retries=0 -- watch this grow abnormally across the first few steps as a sign of multiple cudagraph pools being captured (one per accumulate()/sync branch).\n[Model] Total params: 23.78M\n[Checkpoint] No checkpoint found, starting fresh\nstep      0 | epoch 0.00 | loss 9.4869 | ce 9.4869 | lr 4.00e-06 | tok/s 3,090 | elapsed 1.8min\nstep     50 | epoch 0.00 | loss 8.8555 | ce 8.1037 | lr 1.04e-04 | tok/s 363,404 | elapsed 2.5min\nstep    100 | epoch 0.01 | loss 7.2889 | ce 6.5953 | lr 2.04e-04 | tok/s 517,146 | elapsed 3.0min\nstep    150 | epoch 0.01 | loss 6.0934 | ce 5.7234 | lr 3.04e-04 | tok/s 516,373 | elapsed 3.6min\nstep    200 | epoch 0.02 | loss 5.3833 | ce 5.1494 | lr 4.04e-04 | tok/s 516,285 | elapsed 4.1min\nstep    250 | epoch 0.02 | loss 4.9218 | ce 4.7564 | lr 5.04e-04 | tok/s 516,971 | elapsed 4.6min\nstep    300 | epoch 0.02 | loss 4.6083 | ce 4.5052 | lr 6.04e-04 | tok/s 517,010 | elapsed 5.2min\nstep    350 | epoch 0.03 | loss 4.3647 | ce 4.2413 | lr 7.04e-04 | tok/s 516,763 | elapsed 5.7min\nstep    400 | epoch 0.03 | loss 4.1559 | ce 4.0846 | lr 8.04e-04 | tok/s 516,680 | elapsed 6.2min\nstep    450 | epoch 0.04 | loss 3.9629 | ce 3.9028 | lr 9.04e-04 | tok/s 516,042 | elapsed 6.7min\nstep    500 | epoch 0.04 | loss 3.8155 | ce 3.7532 | lr 1.00e-03 | tok/s 515,703 | elapsed 7.3min\nstep    550 | epoch 0.04 | loss 3.7135 | ce 3.6901 | lr 1.00e-03 | tok/s 515,143 | elapsed 7.8min\nstep    600 | epoch 0.05 | loss 3.6035 | ce 3.5385 | lr 1.00e-03 | tok/s 515,441 | elapsed 8.3min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 601)\nstep    650 | epoch 0.05 | loss 3.5292 | ce 3.5196 | lr 1.00e-03 | tok/s 511,588 | elapsed 8.9min\nstep    700 | epoch 0.06 | loss 3.4675 | ce 3.4325 | lr 1.00e-03 | tok/s 514,574 | elapsed 9.4min\nstep    750 | epoch 0.06 | loss 3.4103 | ce 3.3885 | lr 1.00e-03 | tok/s 514,694 | elapsed 9.9min\nstep    800 | epoch 0.06 | loss 3.3629 | ce 3.3134 | lr 1.00e-03 | tok/s 514,146 | elapsed 10.5min\nstep    850 | epoch 0.07 | loss 3.3342 | ce 3.3325 | lr 1.00e-03 | tok/s 514,010 | elapsed 11.0min\nstep    900 | epoch 0.07 | loss 3.3064 | ce 3.2679 | lr 1.00e-03 | tok/s 513,729 | elapsed 11.5min\nstep    950 | epoch 0.08 | loss 3.2726 | ce 3.2956 | lr 1.00e-03 | tok/s 513,731 | elapsed 12.1min\nstep   1000 | epoch 0.08 | loss 3.2517 | ce 3.1991 | lr 1.00e-03 | tok/s 513,663 | elapsed 12.6min\nstep   1000 | val_loss 3.2163 | val_acc 0.3935 | val_ppl 24.93\nstep   1050 | epoch 0.08 | loss 3.2301 | ce 3.2228 | lr 1.00e-03 | tok/s 477,306 | elapsed 13.2min\nstep   1100 | epoch 0.09 | loss 3.2151 | ce 3.1909 | lr 1.00e-03 | tok/s 513,867 | elapsed 13.7min\nstep   1150 | epoch 0.09 | loss 3.1891 | ce 3.2047 | lr 1.00e-03 | tok/s 513,457 | elapsed 14.2min\nstep   1200 | epoch 0.10 | loss 3.1760 | ce 3.2097 | lr 1.00e-03 | tok/s 513,740 | elapsed 14.8min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 1201)\nstep   1250 | epoch 0.10 | loss 3.1586 | ce 3.1653 | lr 1.00e-03 | tok/s 512,629 | elapsed 15.3min\nstep   1300 | epoch 0.10 | loss 3.1458 | ce 3.1605 | lr 1.00e-03 | tok/s 513,016 | elapsed 15.8min\nstep   1350 | epoch 0.11 | loss 3.1362 | ce 3.1297 | lr 1.00e-03 | tok/s 512,783 | elapsed 16.4min\nstep   1400 | epoch 0.11 | loss 3.1251 | ce 3.0709 | lr 1.00e-03 | tok/s 513,109 | elapsed 16.9min\nstep   1450 | epoch 0.11 | loss 3.1056 | ce 3.0761 | lr 1.00e-03 | tok/s 513,195 | elapsed 17.4min\nstep   1500 | epoch 0.12 | loss 3.1029 | ce 3.0663 | lr 1.00e-03 | tok/s 513,355 | elapsed 17.9min\nstep   1550 | epoch 0.12 | loss 3.0933 | ce 3.1185 | lr 1.00e-03 | tok/s 513,100 | elapsed 18.5min\nstep   1600 | epoch 0.13 | loss 3.0842 | ce 3.0943 | lr 1.00e-03 | tok/s 512,880 | elapsed 19.0min\nstep   1650 | epoch 0.13 | loss 3.0826 | ce 3.0913 | lr 1.00e-03 | tok/s 513,302 | elapsed 19.5min\nstep   1700 | epoch 0.13 | loss 3.0677 | ce 3.0796 | lr 1.00e-03 | tok/s 512,821 | elapsed 20.1min\nstep   1750 | epoch 0.14 | loss 3.0537 | ce 3.0562 | lr 1.00e-03 | tok/s 513,053 | elapsed 20.6min\nstep   1800 | epoch 0.14 | loss 3.0535 | ce 3.0498 | lr 1.00e-03 | tok/s 513,303 | elapsed 21.1min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 1801)\nstep   1850 | epoch 0.15 | loss 3.0419 | ce 3.0738 | lr 1.00e-03 | tok/s 513,070 | elapsed 21.7min\nstep   1900 | epoch 0.15 | loss 3.0397 | ce 3.0310 | lr 1.00e-03 | tok/s 513,479 | elapsed 22.2min\nstep   1950 | epoch 0.15 | loss 3.0311 | ce 3.0196 | lr 1.00e-03 | tok/s 513,245 | elapsed 22.7min\nstep   2000 | epoch 0.16 | loss 3.0267 | ce 3.0684 | lr 1.00e-03 | tok/s 513,926 | elapsed 23.3min\nstep   2000 | val_loss 2.9920 | val_acc 0.4184 | val_ppl 19.93\nstep   2050 | epoch 0.16 | loss 3.0062 | ce 2.9481 | lr 1.00e-03 | tok/s 455,770 | elapsed 23.9min\nstep   2100 | epoch 0.17 | loss 3.0144 | ce 2.9917 | lr 1.00e-03 | tok/s 512,501 | elapsed 24.4min\nstep   2150 | epoch 0.17 | loss 3.0063 | ce 2.9750 | lr 1.00e-03 | tok/s 512,627 | elapsed 24.9min\nstep   2200 | epoch 0.17 | loss 3.0003 | ce 3.0238 | lr 1.00e-03 | tok/s 512,296 | elapsed 25.5min\nstep   2250 | epoch 0.18 | loss 2.9955 | ce 3.0522 | lr 1.00e-03 | tok/s 513,161 | elapsed 26.0min\nstep   2300 | epoch 0.18 | loss 2.9891 | ce 2.9937 | lr 1.00e-03 | tok/s 512,946 | elapsed 26.5min\nstep   2350 | epoch 0.19 | loss 2.9868 | ce 3.0349 | lr 1.00e-03 | tok/s 513,538 | elapsed 27.1min\nstep   2400 | epoch 0.19 | loss 2.9853 | ce 2.9938 | lr 1.00e-03 | tok/s 514,073 | elapsed 27.6min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 2401)\nstep   2450 | epoch 0.19 | loss 2.9859 | ce 2.9862 | lr 1.00e-03 | tok/s 512,995 | elapsed 28.1min\nstep   2500 | epoch 0.20 | loss 2.9795 | ce 2.9690 | lr 1.00e-03 | tok/s 514,757 | elapsed 28.7min\nstep   2550 | epoch 0.20 | loss 2.9669 | ce 3.0139 | lr 1.00e-03 | tok/s 515,337 | elapsed 29.2min\nstep   2600 | epoch 0.21 | loss 2.9701 | ce 2.9526 | lr 1.00e-03 | tok/s 514,861 | elapsed 29.7min\nstep   2650 | epoch 0.21 | loss 2.9582 | ce 2.9649 | lr 1.00e-03 | tok/s 514,656 | elapsed 30.2min\nstep   2700 | epoch 0.21 | loss 2.9663 | ce 2.9320 | lr 1.00e-03 | tok/s 514,338 | elapsed 30.8min\nstep   2750 | epoch 0.22 | loss 2.9491 | ce 2.9213 | lr 1.00e-03 | tok/s 514,772 | elapsed 31.3min\nstep   2800 | epoch 0.22 | loss 2.9532 | ce 2.9117 | lr 1.00e-03 | tok/s 514,600 | elapsed 31.8min\nstep   2850 | epoch 0.23 | loss 2.9520 | ce 2.9820 | lr 1.00e-03 | tok/s 514,801 | elapsed 32.4min\nstep   2900 | epoch 0.23 | loss 2.9492 | ce 2.9720 | lr 1.00e-03 | tok/s 514,757 | elapsed 32.9min\nstep   2950 | epoch 0.23 | loss 2.9453 | ce 2.9705 | lr 1.00e-03 | tok/s 515,305 | elapsed 33.4min\nstep   3000 | epoch 0.24 | loss 2.9349 | ce 2.9573 | lr 1.00e-03 | tok/s 514,309 | elapsed 34.0min\nstep   3000 | val_loss 2.9078 | val_acc 0.4289 | val_ppl 18.32\n[Export] step 3001: 194 tensors (+1 aliased) -> ./checkpoints/model_step3001.safetensors (114.1 MB, 3.28s)\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 3001)\nstep   3050 | epoch 0.24 | loss 2.9406 | ce 2.9532 | lr 1.00e-03 | tok/s 457,204 | elapsed 34.6min\nstep   3100 | epoch 0.25 | loss 2.9372 | ce 2.8889 | lr 1.00e-03 | tok/s 514,810 | elapsed 35.1min\nstep   3150 | epoch 0.25 | loss 2.9347 | ce 2.9377 | lr 1.00e-03 | tok/s 515,096 | elapsed 35.6min\nstep   3200 | epoch 0.25 | loss 2.9202 | ce 2.9010 | lr 1.00e-03 | tok/s 513,595 | elapsed 36.2min\nstep   3250 | epoch 0.26 | loss 2.9277 | ce 2.9291 | lr 1.00e-03 | tok/s 513,924 | elapsed 36.7min\nstep   3300 | epoch 0.26 | loss 2.9218 | ce 2.9293 | lr 1.00e-03 | tok/s 513,359 | elapsed 37.2min\nstep   3350 | epoch 0.27 | loss 2.9181 | ce 2.9201 | lr 1.00e-03 | tok/s 513,195 | elapsed 37.7min\nstep   3400 | epoch 0.27 | loss 2.9123 | ce 2.9581 | lr 1.00e-03 | tok/s 513,384 | elapsed 38.3min\nstep   3450 | epoch 0.27 | loss 2.9138 | ce 2.9242 | lr 1.00e-03 | tok/s 513,341 | elapsed 38.8min\nstep   3500 | epoch 0.28 | loss 2.9156 | ce 2.8908 | lr 1.00e-03 | tok/s 513,644 | elapsed 39.3min\nstep   3550 | epoch 0.28 | loss 2.9005 | ce 2.9377 | lr 1.00e-03 | tok/s 514,124 | elapsed 39.9min\nstep   3600 | epoch 0.29 | loss 2.8988 | ce 2.9116 | lr 1.00e-03 | tok/s 514,246 | elapsed 40.4min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 3601)\nstep   3650 | epoch 0.29 | loss 2.9020 | ce 2.8569 | lr 1.00e-03 | tok/s 514,059 | elapsed 40.9min\nstep   3700 | epoch 0.29 | loss 2.8989 | ce 2.8479 | lr 1.00e-03 | tok/s 514,305 | elapsed 41.5min\nstep   3750 | epoch 0.30 | loss 2.8980 | ce 2.9182 | lr 1.00e-03 | tok/s 514,668 | elapsed 42.0min\nstep   3800 | epoch 0.30 | loss 2.8944 | ce 2.8939 | lr 1.00e-03 | tok/s 513,631 | elapsed 42.5min\nstep   3850 | epoch 0.30 | loss 2.8902 | ce 2.8974 | lr 1.00e-03 | tok/s 513,858 | elapsed 43.1min\nstep   3900 | epoch 0.31 | loss 2.8897 | ce 2.8891 | lr 1.00e-03 | tok/s 514,406 | elapsed 43.6min\nstep   3950 | epoch 0.31 | loss 2.9010 | ce 2.8211 | lr 1.00e-03 | tok/s 514,972 | elapsed 44.1min\nstep   4000 | epoch 0.32 | loss 2.8922 | ce 2.9309 | lr 1.00e-03 | tok/s 514,200 | elapsed 44.7min\nstep   4000 | val_loss 2.8607 | val_acc 0.4348 | val_ppl 17.47\nstep   4050 | epoch 0.32 | loss 2.8933 | ce 2.8909 | lr 1.00e-03 | tok/s 504,264 | elapsed 45.2min\nstep   4100 | epoch 0.32 | loss 2.8866 | ce 2.8892 | lr 1.00e-03 | tok/s 513,817 | elapsed 45.7min\nstep   4150 | epoch 0.33 | loss 2.8814 | ce 2.8858 | lr 1.00e-03 | tok/s 512,576 | elapsed 46.3min\nstep   4200 | epoch 0.33 | loss 2.8874 | ce 2.8429 | lr 1.00e-03 | tok/s 513,298 | elapsed 46.8min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 4201)\nstep   4250 | epoch 0.34 | loss 2.8844 | ce 2.8381 | lr 1.00e-03 | tok/s 513,454 | elapsed 47.3min\nstep   4300 | epoch 0.34 | loss 2.8810 | ce 2.8731 | lr 1.00e-03 | tok/s 514,234 | elapsed 47.9min\nstep   4350 | epoch 0.34 | loss 2.8781 | ce 2.8870 | lr 1.00e-03 | tok/s 514,587 | elapsed 48.4min\nstep   4400 | epoch 0.35 | loss 2.8729 | ce 2.8459 | lr 1.00e-03 | tok/s 514,566 | elapsed 48.9min\nstep   4450 | epoch 0.35 | loss 2.8794 | ce 2.8447 | lr 1.00e-03 | tok/s 514,941 | elapsed 49.4min\nstep   4500 | epoch 0.36 | loss 2.8737 | ce 2.8227 | lr 1.00e-03 | tok/s 515,259 | elapsed 50.0min\nstep   4550 | epoch 0.36 | loss 2.8658 | ce 2.8771 | lr 1.00e-03 | tok/s 515,573 | elapsed 50.5min\nstep   4600 | epoch 0.36 | loss 2.8642 | ce 2.8583 | lr 1.00e-03 | tok/s 515,175 | elapsed 51.0min\nstep   4650 | epoch 0.37 | loss 2.8741 | ce 2.8572 | lr 1.00e-03 | tok/s 514,435 | elapsed 51.6min\nstep   4700 | epoch 0.37 | loss 2.8684 | ce 2.8927 | lr 1.00e-03 | tok/s 514,137 | elapsed 52.1min\nstep   4750 | epoch 0.38 | loss 2.8647 | ce 2.9059 | lr 1.00e-03 | tok/s 512,987 | elapsed 52.6min\nstep   4800 | epoch 0.38 | loss 2.8578 | ce 2.7831 | lr 1.00e-03 | tok/s 513,432 | elapsed 53.2min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 4801)\nstep   4850 | epoch 0.38 | loss 2.8573 | ce 2.8763 | lr 1.00e-03 | tok/s 513,139 | elapsed 53.7min\nstep   4900 | epoch 0.39 | loss 2.8544 | ce 2.8666 | lr 1.00e-03 | tok/s 513,979 | elapsed 54.2min\nstep   4950 | epoch 0.39 | loss 2.8611 | ce 2.8346 | lr 1.00e-03 | tok/s 514,630 | elapsed 54.8min\nstep   5000 | epoch 0.40 | loss 2.8557 | ce 2.7888 | lr 1.00e-03 | tok/s 514,745 | elapsed 55.3min\nstep   5000 | val_loss 2.8249 | val_acc 0.4398 | val_ppl 16.86\nstep   5050 | epoch 0.40 | loss 2.8517 | ce 2.8615 | lr 1.00e-03 | tok/s 505,257 | elapsed 55.8min\nstep   5100 | epoch 0.40 | loss 2.8497 | ce 2.8364 | lr 1.00e-03 | tok/s 514,402 | elapsed 56.4min\nstep   5150 | epoch 0.41 | loss 2.8654 | ce 2.8043 | lr 1.00e-03 | tok/s 514,333 | elapsed 56.9min\nstep   5200 | epoch 0.41 | loss 2.8476 | ce 2.8571 | lr 1.00e-03 | tok/s 513,244 | elapsed 57.4min\nstep   5250 | epoch 0.42 | loss 2.8487 | ce 2.8798 | lr 1.00e-03 | tok/s 513,812 | elapsed 58.0min\nstep   5300 | epoch 0.42 | loss 2.8485 | ce 2.8765 | lr 1.00e-03 | tok/s 513,331 | elapsed 58.5min\nstep   5350 | epoch 0.42 | loss 2.8459 | ce 2.8483 | lr 1.00e-03 | tok/s 513,239 | elapsed 59.0min\nstep   5400 | epoch 0.43 | loss 2.8492 | ce 2.8641 | lr 1.00e-03 | tok/s 513,034 | elapsed 59.5min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 5401)\nstep   5450 | epoch 0.43 | loss 2.8458 | ce 2.8819 | lr 1.00e-03 | tok/s 512,973 | elapsed 60.1min\nstep   5500 | epoch 0.44 | loss 2.8520 | ce 2.8339 | lr 1.00e-03 | tok/s 512,998 | elapsed 60.6min\nstep   5550 | epoch 0.44 | loss 2.8306 | ce 2.7871 | lr 1.00e-03 | tok/s 513,057 | elapsed 61.1min\nstep   5600 | epoch 0.44 | loss 2.8373 | ce 2.8232 | lr 1.00e-03 | tok/s 513,432 | elapsed 61.7min\nstep   5650 | epoch 0.45 | loss 2.8383 | ce 2.8451 | lr 1.00e-03 | tok/s 513,824 | elapsed 62.2min\nstep   5700 | epoch 0.45 | loss 2.8464 | ce 2.8253 | lr 1.00e-03 | tok/s 514,010 | elapsed 62.7min\nstep   5750 | epoch 0.46 | loss 2.8388 | ce 2.7875 | lr 1.00e-03 | tok/s 514,465 | elapsed 63.3min\nstep   5800 | epoch 0.46 | loss 2.8363 | ce 2.8168 | lr 1.00e-03 | tok/s 513,322 | elapsed 63.8min\nstep   5850 | epoch 0.46 | loss 2.8344 | ce 2.8476 | lr 1.00e-03 | tok/s 513,463 | elapsed 64.3min\nstep   5900 | epoch 0.47 | loss 2.8436 | ce 2.8406 | lr 1.00e-03 | tok/s 513,127 | elapsed 64.9min\nstep   5950 | epoch 0.47 | loss 2.8325 | ce 2.8527 | lr 1.00e-03 | tok/s 512,820 | elapsed 65.4min\nstep   6000 | epoch 0.48 | loss 2.8375 | ce 2.8413 | lr 1.00e-03 | tok/s 513,382 | elapsed 65.9min\nstep   6000 | val_loss 2.8020 | val_acc 0.4433 | val_ppl 16.48\n[Export] step 6001: 194 tensors (+1 aliased) -> ./checkpoints/model_step6001.safetensors (114.1 MB, 0.05s)\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 6001)\nstep   6050 | epoch 0.48 | loss 2.8403 | ce 2.8863 | lr 1.00e-03 | tok/s 501,429 | elapsed 66.5min\nstep   6100 | epoch 0.48 | loss 2.8272 | ce 2.8477 | lr 1.00e-03 | tok/s 513,534 | elapsed 67.0min\nstep   6150 | epoch 0.49 | loss 2.8255 | ce 2.8429 | lr 1.00e-03 | tok/s 512,617 | elapsed 67.5min\nstep   6200 | epoch 0.49 | loss 2.8352 | ce 2.8033 | lr 1.00e-03 | tok/s 512,301 | elapsed 68.1min\nstep   6250 | epoch 0.49 | loss 2.8309 | ce 2.8363 | lr 1.00e-03 | tok/s 512,582 | elapsed 68.6min\nstep   6300 | epoch 0.50 | loss 2.8245 | ce 2.7591 | lr 1.00e-03 | tok/s 513,094 | elapsed 69.1min\nstep   6350 | epoch 0.50 | loss 2.8247 | ce 2.8357 | lr 1.00e-03 | tok/s 512,194 | elapsed 69.7min\nstep   6400 | epoch 0.51 | loss 2.8226 | ce 2.7979 | lr 1.00e-03 | tok/s 512,077 | elapsed 70.2min\nstep   6450 | epoch 0.51 | loss 2.8313 | ce 2.8595 | lr 1.00e-03 | tok/s 512,230 | elapsed 70.7min\nstep   6500 | epoch 0.51 | loss 2.8217 | ce 2.7978 | lr 1.00e-03 | tok/s 512,269 | elapsed 71.3min\nstep   6550 | epoch 0.52 | loss 2.8229 | ce 2.8457 | lr 1.00e-03 | tok/s 512,354 | elapsed 71.8min\nstep   6600 | epoch 0.52 | loss 2.8237 | ce 2.8690 | lr 1.00e-03 | tok/s 512,522 | elapsed 72.3min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 6601)\nstep   6650 | epoch 0.53 | loss 2.8218 | ce 2.8206 | lr 1.00e-03 | tok/s 512,178 | elapsed 72.9min\nstep   6700 | epoch 0.53 | loss 2.8174 | ce 2.8573 | lr 1.00e-03 | tok/s 512,334 | elapsed 73.4min\nstep   6750 | epoch 0.53 | loss 2.8150 | ce 2.8370 | lr 1.00e-03 | tok/s 512,728 | elapsed 73.9min\nstep   6800 | epoch 0.54 | loss 2.8180 | ce 2.8580 | lr 1.00e-03 | tok/s 513,022 | elapsed 74.5min\nstep   6850 | epoch 0.54 | loss 2.8281 | ce 2.8132 | lr 1.00e-03 | tok/s 512,968 | elapsed 75.0min\nstep   6900 | epoch 0.55 | loss 2.8178 | ce 2.8154 | lr 1.00e-03 | tok/s 512,240 | elapsed 75.5min\nstep   6950 | epoch 0.55 | loss 2.8207 | ce 2.8321 | lr 1.00e-03 | tok/s 513,304 | elapsed 76.1min\nstep   7000 | epoch 0.55 | loss 2.8173 | ce 2.8385 | lr 1.00e-03 | tok/s 513,304 | elapsed 76.6min\nstep   7000 | val_loss 2.7840 | val_acc 0.4452 | val_ppl 16.18\nstep   7050 | epoch 0.56 | loss 2.8097 | ce 2.8206 | lr 1.00e-03 | tok/s 504,197 | elapsed 77.1min\nstep   7100 | epoch 0.56 | loss 2.8195 | ce 2.7813 | lr 1.00e-03 | tok/s 513,330 | elapsed 77.7min\nstep   7150 | epoch 0.57 | loss 2.8144 | ce 2.8892 | lr 1.00e-03 | tok/s 512,068 | elapsed 78.2min\nstep   7200 | epoch 0.57 | loss 2.8069 | ce 2.7879 | lr 1.00e-03 | tok/s 512,898 | elapsed 78.7min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 7201)\nstep   7250 | epoch 0.57 | loss 2.8141 | ce 2.7636 | lr 1.00e-03 | tok/s 512,311 | elapsed 79.3min\nstep   7300 | epoch 0.58 | loss 2.8097 | ce 2.8011 | lr 1.00e-03 | tok/s 512,491 | elapsed 79.8min\nstep   7350 | epoch 0.58 | loss 2.8209 | ce 2.8291 | lr 1.00e-03 | tok/s 512,155 | elapsed 80.3min\nstep   7400 | epoch 0.59 | loss 2.8133 | ce 2.7726 | lr 1.00e-03 | tok/s 512,242 | elapsed 80.9min\nstep   7450 | epoch 0.59 | loss 2.8112 | ce 2.8194 | lr 1.00e-03 | tok/s 512,583 | elapsed 81.4min\nstep   7500 | epoch 0.59 | loss 2.8085 | ce 2.7988 | lr 1.00e-03 | tok/s 512,421 | elapsed 81.9min\nstep   7550 | epoch 0.60 | loss 2.8048 | ce 2.7982 | lr 1.00e-03 | tok/s 512,031 | elapsed 82.5min\nstep   7600 | epoch 0.60 | loss 2.8135 | ce 2.8221 | lr 1.00e-03 | tok/s 512,657 | elapsed 83.0min\nstep   7650 | epoch 0.61 | loss 2.8040 | ce 2.7481 | lr 1.00e-03 | tok/s 512,617 | elapsed 83.5min\nstep   7700 | epoch 0.61 | loss 2.8124 | ce 2.7929 | lr 1.00e-03 | tok/s 512,612 | elapsed 84.1min\nstep   7750 | epoch 0.61 | loss 2.8025 | ce 2.7220 | lr 1.00e-03 | tok/s 512,839 | elapsed 84.6min\nstep   7800 | epoch 0.62 | loss 2.8077 | ce 2.8605 | lr 1.00e-03 | tok/s 513,316 | elapsed 85.1min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 7801)\nstep   7850 | epoch 0.62 | loss 2.8094 | ce 2.8137 | lr 1.00e-03 | tok/s 513,446 | elapsed 85.7min\nstep   7900 | epoch 0.63 | loss 2.8070 | ce 2.7868 | lr 1.00e-03 | tok/s 512,876 | elapsed 86.2min\nstep   7950 | epoch 0.63 | loss 2.8051 | ce 2.8221 | lr 1.00e-03 | tok/s 512,277 | elapsed 86.7min\nstep   8000 | epoch 0.63 | loss 2.7930 | ce 2.7904 | lr 1.00e-03 | tok/s 512,547 | elapsed 87.3min\nstep   8000 | val_loss 2.7712 | val_acc 0.4467 | val_ppl 15.98\nstep   8050 | epoch 0.64 | loss 2.8000 | ce 2.8436 | lr 1.00e-03 | tok/s 503,450 | elapsed 87.8min\nstep   8100 | epoch 0.64 | loss 2.8027 | ce 2.7912 | lr 1.00e-03 | tok/s 512,381 | elapsed 88.3min\nstep   8150 | epoch 0.65 | loss 2.7961 | ce 2.8081 | lr 1.00e-03 | tok/s 512,735 | elapsed 88.9min\nstep   8200 | epoch 0.65 | loss 2.7932 | ce 2.8361 | lr 1.00e-03 | tok/s 513,529 | elapsed 89.4min\nstep   8250 | epoch 0.65 | loss 2.8066 | ce 2.8162 | lr 1.00e-03 | tok/s 513,161 | elapsed 89.9min\nstep   8300 | epoch 0.66 | loss 2.8007 | ce 2.8113 | lr 1.00e-03 | tok/s 512,837 | elapsed 90.5min\nstep   8350 | epoch 0.66 | loss 2.7934 | ce 2.7807 | lr 1.00e-03 | tok/s 512,514 | elapsed 91.0min\nstep   8400 | epoch 0.67 | loss 2.7868 | ce 2.8165 | lr 1.00e-03 | tok/s 512,058 | elapsed 91.5min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 8401)\nstep   8450 | epoch 0.67 | loss 2.7972 | ce 2.7548 | lr 1.00e-03 | tok/s 511,639 | elapsed 92.1min\nstep   8500 | epoch 0.67 | loss 2.7987 | ce 2.7755 | lr 1.00e-03 | tok/s 512,384 | elapsed 92.6min\nstep   8550 | epoch 0.68 | loss 2.7951 | ce 2.8559 | lr 1.00e-03 | tok/s 512,326 | elapsed 93.1min\nstep   8600 | epoch 0.68 | loss 2.7941 | ce 2.8313 | lr 1.00e-03 | tok/s 512,363 | elapsed 93.7min\nstep   8650 | epoch 0.69 | loss 2.7946 | ce 2.7491 | lr 1.00e-03 | tok/s 512,147 | elapsed 94.2min\nstep   8700 | epoch 0.69 | loss 2.7889 | ce 2.7933 | lr 1.00e-03 | tok/s 512,377 | elapsed 94.7min\nstep   8750 | epoch 0.69 | loss 2.7919 | ce 2.8031 | lr 1.00e-03 | tok/s 512,210 | elapsed 95.3min\nstep   8800 | epoch 0.70 | loss 2.7856 | ce 2.7430 | lr 1.00e-03 | tok/s 512,300 | elapsed 95.8min\nstep   8850 | epoch 0.70 | loss 2.7933 | ce 2.8493 | lr 1.00e-03 | tok/s 511,910 | elapsed 96.3min\nstep   8900 | epoch 0.70 | loss 2.7949 | ce 2.7333 | lr 1.00e-03 | tok/s 511,901 | elapsed 96.9min\nstep   8950 | epoch 0.71 | loss 2.7891 | ce 2.7714 | lr 1.00e-03 | tok/s 511,873 | elapsed 97.4min\nstep   9000 | epoch 0.71 | loss 2.7847 | ce 2.8276 | lr 1.00e-03 | tok/s 512,114 | elapsed 97.9min\nstep   9000 | val_loss 2.7594 | val_acc 0.4480 | val_ppl 15.79\n[Export] step 9001: 194 tensors (+1 aliased) -> ./checkpoints/model_step9001.safetensors (114.1 MB, 0.05s)\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 9001)\nstep   9050 | epoch 0.72 | loss 2.7870 | ce 2.7389 | lr 1.00e-03 | tok/s 501,110 | elapsed 98.5min\nstep   9100 | epoch 0.72 | loss 2.7830 | ce 2.7636 | lr 1.00e-03 | tok/s 512,924 | elapsed 99.0min\nstep   9150 | epoch 0.72 | loss 2.7895 | ce 2.7919 | lr 1.00e-03 | tok/s 512,033 | elapsed 99.5min\nstep   9200 | epoch 0.73 | loss 2.7879 | ce 2.7666 | lr 1.00e-03 | tok/s 512,017 | elapsed 100.1min\nstep   9250 | epoch 0.73 | loss 2.7849 | ce 2.7981 | lr 1.00e-03 | tok/s 512,493 | elapsed 100.6min\nstep   9300 | epoch 0.74 | loss 2.7799 | ce 2.7809 | lr 1.00e-03 | tok/s 512,508 | elapsed 101.1min\nstep   9350 | epoch 0.74 | loss 2.7860 | ce 2.8281 | lr 1.00e-03 | tok/s 513,398 | elapsed 101.7min\nstep   9400 | epoch 0.74 | loss 2.7851 | ce 2.7223 | lr 1.00e-03 | tok/s 512,876 | elapsed 102.2min\nstep   9450 | epoch 0.75 | loss 2.7802 | ce 2.7982 | lr 1.00e-03 | tok/s 512,654 | elapsed 102.7min\nstep   9500 | epoch 0.75 | loss 2.7765 | ce 2.7723 | lr 1.00e-03 | tok/s 512,408 | elapsed 103.3min\nstep   9550 | epoch 0.76 | loss 2.7888 | ce 2.8476 | lr 1.00e-03 | tok/s 512,101 | elapsed 103.8min\nstep   9600 | epoch 0.76 | loss 2.7729 | ce 2.7354 | lr 1.00e-03 | tok/s 512,446 | elapsed 104.3min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 9601)\nstep   9650 | epoch 0.76 | loss 2.7809 | ce 2.7435 | lr 1.00e-03 | tok/s 511,627 | elapsed 104.9min\nstep   9700 | epoch 0.77 | loss 2.7787 | ce 2.7347 | lr 1.00e-03 | tok/s 512,293 | elapsed 105.4min\nstep   9750 | epoch 0.77 | loss 2.7749 | ce 2.7735 | lr 1.00e-03 | tok/s 512,514 | elapsed 105.9min\nstep   9800 | epoch 0.78 | loss 2.7844 | ce 2.7364 | lr 1.00e-03 | tok/s 512,273 | elapsed 106.5min\nstep   9850 | epoch 0.78 | loss 2.7794 | ce 2.7825 | lr 1.00e-03 | tok/s 511,964 | elapsed 107.0min\nstep   9900 | epoch 0.78 | loss 2.7787 | ce 2.7640 | lr 1.00e-03 | tok/s 511,972 | elapsed 107.5min\nstep   9950 | epoch 0.79 | loss 2.7847 | ce 2.7653 | lr 1.00e-03 | tok/s 511,957 | elapsed 108.1min\nstep  10000 | epoch 0.79 | loss 2.7711 | ce 2.7791 | lr 1.00e-03 | tok/s 512,104 | elapsed 108.6min\nstep  10000 | val_loss 2.7461 | val_acc 0.4496 | val_ppl 15.58\nstep  10050 | epoch 0.80 | loss 2.7797 | ce 2.7785 | lr 1.00e-03 | tok/s 503,344 | elapsed 109.1min\nstep  10100 | epoch 0.80 | loss 2.7765 | ce 2.7785 | lr 1.00e-03 | tok/s 512,530 | elapsed 109.7min\nstep  10150 | epoch 0.80 | loss 2.7769 | ce 2.7748 | lr 1.00e-03 | tok/s 512,271 | elapsed 110.2min\nstep  10200 | epoch 0.81 | loss 2.7781 | ce 2.7824 | lr 1.00e-03 | tok/s 512,601 | elapsed 110.7min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 10201)\nstep  10250 | epoch 0.81 | loss 2.7756 | ce 2.7887 | lr 1.00e-03 | tok/s 512,135 | elapsed 111.3min\nstep  10300 | epoch 0.82 | loss 2.7802 | ce 2.7619 | lr 1.00e-03 | tok/s 512,656 | elapsed 111.8min\nstep  10350 | epoch 0.82 | loss 2.7725 | ce 2.7414 | lr 1.00e-03 | tok/s 512,186 | elapsed 112.3min\nstep  10400 | epoch 0.82 | loss 2.7813 | ce 2.7434 | lr 1.00e-03 | tok/s 512,206 | elapsed 112.9min\nstep  10450 | epoch 0.83 | loss 2.7692 | ce 2.7980 | lr 1.00e-03 | tok/s 512,437 | elapsed 113.4min\nstep  10500 | epoch 0.83 | loss 2.7651 | ce 2.7766 | lr 1.00e-03 | tok/s 512,724 | elapsed 113.9min\nstep  10550 | epoch 0.84 | loss 2.7609 | ce 2.7090 | lr 1.00e-03 | tok/s 512,487 | elapsed 114.5min\nstep  10600 | epoch 0.84 | loss 2.7776 | ce 2.7781 | lr 1.00e-03 | tok/s 512,543 | elapsed 115.0min\nstep  10650 | epoch 0.84 | loss 2.7652 | ce 2.7837 | lr 1.00e-03 | tok/s 512,709 | elapsed 115.5min\nstep  10700 | epoch 0.85 | loss 2.7711 | ce 2.7502 | lr 1.00e-03 | tok/s 512,311 | elapsed 116.1min\nstep  10750 | epoch 0.85 | loss 2.7714 | ce 2.7753 | lr 1.00e-03 | tok/s 512,107 | elapsed 116.6min\nstep  10800 | epoch 0.86 | loss 2.7703 | ce 2.7363 | lr 1.00e-03 | tok/s 512,264 | elapsed 117.1min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 10801)\nstep  10850 | epoch 0.86 | loss 2.7697 | ce 2.7669 | lr 1.00e-03 | tok/s 512,363 | elapsed 117.7min\nstep  10900 | epoch 0.86 | loss 2.7707 | ce 2.7214 | lr 1.00e-03 | tok/s 512,390 | elapsed 118.2min\nstep  10950 | epoch 0.87 | loss 2.7722 | ce 2.7706 | lr 1.00e-03 | tok/s 512,513 | elapsed 118.7min\nstep  11000 | epoch 0.87 | loss 2.7740 | ce 2.7819 | lr 1.00e-03 | tok/s 513,479 | elapsed 119.3min\nstep  11000 | val_loss 2.7396 | val_acc 0.4508 | val_ppl 15.48\nstep  11050 | epoch 0.88 | loss 2.7677 | ce 2.7745 | lr 1.00e-03 | tok/s 503,878 | elapsed 119.8min\nstep  11100 | epoch 0.88 | loss 2.7680 | ce 2.7165 | lr 1.00e-03 | tok/s 513,670 | elapsed 120.3min\nstep  11150 | epoch 0.88 | loss 2.7733 | ce 2.7884 | lr 1.00e-03 | tok/s 515,070 | elapsed 120.9min\nstep  11200 | epoch 0.89 | loss 2.7656 | ce 2.7828 | lr 1.00e-03 | tok/s 514,548 | elapsed 121.4min\nstep  11250 | epoch 0.89 | loss 2.7656 | ce 2.7785 | lr 1.00e-03 | tok/s 514,693 | elapsed 121.9min\nstep  11300 | epoch 0.89 | loss 2.7698 | ce 2.8118 | lr 1.00e-03 | tok/s 515,118 | elapsed 122.5min\nstep  11350 | epoch 0.90 | loss 2.7641 | ce 2.7871 | lr 1.00e-03 | tok/s 515,698 | elapsed 123.0min\nstep  11400 | epoch 0.90 | loss 2.7647 | ce 2.7230 | lr 1.00e-03 | tok/s 515,337 | elapsed 123.5min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 11401)\nstep  11450 | epoch 0.91 | loss 2.7665 | ce 2.7360 | lr 1.00e-03 | tok/s 514,922 | elapsed 124.0min\nstep  11500 | epoch 0.91 | loss 2.7649 | ce 2.6975 | lr 1.00e-03 | tok/s 514,601 | elapsed 124.6min\nstep  11550 | epoch 0.91 | loss 2.7574 | ce 2.7868 | lr 1.00e-03 | tok/s 515,090 | elapsed 125.1min\nstep  11600 | epoch 0.92 | loss 2.7619 | ce 2.8172 | lr 1.00e-03 | tok/s 515,199 | elapsed 125.6min\nstep  11650 | epoch 0.92 | loss 2.7655 | ce 2.7617 | lr 1.00e-03 | tok/s 515,270 | elapsed 126.2min\nstep  11700 | epoch 0.93 | loss 2.7593 | ce 2.7297 | lr 1.00e-03 | tok/s 514,894 | elapsed 126.7min\nstep  11750 | epoch 0.93 | loss 2.7649 | ce 2.7866 | lr 1.00e-03 | tok/s 515,130 | elapsed 127.2min\nstep  11800 | epoch 0.93 | loss 2.7605 | ce 2.8025 | lr 1.00e-03 | tok/s 515,079 | elapsed 127.8min\nstep  11850 | epoch 0.94 | loss 2.7676 | ce 2.8043 | lr 1.00e-03 | tok/s 515,966 | elapsed 128.3min\nstep  11900 | epoch 0.94 | loss 2.7642 | ce 2.7443 | lr 1.00e-03 | tok/s 515,651 | elapsed 128.8min\nstep  11950 | epoch 0.95 | loss 2.7670 | ce 2.7430 | lr 1.00e-03 | tok/s 515,633 | elapsed 129.3min\nstep  12000 | epoch 0.95 | loss 2.7615 | ce 2.7911 | lr 1.00e-03 | tok/s 515,206 | elapsed 129.9min\nstep  12000 | val_loss 2.7324 | val_acc 0.4516 | val_ppl 15.37\n[Export] step 12001: 194 tensors (+1 aliased) -> ./checkpoints/model_step12001.safetensors (114.1 MB, 0.05s)\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 12001)\nstep  12050 | epoch 0.95 | loss 2.7615 | ce 2.7475 | lr 1.00e-03 | tok/s 504,613 | elapsed 130.4min\nstep  12100 | epoch 0.96 | loss 2.7576 | ce 2.7657 | lr 1.00e-03 | tok/s 515,245 | elapsed 130.9min\nstep  12150 | epoch 0.96 | loss 2.7540 | ce 2.7406 | lr 1.00e-03 | tok/s 515,251 | elapsed 131.5min\nstep  12200 | epoch 0.97 | loss 2.7652 | ce 2.7572 | lr 1.00e-03 | tok/s 514,915 | elapsed 132.0min\nstep  12250 | epoch 0.97 | loss 2.7549 | ce 2.7503 | lr 1.00e-03 | tok/s 515,172 | elapsed 132.5min\nstep  12300 | epoch 0.97 | loss 2.7691 | ce 2.7684 | lr 1.00e-03 | tok/s 515,038 | elapsed 133.1min\nstep  12350 | epoch 0.98 | loss 2.7582 | ce 2.7499 | lr 1.00e-03 | tok/s 514,919 | elapsed 133.6min\nstep  12400 | epoch 0.98 | loss 2.7596 | ce 2.7423 | lr 1.00e-03 | tok/s 515,212 | elapsed 134.1min\nstep  12450 | epoch 0.99 | loss 2.7578 | ce 2.7880 | lr 1.00e-03 | tok/s 514,558 | elapsed 134.7min\nstep  12500 | epoch 0.99 | loss 2.7557 | ce 2.7271 | lr 1.00e-03 | tok/s 514,918 | elapsed 135.2min\nstep  12550 | epoch 0.99 | loss 2.7533 | ce 2.7727 | lr 1.00e-03 | tok/s 513,856 | elapsed 135.7min\nstep  12600 | epoch 1.00 | loss 2.7600 | ce 2.7319 | lr 1.00e-03 | tok/s 513,420 | elapsed 136.3min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 12601)\nstep  12650 | epoch 1.00 | loss 2.7587 | ce 2.7420 | lr 1.00e-03 | tok/s 513,520 | elapsed 136.8min\nstep  12700 | epoch 1.01 | loss 2.7610 | ce 2.7465 | lr 1.00e-03 | tok/s 513,411 | elapsed 137.3min\nstep  12750 | epoch 1.01 | loss 2.7583 | ce 2.7505 | lr 1.00e-03 | tok/s 514,368 | elapsed 137.8min\nstep  12800 | epoch 1.01 | loss 2.7545 | ce 2.7579 | lr 1.00e-03 | tok/s 515,224 | elapsed 138.4min\nstep  12850 | epoch 1.02 | loss 2.7569 | ce 2.7840 | lr 1.00e-03 | tok/s 514,578 | elapsed 138.9min\nstep  12900 | epoch 1.02 | loss 2.7524 | ce 2.7476 | lr 1.00e-03 | tok/s 515,105 | elapsed 139.4min\nstep  12950 | epoch 1.03 | loss 2.7610 | ce 2.7625 | lr 1.00e-03 | tok/s 514,637 | elapsed 140.0min\nstep  13000 | epoch 1.03 | loss 2.7473 | ce 2.7262 | lr 1.00e-03 | tok/s 514,614 | elapsed 140.5min\nstep  13000 | val_loss 2.7208 | val_acc 0.4531 | val_ppl 15.19\nstep  13050 | epoch 1.03 | loss 2.7548 | ce 2.7918 | lr 1.00e-03 | tok/s 505,607 | elapsed 141.0min\nstep  13100 | epoch 1.04 | loss 2.7565 | ce 2.7792 | lr 1.00e-03 | tok/s 514,981 | elapsed 141.6min\nstep  13150 | epoch 1.04 | loss 2.7531 | ce 2.7616 | lr 1.00e-03 | tok/s 513,859 | elapsed 142.1min\nstep  13200 | epoch 1.05 | loss 2.7436 | ce 2.7340 | lr 1.00e-03 | tok/s 514,633 | elapsed 142.6min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 13201)\nstep  13250 | epoch 1.05 | loss 2.7505 | ce 2.7427 | lr 1.00e-03 | tok/s 514,643 | elapsed 143.2min\nstep  13300 | epoch 1.05 | loss 2.7479 | ce 2.8008 | lr 1.00e-03 | tok/s 515,426 | elapsed 143.7min\nstep  13350 | epoch 1.06 | loss 2.7482 | ce 2.7341 | lr 1.00e-03 | tok/s 515,529 | elapsed 144.2min\nstep  13400 | epoch 1.06 | loss 2.7453 | ce 2.7200 | lr 1.00e-03 | tok/s 515,370 | elapsed 144.8min\nstep  13450 | epoch 1.07 | loss 2.7443 | ce 2.7120 | lr 1.00e-03 | tok/s 514,186 | elapsed 145.3min\nstep  13500 | epoch 1.07 | loss 2.7569 | ce 2.7525 | lr 1.00e-03 | tok/s 513,151 | elapsed 145.8min\nstep  13550 | epoch 1.07 | loss 2.7513 | ce 2.7541 | lr 1.00e-03 | tok/s 512,427 | elapsed 146.3min\nstep  13600 | epoch 1.08 | loss 2.7440 | ce 2.7200 | lr 1.00e-03 | tok/s 512,657 | elapsed 146.9min\nstep  13650 | epoch 1.08 | loss 2.7515 | ce 2.7841 | lr 1.00e-03 | tok/s 512,487 | elapsed 147.4min\nstep  13700 | epoch 1.08 | loss 2.7528 | ce 2.7871 | lr 1.00e-03 | tok/s 512,515 | elapsed 147.9min\nstep  13750 | epoch 1.09 | loss 2.7514 | ce 2.7618 | lr 1.00e-03 | tok/s 512,926 | elapsed 148.5min\nstep  13800 | epoch 1.09 | loss 2.7550 | ce 2.7390 | lr 1.00e-03 | tok/s 513,930 | elapsed 149.0min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 13801)\nstep  13850 | epoch 1.10 | loss 2.7548 | ce 2.7525 | lr 1.00e-03 | tok/s 513,151 | elapsed 149.5min\nstep  13900 | epoch 1.10 | loss 2.7468 | ce 2.7441 | lr 1.00e-03 | tok/s 513,204 | elapsed 150.1min\nstep  13950 | epoch 1.10 | loss 2.7479 | ce 2.7633 | lr 1.00e-03 | tok/s 512,780 | elapsed 150.6min\nstep  14000 | epoch 1.11 | loss 2.7505 | ce 2.7267 | lr 1.00e-03 | tok/s 513,072 | elapsed 151.1min\nstep  14000 | val_loss 2.7143 | val_acc 0.4539 | val_ppl 15.09\nstep  14050 | epoch 1.11 | loss 2.7441 | ce 2.7086 | lr 1.00e-03 | tok/s 503,876 | elapsed 151.7min\nstep  14100 | epoch 1.12 | loss 2.7506 | ce 2.7142 | lr 1.00e-03 | tok/s 513,550 | elapsed 152.2min\nstep  14150 | epoch 1.12 | loss 2.7489 | ce 2.7291 | lr 1.00e-03 | tok/s 513,225 | elapsed 152.7min\nstep  14200 | epoch 1.12 | loss 2.7492 | ce 2.7220 | lr 1.00e-03 | tok/s 514,341 | elapsed 153.3min\nstep  14250 | epoch 1.13 | loss 2.7471 | ce 2.7319 | lr 1.00e-03 | tok/s 513,288 | elapsed 153.8min\nstep  14300 | epoch 1.13 | loss 2.7458 | ce 2.7343 | lr 1.00e-03 | tok/s 513,868 | elapsed 154.3min\nstep  14350 | epoch 1.14 | loss 2.7426 | ce 2.7405 | lr 1.00e-03 | tok/s 514,408 | elapsed 154.9min\nstep  14400 | epoch 1.14 | loss 2.7456 | ce 2.7218 | lr 1.00e-03 | tok/s 513,900 | elapsed 155.4min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 14401)\nstep  14450 | epoch 1.14 | loss 2.7418 | ce 2.7637 | lr 1.00e-03 | tok/s 514,689 | elapsed 155.9min\nstep  14500 | epoch 1.15 | loss 2.7459 | ce 2.7480 | lr 1.00e-03 | tok/s 514,962 | elapsed 156.5min\nstep  14550 | epoch 1.15 | loss 2.7490 | ce 2.6903 | lr 1.00e-03 | tok/s 515,629 | elapsed 157.0min\nstep  14600 | epoch 1.16 | loss 2.7405 | ce 2.6805 | lr 1.00e-03 | tok/s 515,096 | elapsed 157.5min\nstep  14650 | epoch 1.16 | loss 2.7413 | ce 2.7664 | lr 1.00e-03 | tok/s 514,640 | elapsed 158.1min\nstep  14700 | epoch 1.16 | loss 2.7351 | ce 2.7796 | lr 1.00e-03 | tok/s 514,257 | elapsed 158.6min\nstep  14750 | epoch 1.17 | loss 2.7431 | ce 2.6591 | lr 1.00e-03 | tok/s 513,382 | elapsed 159.1min\nstep  14800 | epoch 1.17 | loss 2.7408 | ce 2.6871 | lr 1.00e-03 | tok/s 513,657 | elapsed 159.6min\nstep  14850 | epoch 1.18 | loss 2.7368 | ce 2.7414 | lr 1.00e-03 | tok/s 514,657 | elapsed 160.2min\nstep  14900 | epoch 1.18 | loss 2.7376 | ce 2.7495 | lr 1.00e-03 | tok/s 513,222 | elapsed 160.7min\nstep  14950 | epoch 1.18 | loss 2.7380 | ce 2.7502 | lr 1.00e-03 | tok/s 512,429 | elapsed 161.2min\nstep  15000 | epoch 1.19 | loss 2.7365 | ce 2.7581 | lr 1.00e-03 | tok/s 513,469 | elapsed 161.8min\nstep  15000 | val_loss 2.7118 | val_acc 0.4545 | val_ppl 15.06\n[Export] step 15001: 194 tensors (+1 aliased) -> ./checkpoints/model_step15001.safetensors (114.1 MB, 0.12s)\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 15001)\nstep  15050 | epoch 1.19 | loss 2.7387 | ce 2.7593 | lr 1.00e-03 | tok/s 502,687 | elapsed 162.3min\nstep  15100 | epoch 1.20 | loss 2.7403 | ce 2.7501 | lr 1.00e-03 | tok/s 514,191 | elapsed 162.8min\nstep  15150 | epoch 1.20 | loss 2.7435 | ce 2.7639 | lr 1.00e-03 | tok/s 514,297 | elapsed 163.4min\nstep  15200 | epoch 1.20 | loss 2.7343 | ce 2.7208 | lr 1.00e-03 | tok/s 513,829 | elapsed 163.9min\nstep  15250 | epoch 1.21 | loss 2.7374 | ce 2.7332 | lr 1.00e-03 | tok/s 515,081 | elapsed 164.4min\nstep  15300 | epoch 1.21 | loss 2.7290 | ce 2.7063 | lr 1.00e-03 | tok/s 514,676 | elapsed 165.0min\nstep  15350 | epoch 1.22 | loss 2.7316 | ce 2.7706 | lr 1.00e-03 | tok/s 515,277 | elapsed 165.5min\nstep  15400 | epoch 1.22 | loss 2.7450 | ce 2.7784 | lr 1.00e-03 | tok/s 514,054 | elapsed 166.0min\nstep  15450 | epoch 1.22 | loss 2.7439 | ce 2.7328 | lr 1.00e-03 | tok/s 514,629 | elapsed 166.6min\nstep  15500 | epoch 1.23 | loss 2.7457 | ce 2.7414 | lr 1.00e-03 | tok/s 514,167 | elapsed 167.1min\nstep  15550 | epoch 1.23 | loss 2.7330 | ce 2.7136 | lr 1.00e-03 | tok/s 514,762 | elapsed 167.6min\nstep  15600 | epoch 1.24 | loss 2.7330 | ce 2.7358 | lr 9.98e-04 | tok/s 514,155 | elapsed 168.2min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 15601)\nstep  15650 | epoch 1.24 | loss 2.7381 | ce 2.7639 | lr 9.96e-04 | tok/s 512,461 | elapsed 168.7min\nstep  15700 | epoch 1.24 | loss 2.7323 | ce 2.7383 | lr 9.93e-04 | tok/s 512,674 | elapsed 169.2min\nstep  15750 | epoch 1.25 | loss 2.7361 | ce 2.7349 | lr 9.89e-04 | tok/s 512,500 | elapsed 169.8min\nstep  15800 | epoch 1.25 | loss 2.7301 | ce 2.7006 | lr 9.84e-04 | tok/s 513,152 | elapsed 170.3min\nstep  15850 | epoch 1.26 | loss 2.7297 | ce 2.6967 | lr 9.78e-04 | tok/s 511,984 | elapsed 170.8min\nstep  15900 | epoch 1.26 | loss 2.7259 | ce 2.7234 | lr 9.71e-04 | tok/s 512,646 | elapsed 171.4min\nstep  15950 | epoch 1.26 | loss 2.7405 | ce 2.7727 | lr 9.64e-04 | tok/s 512,255 | elapsed 171.9min\nstep  16000 | epoch 1.27 | loss 2.7321 | ce 2.7162 | lr 9.55e-04 | tok/s 512,920 | elapsed 172.4min\nstep  16000 | val_loss 2.7008 | val_acc 0.4565 | val_ppl 14.89\nstep  16050 | epoch 1.27 | loss 2.7336 | ce 2.7485 | lr 9.46e-04 | tok/s 504,260 | elapsed 173.0min\nstep  16100 | epoch 1.28 | loss 2.7175 | ce 2.6817 | lr 9.36e-04 | tok/s 514,228 | elapsed 173.5min\nstep  16150 | epoch 1.28 | loss 2.7284 | ce 2.7200 | lr 9.25e-04 | tok/s 514,331 | elapsed 174.0min\nstep  16200 | epoch 1.28 | loss 2.7223 | ce 2.7064 | lr 9.14e-04 | tok/s 514,748 | elapsed 174.6min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 16201)\nstep  16250 | epoch 1.29 | loss 2.7209 | ce 2.6876 | lr 9.02e-04 | tok/s 513,680 | elapsed 175.1min\nstep  16300 | epoch 1.29 | loss 2.7134 | ce 2.7119 | lr 8.89e-04 | tok/s 513,213 | elapsed 175.6min\nstep  16350 | epoch 1.29 | loss 2.7222 | ce 2.6854 | lr 8.75e-04 | tok/s 512,564 | elapsed 176.1min\nstep  16400 | epoch 1.30 | loss 2.7207 | ce 2.6801 | lr 8.61e-04 | tok/s 512,666 | elapsed 176.7min\nstep  16450 | epoch 1.30 | loss 2.7223 | ce 2.7212 | lr 8.46e-04 | tok/s 512,555 | elapsed 177.2min\nstep  16500 | epoch 1.31 | loss 2.7116 | ce 2.6817 | lr 8.30e-04 | tok/s 512,707 | elapsed 177.7min\nstep  16550 | epoch 1.31 | loss 2.7153 | ce 2.7655 | lr 8.14e-04 | tok/s 512,445 | elapsed 178.3min\nstep  16600 | epoch 1.31 | loss 2.7122 | ce 2.7218 | lr 7.98e-04 | tok/s 512,537 | elapsed 178.8min\nstep  16650 | epoch 1.32 | loss 2.7107 | ce 2.7351 | lr 7.80e-04 | tok/s 512,635 | elapsed 179.3min\nstep  16700 | epoch 1.32 | loss 2.7096 | ce 2.7676 | lr 7.63e-04 | tok/s 512,089 | elapsed 179.9min\nstep  16750 | epoch 1.33 | loss 2.7089 | ce 2.7320 | lr 7.45e-04 | tok/s 512,638 | elapsed 180.4min\nstep  16800 | epoch 1.33 | loss 2.7072 | ce 2.7475 | lr 7.26e-04 | tok/s 512,722 | elapsed 180.9min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 16801)\nstep  16850 | epoch 1.33 | loss 2.7071 | ce 2.7147 | lr 7.08e-04 | tok/s 512,364 | elapsed 181.5min\nstep  16900 | epoch 1.34 | loss 2.6960 | ce 2.6424 | lr 6.89e-04 | tok/s 512,906 | elapsed 182.0min\nstep  16950 | epoch 1.34 | loss 2.6958 | ce 2.7162 | lr 6.69e-04 | tok/s 512,554 | elapsed 182.5min\nstep  17000 | epoch 1.35 | loss 2.7011 | ce 2.7590 | lr 6.50e-04 | tok/s 513,349 | elapsed 183.1min\nstep  17000 | val_loss 2.6624 | val_acc 0.4620 | val_ppl 14.33\nstep  17050 | epoch 1.35 | loss 2.7022 | ce 2.6949 | lr 6.30e-04 | tok/s 504,889 | elapsed 183.6min\nstep  17100 | epoch 1.35 | loss 2.6943 | ce 2.6557 | lr 6.10e-04 | tok/s 514,698 | elapsed 184.1min\nstep  17150 | epoch 1.36 | loss 2.6849 | ce 2.6454 | lr 5.90e-04 | tok/s 514,605 | elapsed 184.7min\nstep  17200 | epoch 1.36 | loss 2.6777 | ce 2.6970 | lr 5.70e-04 | tok/s 513,805 | elapsed 185.2min\nstep  17250 | epoch 1.37 | loss 2.6788 | ce 2.6647 | lr 5.50e-04 | tok/s 512,651 | elapsed 185.7min\nstep  17300 | epoch 1.37 | loss 2.6785 | ce 2.6216 | lr 5.29e-04 | tok/s 513,081 | elapsed 186.3min\nstep  17350 | epoch 1.37 | loss 2.6750 | ce 2.6945 | lr 5.09e-04 | tok/s 513,108 | elapsed 186.8min\nstep  17400 | epoch 1.38 | loss 2.6736 | ce 2.7298 | lr 4.89e-04 | tok/s 512,315 | elapsed 187.3min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 17401)\nstep  17450 | epoch 1.38 | loss 2.6697 | ce 2.6768 | lr 4.69e-04 | tok/s 512,865 | elapsed 187.9min\nstep  17500 | epoch 1.39 | loss 2.6748 | ce 2.6293 | lr 4.49e-04 | tok/s 512,578 | elapsed 188.4min\nstep  17550 | epoch 1.39 | loss 2.6665 | ce 2.6493 | lr 4.30e-04 | tok/s 511,862 | elapsed 188.9min\nstep  17600 | epoch 1.39 | loss 2.6621 | ce 2.7009 | lr 4.11e-04 | tok/s 512,216 | elapsed 189.5min\nstep  17650 | epoch 1.40 | loss 2.6547 | ce 2.6845 | lr 3.92e-04 | tok/s 512,483 | elapsed 190.0min\nstep  17700 | epoch 1.40 | loss 2.6569 | ce 2.7060 | lr 3.73e-04 | tok/s 512,807 | elapsed 190.5min\nstep  17750 | epoch 1.41 | loss 2.6578 | ce 2.6618 | lr 3.54e-04 | tok/s 512,279 | elapsed 191.1min\nstep  17800 | epoch 1.41 | loss 2.6527 | ce 2.6804 | lr 3.36e-04 | tok/s 511,948 | elapsed 191.6min\nstep  17850 | epoch 1.41 | loss 2.6580 | ce 2.6613 | lr 3.19e-04 | tok/s 512,451 | elapsed 192.1min\nstep  17900 | epoch 1.42 | loss 2.6472 | ce 2.5631 | lr 3.02e-04 | tok/s 511,944 | elapsed 192.7min\nstep  17950 | epoch 1.42 | loss 2.6446 | ce 2.6326 | lr 2.85e-04 | tok/s 512,091 | elapsed 193.2min\nstep  18000 | epoch 1.43 | loss 2.6422 | ce 2.6589 | lr 2.69e-04 | tok/s 512,671 | elapsed 193.7min\nstep  18000 | val_loss 2.6095 | val_acc 0.4698 | val_ppl 13.59\n[Export] step 18001: 194 tensors (+1 aliased) -> ./checkpoints/model_step18001.safetensors (114.1 MB, 0.05s)\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 18001)\nstep  18050 | epoch 1.43 | loss 2.6409 | ce 2.6603 | lr 2.54e-04 | tok/s 480,005 | elapsed 194.3min\nstep  18100 | epoch 1.43 | loss 2.6389 | ce 2.6796 | lr 2.39e-04 | tok/s 512,580 | elapsed 194.8min\nstep  18150 | epoch 1.44 | loss 2.6395 | ce 2.6795 | lr 2.24e-04 | tok/s 512,586 | elapsed 195.4min\nstep  18200 | epoch 1.44 | loss 2.6381 | ce 2.6149 | lr 2.11e-04 | tok/s 513,235 | elapsed 195.9min\nstep  18250 | epoch 1.45 | loss 2.6254 | ce 2.6491 | lr 1.98e-04 | tok/s 512,619 | elapsed 196.4min\nstep  18300 | epoch 1.45 | loss 2.6296 | ce 2.5857 | lr 1.86e-04 | tok/s 512,285 | elapsed 197.0min\nstep  18350 | epoch 1.45 | loss 2.6312 | ce 2.6688 | lr 1.74e-04 | tok/s 512,393 | elapsed 197.5min\nstep  18400 | epoch 1.46 | loss 2.6256 | ce 2.6221 | lr 1.63e-04 | tok/s 512,969 | elapsed 198.0min\nstep  18450 | epoch 1.46 | loss 2.6208 | ce 2.6073 | lr 1.54e-04 | tok/s 513,204 | elapsed 198.6min\nstep  18500 | epoch 1.47 | loss 2.6218 | ce 2.6544 | lr 1.44e-04 | tok/s 512,415 | elapsed 199.1min\nstep  18550 | epoch 1.47 | loss 2.6166 | ce 2.6012 | lr 1.36e-04 | tok/s 512,436 | elapsed 199.6min\nstep  18600 | epoch 1.47 | loss 2.6213 | ce 2.6207 | lr 1.29e-04 | tok/s 512,987 | elapsed 200.2min\n[Save] async checkpoint written -> ./checkpoints/arya_latest.pt (step 18601)\nstep  18650 | epoch 1.48 | loss 2.6192 | ce 2.5554 | lr 1.22e-04 | tok/s 512,288 | elapsed 200.7min\nstep  18700 | epoch 1.48 | loss 2.6205 | ce 2.6088 | lr 1.16e-04 | tok/s 511,698 | elapsed 201.2min\nstep  18750 | epoch 1.48 | loss 2.6232 | ce 2.6280 | lr 1.11e-04 | tok/s 512,009 | elapsed 201.8min\nstep  18800 | epoch 1.49 | loss 2.6128 | ce 2.6425 | lr 1.07e-04 | tok/s 512,967 | elapsed 202.3min\nstep  18850 | epoch 1.49 | loss 2.6144 | ce 2.6497 | lr 1.04e-04 | tok/s 512,090 | elapsed 202.8min\nstep  18900 | epoch 1.50 | loss 2.6191 | ce 2.5731 | lr 1.02e-04 | tok/s 512,580 | elapsed 203.4min\nstep  18950 | epoch 1.50 | loss 2.6170 | ce 2.6598 | lr 1.00e-04 | tok/s 512,755 | elapsed 203.9min\n[Save] state_dict(): 0.00s | 195 tensors | 0.13 GB\n[Save] optimizer.state_dict(): 0.00s | 300 tensors | 0.14 GB\n[Save] _cpu walk: 0.12s\n[Save] torch.save -> /tmp: 0.18s | file 275 MB\n[Save] copyfile -> ./checkpoints/arya_latest.pt: 0.08s\n[Export] step 19000: 194 tensors (+1 aliased) -> ./checkpoints/model_step19000.safetensors (114.1 MB, 0.03s)\n[Training] Complete\n",
+          "mimetype": "text/plain"
+        }
+      ]
+    },
+    {
+      "id": "Hstk",
+      "code_hash": "491df3d6836c4a6956d9cdf3e6e8d89f",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "nWHF",
+      "code_hash": "d108748a7f1e39fb39644a9755a27b91",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "iLit",
+      "code_hash": "a1daefd9105ef45d64a60fc09e306b69",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "ZHCJ",
+      "code_hash": "9aa4360427f7a6b988b89521f957855c",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "ROlb",
+      "code_hash": null,
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "qnkX",
+      "code_hash": "26bf08358a5fd5faeadfb725840c6654",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "TqIu",
+      "code_hash": "8cf0ab5f2f0947afe55db20860f35990",
+      "outputs": [
+        {
+          "type": "data",
+          "data": {
+            "text/html": "<pre class='text-xs'>0</pre>"
+          }
+        }
+      ],
+      "console": [
+        {
+          "type": "stream",
+          "name": "stderr",
+          "text": "/usr/local/lib/python3.13/site-packages/torch/profiler/profiler.py:224: UserWarning: Warning: Profiler clears events at the end of each cycle.Only events from the current cycle will be reported.To keep events across cycles, set acc_events=True.\n  _warn_once(\n",
+          "mimetype": "text/plain"
+        },
+        {
+          "type": "stream",
+          "name": "stdout",
+          "text": "[CLI] Overriding config.py: {'use_liger': False}\n[Setup] device=cuda (cuda)\n\n=== Speed diagnosis: can Splash Attention even run here? ===\n  device: cuda\n  config pattern: 'dense'\n  [BLOCKED] device gate: Splash Attention is TPU-only (kernel.py's call_jax bridge) -- not on TPU, so this alone rules it out here.\n  [OK] architecture gate: kernel.fused_attention (Splash's only entry point) is reached by DenseMHA, used in patterns ('dense', 'pyramid_swa', 'mod') -- all three patterns this codebase currently implements. Current pattern reaches DenseMHA, so this gate passes.\n  Architecture gate passes (all supported patterns reach DenseMHA), but not on TPU -- Splash Attention structurally cannot run on this device regardless of pattern. xFormers (T4-class) or flash_attn (A100+) handle GPU instead; check backend stats below to see which one actually engaged.\n[Tokenizer] Loaded, vocab_size=12288\n[Dataset] Found 6 files in ./training_data (curriculum OFF)\n[Dataset] Cache hit -> ./training_data/.token_cache/tokens_83a770868891c636.npy  (4,179,431,312 tokens)\n[Engram] layers=[1, 8] max_ngram=3 compressed_vocab=9807 (raw 12288)\n[Model] 28.50M params, pattern=dense\n[Engram] 2,532,736 engram params\n[Precision] autocast dtype=torch.bfloat16 (device_kind=cuda)\n[Compile] Note: hardest_file_check uses batch=1 (different shape from learnability_check/gpu_profile_check's batch=8) -- expect one additional recompilation when that section runs, not a bug.\n[Compile] WARNING: reduce-overhead requested via alpha_config.ALPHA_COMPILE_MODE -- this model's tied embedding is a known CUDA-graphs hazard (see train.py). Proceeding anyway since you asked explicitly.\n[Compile] mode=reduce-overhead\n\n=== Learnability check: SKIPPED (--skip-learnability) ===\n  Model is at random initialization -- hardest_file_check is also skipped (its loss ranking would be meaningless on untrained weights). Only gpu_profile_check's speed/roofline numbers run, since those reflect architecture shape only, not learned behavior.\n\n=== Hardest-file check: SKIPPED (--skip-learnability) ===\n  Model is at random initialization -- the loss ranking would be meaningless.\n\n=== GPU profiling (CUDA) ===\n  device: NVIDIA RTX PRO 6000 Blackwell Server Edition\n  SMs: 188  |  total VRAM: 102.0 GB\n\n  --- torch.profiler: 3 warmup + 8 recorded steps ---\n\n  Top operators by self_cuda_time_total (8-step window):\n-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg       CPU Mem  Self CPU Mem      CUDA Mem  Self CUDA Mem    # of Calls  Total MFLOPs  \n-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n## Call CompiledFxGraph felvw6fyfphsgtviq7567hgcflv5...         0.00%       0.000us         0.00%       0.000us       0.000us      83.263ms        59.63%      83.263ms      10.408ms           0 B           0 B           0 B           0 B             8            --  \n                               CompiledFunctionBackward         1.92%       3.378ms        11.51%      20.255ms       1.266ms      81.821ms        58.60%      81.838ms       5.115ms           0 B           0 B           0 B           0 B            16            --  \n## Call CompiledFxGraph f2papzyhk3eoy65khzkyflsjbzrx...         0.00%       0.000us         0.00%       0.000us       0.000us      51.195ms        36.67%      51.195ms       6.399ms           0 B           0 B           0 B           0 B             8            --  \n                                       CompiledFunction         1.94%       3.423ms         7.09%      12.477ms     779.838us      50.828ms        36.40%      50.919ms       3.182ms           0 B           0 B           0 B           0 B            16            --  \ntriton_tem_fused__unsafe_view_flex_attention_backwar...         0.00%       0.000us         0.00%       0.000us       0.000us      21.908ms        15.69%      21.908ms     158.753us           0 B           0 B           0 B           0 B           138            --  \nvoid cutlass::Kernel2<cutlass_80_tensorop_bf16_s1681...         0.00%       0.000us         0.00%       0.000us       0.000us      15.735ms        11.27%      15.735ms      26.715us           0 B           0 B           0 B           0 B           589            --  \nvoid cutlass::Kernel2<cutlass_80_tensorop_bf16_s1681...         0.00%       0.000us         0.00%       0.000us       0.000us      15.098ms        10.81%      15.098ms      34.948us           0 B           0 B           0 B           0 B           432            --  \nvoid cutlass::Kernel2<cutlass_80_tensorop_bf16_s1681...         0.00%       0.000us         0.00%       0.000us       0.000us      14.937ms        10.70%      14.937ms      34.656us           0 B           0 B           0 B           0 B           431            --  \ntriton_tem_fused__unsafe_view_flex_attention_split_w...         0.00%       0.000us         0.00%       0.000us       0.000us       5.449ms         3.90%       5.449ms      42.566us           0 B           0 B           0 B           0 B           128            --  \nvoid cutlass::Kernel2<cutlass_80_tensorop_bf16_s1681...         0.00%       0.000us         0.00%       0.000us       0.000us       4.532ms         3.25%       4.532ms      26.047us           0 B           0 B           0 B           0 B           174            --  \ntriton_poi_fused__unsafe_view_cat_mul_silu_silu_back...         0.00%       0.000us         0.00%       0.000us       0.000us       4.388ms         3.14%       4.388ms      31.798us           0 B           0 B           0 B           0 B           138            --  \nvoid cutlass::Kernel2<cutlass_80_tensorop_bf16_s1681...         0.00%       0.000us         0.00%       0.000us       0.000us       3.829ms         2.74%       3.829ms      29.911us           0 B           0 B           0 B           0 B           128            --  \ntriton_red_fused__to_copy_exp_logsumexp_ne_neg_scatt...         0.00%       0.000us         0.00%       0.000us       0.000us       3.274ms         2.34%       3.274ms     204.596us           0 B           0 B           0 B           0 B            16            --  \ntriton_per_fused__fused_rms_norm_backward__to_copy_a...         0.00%       0.000us         0.00%       0.000us       0.000us       2.714ms         1.94%       2.714ms      24.231us           0 B           0 B           0 B           0 B           112            --  \ntriton_per_fused__fused_rms_norm_backward__to_copy_a...         0.00%       0.000us         0.00%       0.000us       0.000us       2.660ms         1.90%       2.660ms      21.980us           0 B           0 B           0 B           0 B           121            --  \n-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \nSelf CPU time total: 176.020ms\nSelf CUDA time total: 139.630ms\n\n  Chrome trace exported to ./profile_trace.json (open at chrome://tracing or https://ui.perfetto.dev)\n\n  --- torch.utils.benchmark: forward-only vs forward+backward ---\n  forward-only:      4.33 ms/iter  (median 4.33 ms, 20 iters)\n  forward+backward:  16.84 ms/iter  (median 16.84 ms, 20 iters)\n  tokens/step: 8,192  ->  486,582 tok/s (forward+backward, this batch/seq shape)\n\n  --- torch.cuda.Event: raw GPU-side step time ---\n  mean GPU-side step time: 16.79 ms  (20 events)\n  vs torch.utils.benchmark forward+backward: 16.84 ms (gap = Python/CPU dispatch overhead)\n\n  --- Memory ---\n  peak reserved:  2.79 GB / 95.0 GB (2.9%)\n  peak allocated: 2.51 GB\n\n  --- Model FLOPs Utilization (MFU) ---\n  total params: 28.50M  |  non-embedding: 23.78M\n  [SKIPPED] 'NVIDIA RTX PRO 6000 Blackwell Server Edition' not in the known-GPU table (T4/L4/A100/H100) -- add its dense FP16 Tensor Core TFLOPS to GPU_PEAK_FP16_TFLOPS at the top of this file to enable this calculation.\n\n[Roofline] Unavailable -- gpu_profile_check didn't return usable data (likely non-CUDA device or an unrecognized GPU -- see the MFU section above).\n\n=== Summary ===\nLearnability: SKIPPED (--skip-learnability)\n\n=== kernel.py backend usage during the learnability check ===\n  N/A -- learnability check was skipped (--skip-learnability), so no forward passes ran during that phase. See gpu_profile_check's own backend stats above (if run) for kernel dispatch info instead.\n",
+          "mimetype": "text/plain"
+        }
+      ]
+    },
+    {
+      "id": "Vxnm",
+      "code_hash": "472b180a72ef3d7ee4b56780749ad24e",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "DnEU",
+      "code_hash": "d6b0911a49796272ca86fe8f1cfba105",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "ulZA",
+      "code_hash": "42312cfb889e4c7d1d95b40183870910",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "ecfG",
+      "code_hash": "8c8d6ed6bb295c0981866d2d9b707d6f",
+      "outputs": [
+        {
+          "type": "data",
+          "data": {
+            "text/html": "<pre class='text-xs'>0</pre>"
+          }
+        }
+      ],
+      "console": [
+        {
+          "type": "stream",
+          "name": "stderr",
+          "text": "WARNING: The directory '/home/marimo/.cache/pip' or its parent directory is not owned or is not writable by the current user. The cache has been disabled. Check the permissions and owner of that directory. If executing pip with sudo, you should use sudo's -H flag.\n",
+          "mimetype": "text/plain"
+        },
+        {
+          "type": "stream",
+          "name": "stdout",
+          "text": "Requirement already satisfied: liger-kernel in /tmp/uv-venv/lib/python3.13/site-packages (0.8.4)\nRequirement already satisfied: rustbpe in /tmp/uv-venv/lib/python3.13/site-packages (0.1.0)\nRequirement already satisfied: torchao in /tmp/uv-venv/lib/python3.13/site-packages (0.18.0)\nRequirement already satisfied: triton in /tmp/uv-venv/lib/python3.13/site-packages (3.8.0)\nCollecting accelerate\n  Downloading accelerate-1.15.0-py3-none-any.whl.metadata (19 kB)\nRequirement already satisfied: safetensors in /tmp/uv-venv/lib/python3.13/site-packages (0.8.0)\nCollecting helion\n  Downloading helion-1.5.1-py3-none-any.whl.metadata (23 kB)\nRequirement already satisfied: torch>=2.1.2 in /usr/local/lib/python3.13/site-packages (from liger-kernel) (2.11.0)\nRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.13/site-packages (from accelerate) (2.4.6)\nRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.13/site-packages (from accelerate) (26.2)\nRequirement already satisfied: psutil in /usr/local/lib/python3.13/site-packages (from accelerate) (7.2.2)\nRequirement already satisfied: pyyaml in /usr/local/lib/python3.13/site-packages (from accelerate) (6.0.3)\nRequirement already satisfied: huggingface_hub>=0.21.0 in /tmp/uv-venv/lib/python3.13/site-packages (from accelerate) (1.33.0)\nCollecting filecheck (from helion)\n  Downloading filecheck-1.0.6-py3-none-any.whl.metadata (5.2 kB)\nRequirement already satisfied: rich in /tmp/uv-venv/lib/python3.13/site-packages (from helion) (15.0.0)\nRequirement already satisfied: scikit-learn>=1.3.0 in /usr/local/lib/python3.13/site-packages (from helion) (1.9.0)\nRequirement already satisfied: typing-extensions>=4.0.0 in /usr/local/lib/python3.13/site-packages (from helion) (4.16.0)\nRequirement already satisfied: click<9.0.0,>=8.4.2 in /usr/local/lib/python3.13/site-packages (from huggingface_hub>=0.21.0->accelerate) (8.4.2)\nRequirement already satisfied: filelock>=3.10.0 in /tmp/uv-venv/lib/python3.13/site-packages (from huggingface_hub>=0.21.0->accelerate) (4.2.0)\nRequirement already satisfied: fsspec>=2023.5.0 in /tmp/uv-venv/lib/python3.13/site-packages (from huggingface_hub>=0.21.0->accelerate) (2026.9.0)\nRequirement already satisfied: hf-xet<2.0.0,>=1.6.0 in /tmp/uv-venv/lib/python3.13/site-packages (from huggingface_hub>=0.21.0->accelerate) (1.7.0)\nRequirement already satisfied: httpx<1,>=0.23.0 in /tmp/uv-venv/lib/python3.13/site-packages (from huggingface_hub>=0.21.0->accelerate) (0.28.1)\nRequirement already satisfied: tqdm>=4.42.1 in /tmp/uv-venv/lib/python3.13/site-packages (from huggingface_hub>=0.21.0->accelerate) (4.70.1)\nRequirement already satisfied: anyio in /tmp/uv-venv/lib/python3.13/site-packages (from httpx<1,>=0.23.0->huggingface_hub>=0.21.0->accelerate) (4.15.1)\nRequirement already satisfied: certifi in /tmp/uv-venv/lib/python3.13/site-packages (from httpx<1,>=0.23.0->huggingface_hub>=0.21.0->accelerate) (2026.7.22)\nRequirement already satisfied: httpcore==1.* in /tmp/uv-venv/lib/python3.13/site-packages (from httpx<1,>=0.23.0->huggingface_hub>=0.21.0->accelerate) (1.0.9)\nRequirement already satisfied: idna in /tmp/uv-venv/lib/python3.13/site-packages (from httpx<1,>=0.23.0->huggingface_hub>=0.21.0->accelerate) (3.20)\nRequirement already satisfied: h11>=0.16 in /tmp/uv-venv/lib/python3.13/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->huggingface_hub>=0.21.0->accelerate) (0.16.0)\nRequirement already satisfied: scipy>=1.10.0 in /usr/local/lib/python3.13/site-packages (from scikit-learn>=1.3.0->helion) (1.18.0)\nRequirement already satisfied: joblib>=1.4.0 in /usr/local/lib/python3.13/site-packages (from scikit-learn>=1.3.0->helion) (1.5.3)\nRequirement already satisfied: narwhals>=2.0.1 in /usr/local/lib/python3.13/site-packages (from scikit-learn>=1.3.0->helion) (2.24.0)\nRequirement already satisfied: threadpoolctl>=3.5.0 in /usr/local/lib/python3.13/site-packages (from scikit-learn>=1.3.0->helion) (3.6.0)\nCollecting setuptools<82 (from torch>=2.1.2->liger-kernel)\n  Downloading setuptools-81.0.0-py3-none-any.whl.metadata (6.6 kB)\nRequirement already satisfied: sympy>=1.13.3 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (1.14.0)\nRequirement already satisfied: networkx>=2.5.1 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (3.6.1)\nRequirement already satisfied: jinja2 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (3.1.6)\nRequirement already satisfied: cuda-toolkit==13.0.2 in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (13.0.2)\nRequirement already satisfied: cuda-bindings<14,>=13.0.3 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (13.3.1)\nRequirement already satisfied: nvidia-cudnn-cu13==9.19.0.56 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (9.19.0.56)\nRequirement already satisfied: nvidia-cusparselt-cu13==0.8.0 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (0.8.0)\nRequirement already satisfied: nvidia-nccl-cu13==2.28.9 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (2.28.9)\nRequirement already satisfied: nvidia-nvshmem-cu13==3.4.5 in /usr/local/lib/python3.13/site-packages (from torch>=2.1.2->liger-kernel) (3.4.5)\nCollecting triton\n  Downloading triton-3.6.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl.metadata (1.7 kB)\nRequirement already satisfied: nvidia-cublas==13.1.0.3.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (13.1.0.3)\nRequirement already satisfied: nvidia-cuda-runtime==13.0.96.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (13.0.96)\nRequirement already satisfied: nvidia-cufft==12.0.0.61.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (12.0.0.61)\nRequirement already satisfied: nvidia-cufile==1.15.1.6.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (1.15.1.6)\nRequirement already satisfied: nvidia-cuda-cupti==13.0.85.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (13.0.85)\nRequirement already satisfied: nvidia-curand==10.4.0.35.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (10.4.0.35)\nRequirement already satisfied: nvidia-cusolver==12.0.4.66.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (12.0.4.66)\nRequirement already satisfied: nvidia-cusparse==12.6.3.3.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (12.6.3.3)\nRequirement already satisfied: nvidia-nvjitlink==13.0.88.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (13.0.88)\nRequirement already satisfied: nvidia-cuda-nvrtc==13.0.88.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (13.0.88)\nRequirement already satisfied: nvidia-nvtx==13.0.85.* in /usr/local/lib/python3.13/site-packages (from cuda-toolkit[cublas,cudart,cufft,cufile,cupti,curand,cusolver,cusparse,nvjitlink,nvrtc,nvtx]==13.0.2; platform_system == \"Linux\"->torch>=2.1.2->liger-kernel) (13.0.85)\nRequirement already satisfied: cuda-pathfinder>=1.4.2 in /usr/local/lib/python3.13/site-packages (from cuda-bindings<14,>=13.0.3->torch>=2.1.2->liger-kernel) (1.5.6)\nRequirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.13/site-packages (from sympy>=1.13.3->torch>=2.1.2->liger-kernel) (1.3.0)\nRequirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.13/site-packages (from jinja2->torch>=2.1.2->liger-kernel) (3.0.3)\nRequirement already satisfied: markdown-it-py>=2.2.0 in /tmp/uv-venv/lib/python3.13/site-packages (from rich->helion) (4.2.0)\nRequirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.13/site-packages (from rich->helion) (2.20.0)\nRequirement already satisfied: mdurl~=0.1 in /tmp/uv-venv/lib/python3.13/site-packages (from markdown-it-py>=2.2.0->rich->helion) (0.1.2)\nDownloading accelerate-1.15.0-py3-none-any.whl (394 kB)\nDownloading helion-1.5.1-py3-none-any.whl (4.7 MB)\n   \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 4.7/4.7 MB 29.2 MB/s  0:00:00\nDownloading triton-3.6.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (188.3 MB)\n   \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 188.3/188.3 MB 42.2 MB/s  0:00:04\nDownloading setuptools-81.0.0-py3-none-any.whl (1.1 MB)\n   \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 1.1/1.1 MB 220.5 MB/s  0:00:00\nDownloading filecheck-1.0.6-py3-none-any.whl (24 kB)\nInstalling collected packages: triton, setuptools, filecheck, helion, accelerate\n  Attempting uninstall: triton\n    Found existing installation: triton 3.8.0\n    Uninstalling triton-3.8.0:\n      Successfully uninstalled triton-3.8.0\n  Attempting uninstall: setuptools\n    Found existing installation: setuptools 84.0.0\n    Uninstalling setuptools-84.0.0:\n      Successfully uninstalled setuptools-84.0.0\n\nSuccessfully installed accelerate-1.15.0 filecheck-1.0.6 helion-1.5.1 setuptools-81.0.0 triton-3.6.0\n",
+          "mimetype": "text/plain"
+        },
+        {
+          "type": "stream",
+          "name": "stderr",
+          "text": "WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\n\n[notice] A new release of pip is available: 25.3 -> 26.2.1\n[notice] To update, run: pip install --upgrade pip\n",
+          "mimetype": "text/plain"
+        }
+      ]
+    },
+    {
+      "id": "Pvdt",
+      "code_hash": "b23d6205e0dff43a7a9b1cab8bd8e97a",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "ZBYS",
+      "code_hash": "f103139918c160f7b8011d902c51c124",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "aLJB",
+      "code_hash": "4e534c092104c4c90c1278ab93710faf",
+      "outputs": [
+        {
+          "type": "data",
+          "data": {
+            "text/html": "<pre class='text-xs'>0</pre>"
+          }
+        }
+      ],
+      "console": [
+        {
+          "type": "stream",
+          "name": "stderr",
+          "text": "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n",
+          "mimetype": "text/plain"
+        },
+        {
+          "type": "stream",
+          "name": "stdout",
+          "text": "\u26a0  ftfy not installed \u2014 mojibake repair skipped. Run `pip install ftfy` for full encoding cleanup.\n\u26a0  No HF_TOKEN found. Proceeding unauthenticated \u2014 downloads may be slow/throttled. Add HF_TOKEN to Colab Secrets (or set the env var) for faster, more reliable downloads. Note: bigcode/the-stack-v2 REQUIRES authentication + data-access agreement at https://huggingface.co/bigcode/the-stack-v2\nDownloading LittleCurriculum, target 5000.0 MB -> ./training_data/littlecurriculum_5000mb.txt\n\u2713 ./training_data/littlecurriculum_5000mb.txt | 5000.0/5000.0 MB (100.0%) | 3,861,797 lines | 175,867 filtered | 2,768,361 dupes dropped\n  \u2713 verified: 200,000 lines sampled, 0 artifacts\nDownloading SmolLM-Corpus / cosmopedia-v2, target 5000.0 MB -> ./training_data/smollm_cosmopedia_v2_5000mb.txt\n\u2713 ./training_data/smollm_cosmopedia_v2_5000mb.txt | 5000.0/5000.0 MB (100.0%) | 1,409,211 lines | 16,022 filtered | 7,891 dupes dropped\n  \u2713 verified: 200,000 lines sampled, 0 artifacts\nTrying TinyStories source: roneneldan/TinyStories\nDownloading TinyStories (roneneldan/TinyStories), target 1000.0 MB -> ./training_data/tinystories_1000mb.txt\n\u2713 ./training_data/tinystories_1000mb.txt | 1000.0/1000.0 MB (100.0%) | 1,175,331 lines | 109 filtered | 114,855 dupes dropped\n  \u2713 verified: 200,000 lines sampled, 0 artifacts\n",
+          "mimetype": "text/plain"
+        }
+      ]
+    },
+    {
+      "id": "nHfw",
+      "code_hash": "950061ed21b5034668672f8f1537fc20",
+      "outputs": [
+        {
+          "type": "data",
+          "data": {
+            "text/markdown": "<span class=\"markdown prose dark:prose-invert contents\"><h2 id=\"inference-script\">Inference Script</h2>\n<span class=\"paragraph\">This script loads the trained model from <code>arya_latest.pt</code> and demonstrates how to generate text using a given prompt.</span></span>"
+          }
+        }
+      ],
+      "console": []
+    },
+    {
+      "id": "xXTn",
+      "code_hash": "ccc1222007395c4c23a2ca9294992615",
+      "outputs": [
+        {
+          "type": "data",
+          "data": {
+            "text/html": "<pre class='text-xs'>0</pre>"
+          }
+        }
+      ],
+      "console": [
+        {
+          "type": "stream",
+          "name": "stderr",
+          "text": "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n",
+          "mimetype": "text/plain"
+        },
+        {
+          "type": "stream",
+          "name": "stdout",
+          "text": "\u26a0  ftfy not installed \u2014 mojibake repair (e.g. '\u00ce\u00bcC' -> '\u03bcC', '\u00c3\u2014' -> '\u00d7') will be skipped. Run `pip install ftfy` for full text-encoding cleanup \u2014 this matters especially for NCERT, which has verified mojibake in the source data.\n\u26a0  No HF_TOKEN found (Colab Secrets key icon -> add secret named HF_TOKEN -> your HF write-access token -> toggle notebook access). Proceeding unauthenticated \u2014 downloads may be slow or appear to hang due to rate limiting, not actually broken.\nDownloading FineWeb-Edu (sample-10BT), target 4000.0 MB -> ./training_data/fineweb_edu_4000mb.txt\n\u2713 ./training_data/fineweb_edu_4000mb.txt | 4000.0 MB / 4000.0 MB (100.0%) | 899,945 lines kept | 16,690 filtered | 2,736 duplicates dropped\n  \u2713 verified: 200,000 lines sampled, 0 markup/encoding artifacts\nDownloading Cosmopedia (auto_math_text), target 2000.0 MB -> ./training_data/cosmopedia_2000mb.txt\n\u2713 ./training_data/cosmopedia_2000mb.txt | 2000.0 MB / 2000.0 MB (100.0%) | 808,583 lines kept | 176,271 filtered | 0 duplicates dropped\n  \u2713 verified: 200,000 lines sampled, 0 markup/encoding artifacts\n",
+          "mimetype": "text/plain"
+        }
+      ]
+    },
+    {
+      "id": "AjVT",
+      "code_hash": "9421950de89c80b932668244f9cb90bf",
+      "outputs": [
+        {
+          "type": "data",
+          "data": {
+            "text/html": "<pre class='text-xs'>0</pre>"
+          }
+        }
+      ],
+      "console": [
+        {
+          "type": "stream",
+          "name": "stderr",
+          "text": "[Inference] Loaded checkpoint (step 19000, 23.78M params) onto cuda\n",
+          "mimetype": "text/plain"
+        },
+        {
+          "type": "stream",
+          "name": "stdout",
+          "text": "Internet is a network of virtual computers that can be accessed and shared by 3rd parties. These networks are typically large, but some common examples include the Internet of Things (IoT), mobile phones, IoFLs (International Intelligent Directory) or Microsoft Applications Service U.S.A. A company called SATIN has been creating a network for thousands of people all over the world using its website to connect them with other businesses operating inside it \u2013 and we have just created one very well-connected city! The City on Mars is an incredible place where you could visit from anywhere in space anytime. One great thing about exploring this fantastic planet is taking out your smartphone at night! This means watching video clips as astronauts get off their telescopes searching for life before getting too close to anything else they see making sense because NASA\u2019s satellite isn\u2019t ready yet; also if there were a radio ferry system going around with light pollution coming into our eyes so\n",
+          "mimetype": "text/plain"
+        }
+      ]
+    },
+    {
+      "id": "pHFh",
+      "code_hash": "e181efcf155eef21719f37511de1f32f",
+      "outputs": [
+        {
+          "type": "data",
+          "data": {
+            "text/markdown": "<span class=\"markdown prose dark:prose-invert contents\"></span>"
+          }
+        }
+      ],
+      "console": []
+    },
+    {
+      "id": "NCOB",
+      "code_hash": "9573a4817336aebce40126886c37e8aa",
+      "outputs": [
+        {
+          "type": "data",
+          "data": {
+            "text/markdown": "<span class=\"markdown prose dark:prose-invert contents\"><span class=\"paragraph\">Now, let's define a function to generate text and then test it with an example prompt.</span></span>"
+          }
+        }
+      ],
+      "console": []
+    },
+    {
+      "id": "aqbW",
+      "code_hash": "9541eaa3a371cb54992795cf9ea79a97",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "TRpd",
+      "code_hash": "e2e709938b0cb769e6643c21951a1aa7",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "TXez",
+      "code_hash": "9efe733301c0f8f6d66eccbd830a0937",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "dNNg",
+      "code_hash": "08809d03fe76b2df1122d25d14fe74d5",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "yCnT",
+      "code_hash": "13122d375dfe64b0638eed09336e688c",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "wlCL",
+      "code_hash": "c8fc96583ecaa55247aa6caf3e855d22",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "kqZH",
+      "code_hash": "d6e07408db81621dabaca73882dcacb1",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "wAgl",
+      "code_hash": "9ea3e14a9bb177fa422b4038a8daaca4",
+      "outputs": [
+        {
+          "type": "data",
+          "data": {
+            "text/html": "<pre class='text-xs'>0</pre>"
+          }
+        }
+      ],
+      "console": [
+        {
+          "type": "stream",
+          "name": "stdout",
+          "text": "[token_train] 6 files, 17.89 GB corpus, vocab_size=12288\n[token_train] Trained. mergeable ranks = 12287\n[token_train] tiktoken.Encoding built: n_vocab=12288, <|bos|> id=12287\n[token_train] Saved tokenizer/tokenizer.pkl\n",
+          "mimetype": "text/plain"
+        }
+      ]
+    },
+    {
+      "id": "rEll",
+      "code_hash": "c72b1f6f545da1a69cbaa110db3ed3da",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "dGlV",
+      "code_hash": "b5cb5e47bc909cfe489c2c8e8038ad4f",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "SdmI",
+      "code_hash": "d10c338ecd83ffd0ca2127d594ec829b",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "lgWD",
+      "code_hash": "dc7ae64fa54ee4cc7080f818cda4726c",
+      "outputs": [
+        {
+          "type": "data",
+          "data": {
+            "text/plain": ""
+          }
+        }
+      ],
+      "console": []
+    },
+    {
+      "id": "yOPj",
+      "code_hash": "2037cfc3a9f1a07e5812e64c25fcd5d7",
+      "outputs": [
+        {
+          "type": "data",
+          "data": {
+            "text/markdown": "<span class=\"markdown prose dark:prose-invert contents\"><span class=\"paragraph\">\u2705 Pushed to <a href=\"https://huggingface.co/RuVI-AI/pragya-slm\" rel=\"noopener noreferrer\" target=\"_blank\">https://huggingface.co/RuVI-AI/pragya-slm</a> \u2014 28 files (1 checkpoint)</span></span>"
+          }
+        }
+      ],
+      "console": [
+        {
+          "type": "stream",
+          "name": "stdout",
+          "text": "[auth] ArushBuilds (type=user)\n[repo] ready: https://huggingface.co/RuVI-AI/pragya-e2\n[preflight] 27 local file(s) match ALLOW:\n  __marimo__/session/notebook.py.json\n  __pycache__/tokenizer.cpython-313.pyc\n  checkpoints/arya_latest.pt\n  checkpoints/model_step12001.safetensors\n  checkpoints/model_step15001.safetensors\n  checkpoints/model_step18001.safetensors\n  checkpoints/model_step19000.safetensors\n  checkpoints/model_step3001.safetensors\n  checkpoints/model_step6001.safetensors\n  checkpoints/model_step9001.safetensors\n  lock.txt\n  profile_trace.json\n  tokenizer.model\n  tokenizer.py\n  tokenizer.vocab\n  tokenizer/tokenizer.pkl\n  tokenizer/tokenizer_config.json\n  tokenizer_1.model\n  tokenizer_1.py\n  tokenizer_1.vocab\n  training_data/.token_cache/tokens_83a770868891c636.ranges.json\n  training_data/cosmopedia_1000mb.txt\n  training_data/cosmopedia_2000mb.txt\n  training_data/fineweb_edu_4000mb.txt\n  training_data/littlecurriculum_5000mb.txt\n  training_data/smollm_cosmopedia_v2_5000mb.txt\n  training_data/tinystories_1000mb.txt\n[upload] starting (whitelist: ['*.safetensors', '*.json', '*.model', '*.txt', '*.md', 'tokenizer*', 'tokenizer_files/**', '*.pkl', '*.pt', 'checkpoints/**'])\n",
+          "mimetype": "text/plain"
+        }
+      ]
+    },
+    {
+      "id": "fwwy",
+      "code_hash": "35717b83aa01ab740b9698a3184aeb99",
+      "outputs": [],
+      "console": []
+    },
+    {
+      "id": "LJZf",
+      "code_hash": "8ba9c69f5c18b12e4f905eb4e412f6e8",
+      "outputs": [],
+      "console": []
+    }
+  ]
+}

checkpoints/arya_latest.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e8efbc58dec6cda8fe5b14129508794260de7eadf0f40bcf746b8f982ff0fd2
+size 274635573

checkpoints/model_step12001.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d82e6da60c85524cc26c3ea9ddeaff4e20548e8f0d9ce804cf00d2d2addb6a65
+size 114127224

checkpoints/model_step15001.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1bcc5d7155d8ee0bb4dc98f96bb4d82b46137961174c2c55989d738ab932663
+size 114127224

checkpoints/model_step18001.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb95505328e07b56e55f0343a867a8588fb78acff1020a0226a179dffdb4531a
+size 114127224

checkpoints/model_step19000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6bda020f242b2643ac1f753b31e43d178f76eccc81899ffe26798a33a0189a9
+size 114127224

checkpoints/model_step3001.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e15b76f9b0706af195813d4d2e4f3265c8e0457cf24cceb35b63d9cfb147441
+size 114127224

checkpoints/model_step6001.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa6fea4642535eaa41172ff078b285d52023c7b8ab486b7f922f63c84d78e894
+size 114127224

checkpoints/model_step9001.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48b3307b583660def86ceefeda407cc13be6211a181d7fdc89b48a590cee92ab
+size 114127224

lock.txt

@@ -0,0 +1,610 @@
+# This file was autogenerated by uv via the following command:
+#    uv export --script /marimo/notebook.py --cache-dir /usr/local/.uv-cache-user --prune=click --prune=docutils --prune=itsdangerous --prune=jedi --prune=loro --prune=marimo --prune=markdown --prune=narwhals --prune=openai --prune=packaging --prune=psutil --prune=pygments --prune=pymdown --prune=pyyaml --prune=starlette --prune=tomlkit --prune=uvicorn --prune=websockets --prune=python-lsp-server --prune=python-lsp-ruff --prune=duckdb --prune=polars --prune=pyarrow --prune=sqlglot --prune=altair --prune=ruff --prune=anthropic --prune=anywidget --prune=comm --prune=ipython --prune=ipywidgets --prune=matplotlib-inline --prune=matplotlib --prune=numpy --prune=pandas --prune=requests --prune=scikit-image --prune=scikit-learn --prune=scipy --prune=seaborn --prune=traitlets --prune=typing-extensions --prune=urllib3 --prune=pymde --prune=torch --prune=torchvision --prune=pillow --prune=argparse
+absl-py==2.5.1 \
+    --hash=sha256:286e71c82c1a38e75bbcf185f9b37d0305ad7786535107cb49bf4df9ff2e1f95 \
+    --hash=sha256:721200f2f0e9960f2ca9dc3a2a706b201f5f75d812c158f059cbbe29eeafbdb8
+    # via
+    #   keras
+    #   tensorflow
+    #   torch-xla
+annotated-doc==0.0.5 \
+    --hash=sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101 \
+    --hash=sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb
+    # via typer
+anyio==4.15.1 \
+    --hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101 \
+    --hash=sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94
+    # via httpx
+astunparse==1.6.3 \
+    --hash=sha256:5ad93a8456f0d084c3456d059fd9a92cce667963232cbf763eac3bc5b7940872 \
+    --hash=sha256:c2652417f2c8b5bb325c885ae329bdf3f86424075c4fd1a128674bc6fba4b8e8
+    # via tensorflow
+bitsandbytes==0.50.2 \
+    --hash=sha256:4311f52a880b341bada639e4edd1a3c8d786830c9c93cdde29eaa1f062c8f8e5 \
+    --hash=sha256:55348a9a4a21bfd99cf8c7b32fe67b4030ae5c2a05738e03c1747f65fa6ec283 \
+    --hash=sha256:8437ab68a04ea56daf1d6ecb54230fb1d88be4b89fe2d79bc399bc0203b487cf \
+    --hash=sha256:c697963c8fda3dcd0d7ebd9b5211ae4067feef7cd06e0350d4e816a434fe683d \
+    --hash=sha256:d5772560dd94c4d9c57f50c9b017450a1707f7687bfd4b3dc86f7342aafe721e
+certifi==2026.7.22 \
+    --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \
+    --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55
+    # via
+    #   httpcore
+    #   httpx
+colorama==0.4.6 ; sys_platform == 'win32' \
+    --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \
+    --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6
+    # via
+    #   tqdm
+    #   typer
+einops==0.8.2 \
+    --hash=sha256:54058201ac7087911181bfec4af6091bb59380360f069276601256a76af08193 \
+    --hash=sha256:609da665570e5e265e27283aab09e7f279ade90c4f01bcfca111f3d3e13f2827
+    # via fla-core
+filelock==4.2.0 \
+    --hash=sha256:2ff5690882e8cdb00ef31fb3d01a3094c29f30985426c59495afb1733f3b7238 \
+    --hash=sha256:7a60906c75227cf04d0c273afadc8219400f11aeb13cc69591d4f6cdc6c8036e
+    # via huggingface-hub
+fla-core==0.5.2 \
+    --hash=sha256:5e830c85bad3d0d34677f98ac7074d08687a3756f0f0499d95ceb96eb6920761 \
+    --hash=sha256:9360fc412f784c1c8f05c320ec2902d5d968764178c9b8a92efc919e17a39680
+    # via flash-linear-attention
+flash-linear-attention==0.5.2 \
+    --hash=sha256:c053d3a75c8f5b725063f719ae6bce0f4d9dac6da32735be8ae7d485a6c9820f \
+    --hash=sha256:dcf405d81f5426393b59037097aa700d0f4a841465d5028d5aa543f4502f2400
+flatbuffers==25.12.19 \
+    --hash=sha256:7634f50c427838bb021c2d66a3d1168e9d199b0607e6329399f04846d42e20b4
+    # via tensorflow
+fsspec==2026.9.0 \
+    --hash=sha256:0f08147951c8cb31d844c3547d631053b127863b60be04cf06e121333ee0e2fe \
+    --hash=sha256:8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f
+    # via huggingface-hub
+gast==0.7.0 \
+    --hash=sha256:0bb14cd1b806722e91ddbab6fb86bba148c22b40e7ff11e248974e04c8adfdae \
+    --hash=sha256:99cbf1365633a74099f69c59bd650476b96baa5ef196fec88032b00b31ba36f7
+    # via tensorflow
+google-pasta==0.2.0 \
+    --hash=sha256:b32482794a366b5366a32c92a9a9201b107821889935a02b3e51f6b432ea84ed \
+    --hash=sha256:c9f2c8dfc8f96d0d5808299920721be30c9eec37f2389f28904f454565c8a16e
+    # via tensorflow
+grpcio==1.84.0 \
+    --hash=sha256:026d757df86c5b7a41de8200b9a2cda454aaa5004cb0c7e3374c66eb82f61499 \
+    --hash=sha256:08735e3d08d24ab3132cf87e2e5dea8746cabcc7d676c2b0b7362f195feef9d9 \
+    --hash=sha256:15bb76489e337fc492685c9758e2fd4d4ab516b901ad830dc5a91987decf00be \
+    --hash=sha256:19aaf172fc2edbefccce3f6e92c5150975dbe56c45744e9e87cf72ebdf85bfbe \
+    --hash=sha256:209414080da8c20af94df1395b635da52dd57b5edc9e917e1deca0dc1c4bb55e \
+    --hash=sha256:2c024da73b296f040b8360e60bd73a659b230093684a438da0e1260f34cc724e \
+    --hash=sha256:3de427b05f244ba2c2a9bdc67e7a6731c8340811524ecc4435466549f8af1d17 \
+    --hash=sha256:4119efa6519871719ad81f33bc95ab87857dcb1c5801f30a6e592f2c41164169 \
+    --hash=sha256:42959bd50dd660ffc3f2a9bec15a6da4f9aaa0dda555d59ff2d2e80b908456a8 \
+    --hash=sha256:47ecf0d9b81d981f07b61bd89eced9d2582f5eaacc3aaa36ad27f81aef70a27f \
+    --hash=sha256:57dc36a5ab0e676f5f6e171de2917fd0aef73f32a9aaf23956bfe19997a30bd1 \
+    --hash=sha256:5933a052946873d01a42119a05420d669bdca436aeba2d1851988ccb12b421c0 \
+    --hash=sha256:5deda5b4bf62769eb98c119cca43d40e1231e34846b19db5cdea821d446a2253 \
+    --hash=sha256:61386101ecaa096b694d0dd278caf99a56aeec78440cc17e918eef0b50f2d567 \
+    --hash=sha256:659728f20fc7a0933ed7b1945435e31014b97ab8a5a7edcbaa70da4794aeb191 \
+    --hash=sha256:70bb4ce8be0c5606bec259cbd7152374470396413b7863a658a08c849e6b29ff \
+    --hash=sha256:800b7e00d92553313c0463c200087930aa78678ec1d528193aeb50906f55989b \
+    --hash=sha256:82da34ae4f639c73ac46e521e00c0a49bf86f717b9fb1f405f133e98731e38dc \
+    --hash=sha256:986e9751d416d7a6eaa2fecdac38da63153d63a4b340ba7d624889c490451500 \
+    --hash=sha256:9b73836ba0e16fcbb57c31cf6cbc2907c8d8c790b83679df454b74bd15e0be04 \
+    --hash=sha256:9bab4cf571653a8afffb83ce21aa27b51dfe629b526b7b6adec35491fe1fc2ea \
+    --hash=sha256:b61692f0069b3eee2fc8a3a1b7f6c044df9e03fede6ce69b3ca832e1c39f26c5 \
+    --hash=sha256:b8c62888c3e49debf37ad9773e3c02f77b0c1e811f8fb0962f2b6c3bbab5b97a \
+    --hash=sha256:c5559b492007dc09b4de9b95dab05f0b5e53547aad230cf07e46c7dd017a3be5 \
+    --hash=sha256:e094dd21f077af8194923fc263cad872eaa1802bb0156fd7e5ae18e99cd86715 \
+    --hash=sha256:e41c3993eee896c617dbd8a505085d28b6e84a0445ed9a1f40f95808473cf678 \
+    --hash=sha256:e88d304f094f4937bc27ec6a435e218a084168f11ec630c8d5d39b431d08d81d \
+    --hash=sha256:e90e3bdf7b5eac005fef631adae9cafde16f922def207b80a7c46b253c18ad20 \
+    --hash=sha256:edb6f87fc60ff438557291501b3e16c7a77c3b01a52d782cf276dccc7c5dd89c \
+    --hash=sha256:f6d178ba6dc8e82976c184b65fddde172d054c17237993a3e083efe4f134d55b \
+    --hash=sha256:fff5ef3fe1bba7d6147e5f19e01e5e122ac2c076486887ddcb8d42e663400fbe
+    # via tensorflow
+h11==0.16.0 \
+    --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \
+    --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86
+    # via httpcore
+h5py==3.14.0 \
+    --hash=sha256:2372116b2e0d5d3e5e705b7f663f7c8d96fa79a4052d250484ef91d24d6a08f4 \
+    --hash=sha256:aa4b7bbce683379b7bf80aaba68e17e23396100336a8d500206520052be2f812 \
+    --hash=sha256:ae18e3de237a7a830adb76aaa68ad438d85fe6e19e0d99944a3ce46b772c69b3 \
+    --hash=sha256:d90e6445ab7c146d7f7981b11895d70bc1dd91278a4f9f9028bc0c95e4a53f13 \
+    --hash=sha256:e8cbaf6910fa3983c46172666b0b8da7b7bd90d764399ca983236f2400436eeb \
+    --hash=sha256:ef9603a501a04fcd0ba28dd8f0995303d26a77a980a1f9474b3417543d4c6174
+    # via
+    #   keras
+    #   tensorflow
+hf-xet==1.7.0 ; platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' \
+    --hash=sha256:0a9e802f33bf50c851abe45fc5380e61f959e2d369647d6742b79ad9d6c27cab \
+    --hash=sha256:19c0e64f14175ccb6a1aff69e0d2ab9ec5269a560e6687abaf2b3fa4f73de7cd \
+    --hash=sha256:2814a6e999d13464c4d679b788cc5d784eb5a4edfc638a31f10e9a11ab531ef8 \
+    --hash=sha256:2b7bb5727889b0f2436dbaaad8fc4c3e66b8240d992716989e0c086b4278b1bc \
+    --hash=sha256:4ee5e05a627f5ab5bad7a86582277d645556ea1e199903aae19e033a392aa13a \
+    --hash=sha256:57bc157b8b7fe3bee9dcb9af7f3da8de41801c3b31a9ef68a77a33c6a6be382f \
+    --hash=sha256:59fba37039233c7fcbe196817d6cdcf1b40dfb17b410f229d85b0cf0a1848da4 \
+    --hash=sha256:757168feb5679647c0bb13ee5d0faebe799c4dff9051419885a566ebd79f949d \
+    --hash=sha256:80f79dae613ce9e0ea1fd1ae15616ca9ac74aed4c770aabc199c4f03ebecc863 \
+    --hash=sha256:87dab080f8f7d32781c2586904e3603f4e60d09bfc727706c3ae419e0829beeb \
+    --hash=sha256:acc3851cf2576a8fb2ae926da863f4efabe21303cf292e9a44332802ab0dcc6a \
+    --hash=sha256:b01fe18dbbd151a2403d2c64ed30dc6547b00d6babab9a617d77c7acdb81ee66 \
+    --hash=sha256:b91569d5f1b61c34b043687da02c05dd3604f3d329e7868510bf3f7971599006 \
+    --hash=sha256:d406ec79053c0871817f700c2ac8c36ba0d87f9c34b7458b0f0063bb218b0466 \
+    --hash=sha256:e3e88a7a75d7d95cbee1f37dc31341d6201124cf21c6c4b1dfab8ccba9b09e0f \
+    --hash=sha256:fa029678be1ba7f953c409b0b27bf15cc69cd1c9b3a674fbd78856ebefca1052 \
+    --hash=sha256:fcfd6c22418e57dd5b3aea649e813b2e2cfb2aebf317b210d90f1fe4b3018b52
+    # via huggingface-hub
+httpcore==1.0.9 \
+    --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \
+    --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8
+    # via httpx
+httpx==0.28.1 \
+    --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \
+    --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad
+    # via huggingface-hub
+huggingface-hub==1.33.0 \
+    --hash=sha256:04e434b06e100eddbce9a6e817d72693a7884b10a79bd67ab48080d5c07eb899 \
+    --hash=sha256:367be21a201db9523eddf8aeac7048f2602c1b308691c97640d5e72ed188007e
+    # via
+    #   tokenizers
+    #   transformers
+idna==3.20 \
+    --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \
+    --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c
+    # via
+    #   anyio
+    #   httpx
+jax==0.11.2 \
+    --hash=sha256:540dc0bed96bd5a0d8acca15cf16198fea98ef2fc151d40bdf1ace49eb76efd9 \
+    --hash=sha256:59e7ed9bd9ace049f4856752fe580c890e0164e1c023f40fadac386408daad66
+jaxlib==0.11.2 \
+    --hash=sha256:061468eb5ac6b6213215fd1102b237bcbc93f24480c881245b73619d0c2a321e \
+    --hash=sha256:1f31d8a6b1fb13181ff1807757506d10f6e7cba6b2707f7f80f73b853cb2ff22 \
+    --hash=sha256:33d9c43f729f0e5b597f3fca73b2e1b342877e063a7e259ab57b6cc6e686369e \
+    --hash=sha256:4b553584279c42018dae4ede7963c7db6cdb35d1f36804b18a07256146c4529f \
+    --hash=sha256:523e72d6e8d188b30bee9b6c62e5e9b769ac118aab5f07343588ba64d45cf9f3 \
+    --hash=sha256:5b052113f7a8acb0b6f14b356c83bb0f1e7ba90df0b22bbcf476d3750de5ea22 \
+    --hash=sha256:5f3cac8d7030c1f80a182025d7171ea3b1245162a710f6e58d25ee3fe1748aac \
+    --hash=sha256:5f9d3833cb2bea5e346bd2138ba74992017e0841c63d890a0157a13f9cbf439e \
+    --hash=sha256:75db0922c72c511c71f9d022d7dcaca0d5492d30ad815041857fc447948c91f3 \
+    --hash=sha256:766bd90e27a0ff53b87bb9d70c73565ec80b9b1165320ccb8f72de46b85510df \
+    --hash=sha256:8df25d031827658fa434e6a8ff5c749700711fcc1553757d4bde4621019180cb \
+    --hash=sha256:a17074e2c58fa5610f6ec61955939d53e6372f59931d75eb1010926e6d81565d \
+    --hash=sha256:b2965e43db9951a4b5e11dfd437477944fa5a4cc4f4fb969725788b90ebd7c09 \
+    --hash=sha256:c2563f58a5d16f6b8d68bb65c1ba83e9ec81bea0d93cd3b1a73b0a5ffe76c1bb \
+    --hash=sha256:d100ddb54a68d59cc3103729082324fccc4ca65e1d1b03c325208c43dd5c3c40 \
+    --hash=sha256:dc154e762e6a5b19d5061e69397edccfea8c3acfbc831e487e2f6e403a218d3a \
+    --hash=sha256:ea5e58673775ec7d3f46ade4e2d5a0d1673f30bb76f2a70c9b0036eafe5e4053 \
+    --hash=sha256:edad6b2a18d63dc2964cc97a16d126dad41621edac79d651e0f6c8785d891369
+    # via jax
+keras==3.15.1 \
+    --hash=sha256:836460e480930acbd19bb7a17e62f9ecad40e8a9af9a651fc8d0586a96d27e20 \
+    --hash=sha256:92ae7c1dd7f61041953dc2d42253181ee099086f0b58ae36fcf98147a6f08a29
+    # via tensorflow
+libclang==18.1.1 \
+    --hash=sha256:0b2e143f0fac830156feb56f9231ff8338c20aecfe72b4ffe96f19e5a1dbb69a \
+    --hash=sha256:3f0e1f49f04d3cd198985fea0511576b0aee16f9ff0e0f0cad7f9c57ec3c20e8 \
+    --hash=sha256:4dd2d3b82fab35e2bf9ca717d7b63ac990a3519c7e312f19fa8e86dcc712f7fb \
+    --hash=sha256:54dda940a4a0491a9d1532bf071ea3ef26e6dbaf03b5000ed94dd7174e8f9592 \
+    --hash=sha256:69f8eb8f65c279e765ffd28aaa7e9e364c776c17618af8bff22a8df58677ff4f \
+    --hash=sha256:6f14c3f194704e5d09769108f03185fce7acaf1d1ae4bbb2f30a72c2400cb7c5 \
+    --hash=sha256:83ce5045d101b669ac38e6da8e58765f12da2d3aafb3b9b98d88b286a60964d8 \
+    --hash=sha256:a1214966d08d73d971287fc3ead8dfaf82eb07fb197680d8b3859dbbbbf78250 \
+    --hash=sha256:c533091d8a3bbf7460a00cb6c1a71da93bffe148f172c7d03b1c31fbf8aa2a0b \
+    --hash=sha256:cf4a99b05376513717ab5d82a0db832c56ccea4fd61a69dbb7bccf2dfb207dbe
+    # via tensorflow
+liger-kernel==0.8.4 \
+    --hash=sha256:9a5f184020080917111aba265ccf260547d7d93274829d6f02196d387a6a4111 \
+    --hash=sha256:e4ef9b7a30c69f207ffbe37fdf95e391ba72ac5b4ee0a333f55732c47e661750
+markdown-it-py==4.2.0 \
+    --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \
+    --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a
+    # via rich
+mdurl==0.1.2 \
+    --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \
+    --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba
+    # via markdown-it-py
+ml-dtypes==0.6.0 \
+    --hash=sha256:084dfe51a7ad58b171f05115f8226ed4233a454a1611371947e806e76f0c638d \
+    --hash=sha256:26b1f1fa4f0435a2946859823f6e2bf06796f1e9f10f5a05b08a5e3c8f46ff69 \
+    --hash=sha256:28d676428b104bb9717b0928bc5c5129f2d6b51b6727587cc4289e7bf8713cb5 \
+    --hash=sha256:317be9967fb84b0ce4e80e6b1bf71213d21971621cf6f1e501a63602a95297bf \
+    --hash=sha256:31f1ce979d31a357e95aa81812f20412c8c954fa43c44ee3ead1e1c8a78575ef \
+    --hash=sha256:3be9911d953f97cddded4b9961d7b650473b7e55806d20f6176f8356dfe7b38e \
+    --hash=sha256:3e169214e0d80ff1c038e1b3017e33c23e43bdf948d42d31de8283111c7e2fa3 \
+    --hash=sha256:488c99ab181a2f59d9ec3b12c5fa11ec904e92be2c4ba18cded54dd7501208fe \
+    --hash=sha256:573b11f3c327e17ef3826d266e676cf1149a1f3016f822a05f2306c55d8246bf \
+    --hash=sha256:57ed0d6b4ac5e7868361303a9c57fbcf63b768236ee14456f585dfcf260d0292 \
+    --hash=sha256:5e60251d32ced5598972e4d5e06a2f044341f9291402551a3f6f0ec44f9299b0 \
+    --hash=sha256:84fa136b8602c8c39e3b6cb24918960cd6f36cade7a70376f56770729cd56510 \
+    --hash=sha256:8f490c003369ce60e514a0c3b12374f05274c101fee1bead6740ec8a564032b0 \
+    --hash=sha256:9c6ad60af4102789a5c09824004beade2f7f28cd1cd581ee5c170d9dc2fbb00e \
+    --hash=sha256:b1b503864fada3f74fabf8d9fee7b4c1cbe956301e6fdece975d5f77c2fce958 \
+    --hash=sha256:b76fa1d3f92967d58289ac47ab7458ede66e6f3527fff3e59142aee57d9307cd \
+    --hash=sha256:ce7563e0b1a4482cbc1b4a6272145e54e4489e54fe7428f94908c3d87103abfa \
+    --hash=sha256:d4f1b9329a251e4affe3bb58f4d3e2db22a714396fd7ffb40d0b5db423c24d17 \
+    --hash=sha256:d574c2b28921dc72e869df248f1a278f6eee176a1f237c8642e1a71eb15f3977 \
+    --hash=sha256:de9d14748dbf3968951436ef514a29c9d1fe438aa680d110134ee2f7a9f9df18 \
+    --hash=sha256:e25bb3b0ad1217b60626e4ed45b10ca170c41d99fbe44a12bebc1e07ec4aad55 \
+    --hash=sha256:e2d6149f3a57f405bcad5fb41e03218b8373936253f23e1ca84c0108abbc3392 \
+    --hash=sha256:e74266ca8e97874a937b7646378c178025650a236584f7474d10d8086a6edea3 \
+    --hash=sha256:f4adb4af61516510d786cf8c01851a66f6d3ddfa79e1144deaa5b40d8507231e \
+    --hash=sha256:f6cb525101b6b903779188c1e9e9490c343b455ab822883e02cf01e5547338d2 \
+    --hash=sha256:fb87f46b4f7ad7b5d3ad8f4b452b024bd4229d44c8ff934798c1fe656210387a
+    # via
+    #   jax
+    #   jaxlib
+    #   keras
+    #   tensorflow
+namex==0.1.0 \
+    --hash=sha256:117f03ccd302cc48e3f5c58a296838f6b89c83455ab8683a1e85f2a430aa4306 \
+    --hash=sha256:e2012a474502f1e2251267062aae3114611f07df4224b6e06334c57b0f2ce87c
+    # via keras
+opt-einsum==3.4.0 \
+    --hash=sha256:69bb92469f86a1565195ece4ac0323943e83477171b91d24c35afe028a90d7cd \
+    --hash=sha256:96ca72f1b886d148241348783498194c577fa30a8faac108586b14f1ba4473ac
+    # via
+    #   jax
+    #   tensorflow
+optree==0.20.0 \
+    --hash=sha256:0115c3d520b217e42521cceaa4cbb2f6c4d029e82172aed7cfe6a1b9a5aa1d5f \
+    --hash=sha256:05189c3a4dda6acaa6f3421eec729238b3b8467e00345f84aee359f8881fa4de \
+    --hash=sha256:05bb1aeb21e58ace1ba086408a4c0716097e90337a7485e65939703f365346c8 \
+    --hash=sha256:0acca4b7e82b1f53b413e7c2f63fdfc32ba5465ba85bf5ee588e6323cc5666e3 \
+    --hash=sha256:0c151ba6e69331c23048e6115d849e3d42e92641d2399e35b920980ced7e1686 \
+    --hash=sha256:145053db62a8dc82c02e257b169e0723fd68d814344ef581e05988d0bcc42430 \
+    --hash=sha256:14be5cd9e9a3cadf3cc5417c260a625a4459f3833fdb99f6c5c453590da65423 \
+    --hash=sha256:1adb881b8759153f2b6cef683ad0ed1a731a11a7c1e2d8518585ff9b3e242399 \
+    --hash=sha256:29146b4fbb660dd01235c903ff30c4c56f49b6c4452efcde5ec1a0583d3e4e75 \
+    --hash=sha256:2a57982031588102dda8425b33580e1cfe073e275b622be5544baa3c3844b326 \
+    --hash=sha256:2a662b53cbd9179b17839f0a8ed476a4ea7cc5c3ae41658ccc3f8be62508a0cc \
+    --hash=sha256:2c2d083ce1b508fcb2e091fbb29dfe60d9341ea0a9c7758fc424fedce0712b2b \
+    --hash=sha256:359c2e711e4b3b1c5f169fb3b558e66d93b9eb471eb1cc523fbeefbfc487a542 \
+    --hash=sha256:3f34d8f6d154d2320fdef38c3b555621fa21d3aae9b1b75b3c8e8c1b8e6f10ce \
+    --hash=sha256:406d93dc1f6b53aceaf5df2967a9661cc85120897e2f8f239f371dd1519334de \
+    --hash=sha256:40942067473fc357b484962b68af117f79a012d94cd8fa8e744c1d27c8dd9908 \
+    --hash=sha256:44e21ac24a30ee945693c3a904af582a461d4085d8222ebe93d546381cbe4b99 \
+    --hash=sha256:4d454fb9e752cc6a7d85cdfadc3940964fb0ed2c5d0608a4ac47384ccd0476a1 \
+    --hash=sha256:4fcdce2e37e37272d058d6ac3694d3789cc8ff625c3dd5887b360bc6c47114cd \
+    --hash=sha256:5066ea9c1529d3a641913d9a7ed6c7585512be61367d8f6a588f629692fccae5 \
+    --hash=sha256:5562aeb48cbdcb295511ceb7613a0cd7b41231f9363fe8ca29cf454cfc3eafb4 \
+    --hash=sha256:589ae047414b82320c08430947d9a2f6e6b18dc6dd001a363504e96c3a96caa1 \
+    --hash=sha256:58a7608d67a3c673782e408ae40b593cac3c227e3733da5b75dd0b37b2b80c86 \
+    --hash=sha256:5cbf9b98f7603ab73ff17f72839ee7b7c314b31a4cdc249614d2582ec63b0f7d \
+    --hash=sha256:5e42a58b5fcb26d9f081a392312953bf1c8a2b4ee11ccfaa5c0000c91b40d645 \
+    --hash=sha256:630ede7bbc856ce9a6634b299e8ec3e7ea34d1793109d984c71210af0418146c \
+    --hash=sha256:635dc29eaa1a145630ab5463bd5fe51a29fd391a6b72f8a895f8873990d27b66 \
+    --hash=sha256:657aee57be88496c8a34cb12923388fe44bbe27d6849e4e34d10f05ae8504411 \
+    --hash=sha256:68339b214564651e9104317dfd407e7e8fabb000b6283a82a9536f3ddfe6f534 \
+    --hash=sha256:69b8241ddb53442a1aec51137b3d7bc5bd7228ba9b75f2d5f5d11bc04c0efe43 \
+    --hash=sha256:69c431fe171aa91239d79ec8271c74ec04986d303391359237473d344380618d \
+    --hash=sha256:7228c719c3e4b4f038ede53bd2e54053752618f7c58ee4034b45ee15a4daf496 \
+    --hash=sha256:72b5ef124b2c42aedb277a296635a3d54372268d877aabc9cc1fbddcd12c6815 \
+    --hash=sha256:74cefe145c3190d15d25edacf79b3c0d0bece94d4073246ae2f25b247d117778 \
+    --hash=sha256:7c740a784930f9263a8fe60f88875975af426c788de174ab2e725487d4cc6a63 \
+    --hash=sha256:7d013498c71a551b7da8e2adabac54a3741b728542d20c7a6c2c8951e6454dcf \
+    --hash=sha256:7fa87e9facdbf46e9298aedf1501e814335edb099a3c8b31a334eb3610128521 \
+    --hash=sha256:8052cd891c418d3f86da3082bf10ebcb7a090d82758ec39f80a2a07ef4a72d99 \
+    --hash=sha256:856194096d048b0bdf82f67071af428daefcfdce922dbce5f1f6165fb6e55223 \
+    --hash=sha256:8a4db81ae650a3af593da9c56fe22fae6e6260f742321d460ec615cfcdc85f5e \
+    --hash=sha256:8d84a5d8ba4cf53c4f1c7108ca4b7df08a3418f50851dc73aadd9a7a36695d77 \
+    --hash=sha256:94e48844309fdb24895d7f237d71264d3e13ec85374ce6624d237be379a6b81d \
+    --hash=sha256:97d35d8cefa59e3b2fa91363bbb674e970909617c81e6a8dcc53d127e7554022 \
+    --hash=sha256:97e2adcbd70eb451f6449786e89d053524cc0c7d83eb1fa469d8c3017d8c430e \
+    --hash=sha256:9a5c220a1c83575e9384305550204af07811a009de0862c18823ab74080da4c0 \
+    --hash=sha256:9a789aaf500d8cee51897c6eb4ccc5c095e00c8115ac0b74d3ddd361bd3d69d0 \
+    --hash=sha256:a79215db7e264da0d2366873ba27f5642b7995f34c13a8c1fdf54a2731f5b020 \
+    --hash=sha256:a8646f91435d9d0fe2fe2f4c408da85348d499dde846a558ec1ee9b382c73b14 \
+    --hash=sha256:a983265f90b12b727317f1cc7ef893e5e501c796df9929149bf05cc20173321d \
+    --hash=sha256:ac38b15a33a30533bd557c52b9b49c2c3cb11283f3d52474ecdcbc46d2dabe59 \
+    --hash=sha256:ad7e33c477858aa69be10dc5b992dc5a588f0d54f63a65be533d4aaa5b9d0876 \
+    --hash=sha256:b3efc6052ef752d4242b8723dd89dd2d171de0f57942508398effb3fcab10fe9 \
+    --hash=sha256:b57434c2f53e6c72a2f07a3fd6a88646aff74d7e81bcbdd8b18cdcc37060791a \
+    --hash=sha256:b769e8e6dca38359f59a7dc215de7498723f572932ee7ef4d34fcb0a2235b8fb \
+    --hash=sha256:b9518db7abe909668fb89c14d377a776c3db69d3b4e978f32a87ce6d9409e506 \
+    --hash=sha256:b99f99272e47148aa506bb3cc6b5341e98168fd673d0dd918364a5c911adaeda \
+    --hash=sha256:ba5eb068335b09b389e009fdc833d6e3930ce0092ecfac86525cd182ce7c92a7 \
+    --hash=sha256:c0c16bf65e848d2275442daa797c5e527c48272816dcd30adda2bd987c1dd3f8 \
+    --hash=sha256:c3cac6fef8a9632dc0d7f44478a1f940869c072998ba1e9647d85284aede3816 \
+    --hash=sha256:c7403eb0f2b2a060a97803a8f52904212df85ed6cff4b0eeed9cc6e38c2cf88d \
+    --hash=sha256:ce0a48b6fb4a4237ae8330abe3a30d44acec0274767194382b502bfe45eee157 \
+    --hash=sha256:cebd66fa4ed5e1c5b35ff7c8b282047fe145fec4fb643eef422666b3e2aa20ff \
+    --hash=sha256:d1034c2cc02ec12ed413727664f015a681e2eea551e05640951b05b06c07f084 \
+    --hash=sha256:d3ae8aecd7f1da2a3c798f95cb19a648716d35e73cf125f699f8a8fa564b2ae7 \
+    --hash=sha256:dc47de3e63d7964d7b2c2b9f83f86e2878d0d29b316316f5536b705bddc7f899 \
+    --hash=sha256:de0806d57c418269a62e9abbf245479eb43001b258d94675428d01517e565c4d \
+    --hash=sha256:deae3089a2384638b1dd9dd3cdb53121353c3dfb01abd5ad13b6a1e68e07ee9e \
+    --hash=sha256:e490bb81d0a6fc93a0175cd64fe412c969ba732536dfb6a278c3328118b530f4 \
+    --hash=sha256:e65c65663d1c2ab6fb0cbb490c5762a378f9d0edb305efb7dd7116e60b548777 \
+    --hash=sha256:e7258dda94e7cb7a37bfbefe74206b2c9d5edfd8d797672b4c2f29ed327a6403 \
+    --hash=sha256:ee32a100024944eea80396dba790e96607998de4cb515ab9c928eff634a5cbb3 \
+    --hash=sha256:eefb6f5cedc3ded670a79bcde0985d92e99e0c807787427c310adafafe1b9671 \
+    --hash=sha256:f2c7a010106766edaccc1209fa31316ed921a26c714f2a8f228a9e325d3818b3 \
+    --hash=sha256:f8a155152d8ad308d7e00016a23c0928f4a3feb516b9ee5af7e52761a61e25f3 \
+    --hash=sha256:f9faccc47bef4b37f53e201b708c3e3288e143d430d8a1a78e571b554dc8233c \
+    --hash=sha256:fa0340d92215264634d8acc8f2f6e44d32cfffb3ce5334bb58ffada89fc56d7b
+    # via keras
+protobuf==7.36.2 \
+    --hash=sha256:497d0463ff3316681da6c0b9e8d06cb465d61abce00b613ab42226175644d1bb \
+    --hash=sha256:89f23aa53c24553a2416fd4fd1ec06f74fa42b14b546d8883128813f775bbfd2 \
+    --hash=sha256:912c1221170e16c08d1f086762f563dd61ff83c18b5fa6652952dfaded66f728 \
+    --hash=sha256:a300819d441e078a5608c0d3c709796bb548136058fda017ae51d425b44fd353 \
+    --hash=sha256:bdb3a345d48db958e6ce1f18e508beb0cc981d64f24088427549c866cd039f1e \
+    --hash=sha256:cbc70b17ee27e28894c7fee8bb04be1abead49e936bc70eb60052531eee2079e \
+    --hash=sha256:e11e1f0180583a2af89db6a2ecd9e8dc40aa6d2988ca175bfd0e6d12ea72d74e \
+    --hash=sha256:f4fee11ec330d238b34a05c9b675f693c20415d1c5bd7d5320cc2f8a798eb9cf
+    # via tensorflow
+py-spy==0.4.2 \
+    --hash=sha256:142887e984a4e541071c99a4401ff8c3770f255d329dbd0f64e8c1dd51882cce \
+    --hash=sha256:1ccf688393105111684435f035bc14ec3f22117dd2b85b2414612cf27a22755a \
+    --hash=sha256:24720573f95230653b457671a1dcc3c5a381fcf4e92677761e328a430ad251b2 \
+    --hash=sha256:7f1c6d9b0e2379ead5bf792df43f4cf36153aa79e6dda4fb8ac7740cf8017110 \
+    --hash=sha256:8b06a353c177677e4e1701b288d8c58e2f8d4208ee81a8048d9f72ba800918f8 \
+    --hash=sha256:90e600b27bb6bb40479637baca5a5b4bc2ba3395c93d889e672315d93042c4ae \
+    --hash=sha256:a0e6f6810ccf0fc5e64e85e0182a5b626c4496eec01b14fb8755154b363a4831 \
+    --hash=sha256:aeb0323409199c785f730645e9f4bb7a7b9ca2c481f2c331a55642b5d13fa52f
+regex==2026.9.29 \
+    --hash=sha256:01000ddf0e3ffef97f2413ceb514f6313040106b6d18a03ee00a4fe35c1eb1db \
+    --hash=sha256:0166844493626c5015c6088ee15c9ca2fd060ca15b7641d1657da6a58432ae33 \
+    --hash=sha256:044265d77d94f5e3cb2fd72c76723807c429cb8c533e9d4672d0334a6f14f588 \
+    --hash=sha256:0b65c72739f981377c9c22e0c5c3cd7f42da7bd8a3c9209330fac772c7d893ed \
+    --hash=sha256:0c992c19cd45058a4b92f68f139c93db168b48fb1f322c9a7cd620806afb6b51 \
+    --hash=sha256:0cc63b5e47c12a48d90c7e9d7de6a035dd14f62868aaedbb4e0ff8ba2b8bfe7b \
+    --hash=sha256:0def9fb6abac55492d6d51cddb7225d07d6f279e774e0adc08569a54a5fc8d46 \
+    --hash=sha256:0fd2c901cc307a745ad4bc87f20060d7a0825a3371d1e93488af22e7a387f78f \
+    --hash=sha256:121a76a0985db80ceae9e171c337f8c927868e37d01b54e3ce87bc87f9c6a208 \
+    --hash=sha256:19959129885356df0e97556856f77eb2888380dac18bed075a7c05c5128c618d \
+    --hash=sha256:1ba8c6a416569ce0d37e83e28a254a61dc99a419084dfb6476cea02d997f74fa \
+    --hash=sha256:1c2a0026062abcc321a53db4a185ceba0b59a66b5d37b0808917a88b55a5257f \
+    --hash=sha256:2089fe39c406784d90101c726755ffa1497bb74638fd434300d2b88006186de8 \
+    --hash=sha256:26ec4ccce55aa533fbd603d08911b01101a8fcfec987845ac3ae2c7087b2bde3 \
+    --hash=sha256:2f7f7aa47b229f2b39a2ae2596d2ad5625d77b5eb9856fac2dab3eb506cdd0a0 \
+    --hash=sha256:31b003f9a070335e2a8233ee9b14a3ca8e6d792012ae011f741bf0aaf11744c5 \
+    --hash=sha256:32ab11df9677ca80bcbb5fe4eb1da9109a5019239a054836efc6fa1c64e683cf \
+    --hash=sha256:33026515aebc0e70d1c89978e53e8d695d35d9e472f8d5b34465ba3c74028650 \
+    --hash=sha256:34b6925af9853bf461950e6508910f179fd6e9b1a7ec8548e069606b7e51a26b \
+    --hash=sha256:3e778bfccd63075167709136afbc251c1f683758d5bf49c803c60ac3f894ce6b \
+    --hash=sha256:3f1e6cb402a89457582cd696f982559217d13484a193202c394015297968c86d \
+    --hash=sha256:42e82e578c904445d4c8a35b8f28052cf567593215fa5db06266fbc6f77aaa2e \
+    --hash=sha256:4408b2b27a95ca8cc48b7411945753773353b5c93b307754781086c99d3a576f \
+    --hash=sha256:446654b29bfaa30500d80947eda42cef1449dc8a87f4e3cf061cc8485d3a1f0b \
+    --hash=sha256:4d7d93613b01b0199961330e49cfc52d479b3d5776c56c691db31130c0a07d91 \
+    --hash=sha256:554bffadcbcb6d5f4e5fb10a61cc52084b9a63d1dab5f10bcd2c4343972e8e2c \
+    --hash=sha256:61956f074ecd123f55adca68ee3eab46e6a07ad3f8e64e6db95dfacb444f55c4 \
+    --hash=sha256:65b408d8fcb273e3499e7ef2ce796810da1becd208c7fb4373692a242d79d461 \
+    --hash=sha256:686ac5350fceae63830bb98805fcb8039325bf4c06d9f6f048ff65229d5bffa5 \
+    --hash=sha256:6a1a824fbed817e0a891103886b68f063b1e83cc51bc97192a90a60195a9291f \
+    --hash=sha256:720537c7ea6f80dc61913184edb0ce2497a306b39ef19f28505b322553d52bdb \
+    --hash=sha256:724184b4aafed865e4f13ca313fdcb43024300c028ec67319cfa16847d84685e \
+    --hash=sha256:7c03031610e3e6ed1768a2b7a8fc84637c1257b50c5eacaf094c6e17a84fc563 \
+    --hash=sha256:80a5ea3b4fd9d6a5b9a44f7976a9acaaab35aa3c1f6b29e5bd857dfabaded223 \
+    --hash=sha256:80c7cadd3fd2bfde5df8aa0787e315812cad0c313a753095d02f4c2b6c01677b \
+    --hash=sha256:80ea96f5c1a30bf09007d48466521d9c294bebe197c708c3359096e3e3691632 \
+    --hash=sha256:864e9b87ac33c3fb9fb4ad48166d4fdb579c351d5c77deb0d34bccb36a775cd9 \
+    --hash=sha256:888d60953908dcf761aa320c3e390ab8556efbdb551ace63921de90f6ae0848d \
+    --hash=sha256:8b5fcc4771732191b2b7d1dd68d8f0353f47f8d90b6150f6dce58bf1112442cb \
+    --hash=sha256:8f39588af4731c8923c26810eb3b33f76f17633985e40f59c3cd45a33805a895 \
+    --hash=sha256:9173db3be74a35cb6731701094b98120f7ee4876a287882a59cdea1fa7da342f \
+    --hash=sha256:92f05c9c42bde5785dc48770bc2194d9f7442544156f951e19cd31b096cec562 \
+    --hash=sha256:951733b1bbdb71e377cec567b409f1a7881b47cfcad84121aa74cb575fa425ea \
+    --hash=sha256:957bb708e8057ab1649ba566456429d691ec9b90d1c9ad1af1ba7ffbbeaf05f2 \
+    --hash=sha256:a5300757f8a68f5b6cc33f57338d72a0e3589c5cc9ad5f8504ea06f028be582a \
+    --hash=sha256:a540abfab208e1b7ef2df231c40ef3b6cbb30a0aad6204e9b6a81c10a6794628 \
+    --hash=sha256:a64b85a4760337cfefdb27d42da6ed8b58e8cde3f2d57b6ef43e76ef6ea9ef47 \
+    --hash=sha256:a655d34b2a6943af32401f3d94f72e9d731f6ad16285815550bf2b4ee69d420a \
+    --hash=sha256:a714befaacbd10092ffe4cea0d3c5f008fb9efe9bc322c715bcdfdee414b9a3d \
+    --hash=sha256:addd736a0547d553283adaf4e05d7104e7f2c7b0b092e9b4d28756825f14531f \
+    --hash=sha256:b11b589e00095ec69cf79841a76360f9b079e95b0368a25b5ebb951ab0c157ff \
+    --hash=sha256:b3e445b66c80b4eb4234e855ce94d9adc183eedbd632816228d89930b91b2c5b \
+    --hash=sha256:b89efc38431793d28b7cd91227e2f952ad7c48df19132b17f43a5fec3c14143b \
+    --hash=sha256:b97a38fb4c732b6832db6bf108963adbcd82ef1268ba2025dce390f45af75efa \
+    --hash=sha256:bb90e7177944b6684738c1fc36aabd2dd00d1de3be7dbe09f91e196f1bc0dc81 \
+    --hash=sha256:bf3c49863c23a1ad6da9c30351aed6cff8d5ddbeb63c5c8420ae54e98c7d0138 \
+    --hash=sha256:bf48516e35cf848390ea68850aba53e7c333720d2945b4d2c25b69fc5171723f \
+    --hash=sha256:bfc71e6d970419c1309b3640305298643e2a734cad3f7cfb6d2ddee4175ab53d \
+    --hash=sha256:c03c6eb6ece86dfdcbb34799efaa339b093132e1aceed491ba5e08fe06cdf699 \
+    --hash=sha256:c3589f40749acce747510bf5d589d54e376cb0930ea58b35effac97e5312b0c1 \
+    --hash=sha256:c4e38dd8f39c43a91d2410ad2b85610701b0979342c3df1d69eaf8e838c757d8 \
+    --hash=sha256:c6c8fabf1dafc1f1ddcbb67896d3f93efb092e8c4b6322d7389b944e76a484e5 \
+    --hash=sha256:c90fcf7804ea0a54b896ce0f2b9565350220b8d4890fd0db461a476a4c687963 \
+    --hash=sha256:c9b602fae1e00b7c035d661ce85575365719192a7b46784bd71cf64c68053aa0 \
+    --hash=sha256:d06fcdecc10fc7954d7c8f27a03c96055fe525274dc84a7b0dbdc3d6b9e03dab \
+    --hash=sha256:d49c18f1ea294cf4adde2e5ac256e98c82ea9d708462ce4bf799dffa7cfe8a2c \
+    --hash=sha256:d7cab119d0df0b9413f106b4d7fc34f2872d3574ed3806fb48959c830b1537da \
+    --hash=sha256:d9b77b25b4f395f92de6099ab08e8ae2bc7e51dfe157f22900902243a5cc90c7 \
+    --hash=sha256:ddfa987262763c3c22a8367d2a49c244b018a74c3a8e3ab1a864119ad45c5633 \
+    --hash=sha256:e1172147d28d8fbcf8cb8d26c41506169f5ad8fe9ec969cb116835a19d4d8eca \
+    --hash=sha256:e11edba5bc344a32b029a7af9d4b3173982dd79eeafa0b9dbd787364414b0509 \
+    --hash=sha256:e2c89e9b762c57f59d5e99ee8b20202adb892e35f8d3485741340999ca55058e \
+    --hash=sha256:e31f72490b7c12f7790e1e25c3afffd20503ee1bfb43461d7838b871ff244b19 \
+    --hash=sha256:e8c65ef3862a8ad6e86492b6ed9327805dd66904c012bd3649dc67d822ed6c34 \
+    --hash=sha256:ebb8912f565b8cdbbf27debfe00df04202c20e2f651b9e32767930c5eace3621 \
+    --hash=sha256:ed511a0708e2297e1d6431e7fb217e3402791e491e02da800658ace4973df1bb \
+    --hash=sha256:f0fe9834e5aeccaf19a0d8feb296d66a24be1a7c9922002f842a682cd5abb787 \
+    --hash=sha256:f1a0d5117230dd46b399a30a38afa44f79c99f3168988fdc4f425c3f928b39df \
+    --hash=sha256:f37964e4a5e993d2fd45147741e9dff7f34a2d8c00ab94c4ea0514a4677f959e \
+    --hash=sha256:fb99cc9d45f48895d9d67f6a0b8a57f08d39c174d9f25ad97a313e0470267b1c \
+    --hash=sha256:fe3fa1dd453ed5c7f5ea23a26218329790ed7197a99b90e94330e313959a7f52
+    # via transformers
+rich==15.0.0 \
+    --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \
+    --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36
+    # via
+    #   keras
+    #   typer
+rustbpe==0.1.0 \
+    --hash=sha256:02d123e72fe9253c92904bfe2ba35afc816576b2cdbb432a96001e75bafb888e \
+    --hash=sha256:18765f62ac579a9ff9e89c611f9c9b9e46bd1adde9be3f59c00b6eb4e1f28b3a \
+    --hash=sha256:2b0b77591c9e836df41ad0b30be9ec519a708c477cbf82eedaf839e7a9b10101 \
+    --hash=sha256:3eeb8efae3d10a3b6640a1e2bdc7f1e55a15f867bdae9efb3d8f0757b01d9d3a \
+    --hash=sha256:6a59c05a8123d3a8e8815106fd1938a9499d4fdaf5cf00351fa7d3b5cc4f8ad6 \
+    --hash=sha256:88d1482ccadf5e29b524b13740b3a5e1f4e454a048684885d894fd1a9930617a \
+    --hash=sha256:a0df5172a813c982d31673d9de32dd053ddbb64ced2b97709a85d2e3c6a6cd28 \
+    --hash=sha256:b1229e70c2d091faf8c0a50951e2e734b3b810d1d2b7677cd49d86dc3853c283 \
+    --hash=sha256:b5ceb789bb93a82547c0ed7277ecc01047eaf0eeea6bbc0a21420e65e5fb553a \
+    --hash=sha256:bc0a8dd8b30860e3a4889ab7cf7c04a2614f8fc77c191efde1500aa054484efa \
+    --hash=sha256:c525072e521a5cf729474a0ea6c83b1b16973b877098ee7060eac4bbacd46c7a \
+    --hash=sha256:cc5bebc9990071e400bbe304304af3d5757522bb2a1177e2c3517f11ad28f0eb \
+    --hash=sha256:e0c9216f9e38558f0939f6e34cc78d5517d7a02026c1a35b271ca82e9b522539
+safetensors==0.8.0 \
+    --hash=sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358 \
+    --hash=sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f \
+    --hash=sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d \
+    --hash=sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d \
+    --hash=sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0 \
+    --hash=sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc \
+    --hash=sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235 \
+    --hash=sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98 \
+    --hash=sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4 \
+    --hash=sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846 \
+    --hash=sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca \
+    --hash=sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0 \
+    --hash=sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25 \
+    --hash=sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452 \
+    --hash=sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d \
+    --hash=sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78 \
+    --hash=sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774
+    # via transformers
+setuptools==84.0.0 \
+    --hash=sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670 \
+    --hash=sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73
+    # via tensorflow
+shellingham==1.5.4 \
+    --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \
+    --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de
+    # via typer
+six==1.17.0 \
+    --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \
+    --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81
+    # via
+    #   astunparse
+    #   google-pasta
+    #   tensorflow
+tensorflow==2.21.0 \
+    --hash=sha256:76cccbe0a95d9392dee1ae501ae0656b6c73c1cac29a7f8f32d570e0670863f7 \
+    --hash=sha256:a10abdfb8b1189210c251021a3f153b7ccc52a8e6521351f3dc3331e8ba593e0 \
+    --hash=sha256:a145ed46c58192b7c3f9916d070caf4f6afc6dc7e5511f83dd97677f4c4947f4 \
+    --hash=sha256:e9d8da8dcab9650efb45f032ba70af2f016f907e6e0c6bda29dd101bba945406
+termcolor==3.3.0 \
+    --hash=sha256:348871ca648ec6a9a983a13ab626c0acce02f515b9e1983332b17af7979521c5 \
+    --hash=sha256:cf642efadaf0a8ebbbf4bc7a31cec2f9b5f21a9f726f4ccbb08192c9c26f43a5
+    # via tensorflow
+tokenizers==0.23.3 \
+    --hash=sha256:114e2b55ed177179d59f4ab98200a4471e11e78f9e4b5a922d146740f96fcf52 \
+    --hash=sha256:1554a6eed34d9d6a78d23360f4e06df8dffab1ae08c7e8488e0b3e3b36cc266f \
+    --hash=sha256:1ebf28794e7e4954e20a7f70fbea410b2d1f0418f7dbbca97ca384fcfef38c25 \
+    --hash=sha256:1f0823bb00c5fdc98e487354d54dd55a03848d61a1a0bf29a68c77f24f3b26c3 \
+    --hash=sha256:2a89614730d7b80940a5d2ed9320e1ec8add5a745c6151d8d05071b7215505b6 \
+    --hash=sha256:376851d22bcf9d650a5c3090bb83e6cf9e895fbf0595369fa4cd43c1f69b5f87 \
+    --hash=sha256:5cc24bb457dd4a8af89c8fcb40074d570129ec473df2a866c276ee55db4749d7 \
+    --hash=sha256:68649e97d5b43c44c031d8d848874a6eecae8f8fe40ea989aa777a5a83aca716 \
+    --hash=sha256:7e48734d2de9260d86f03ab056d2cfeeff3869f61dbd49aaa15a2793b5f3458b \
+    --hash=sha256:82eb480f6f1c21cea3349dec32cf1a6384c6c1e775f00f83b0d51197bc013687 \
+    --hash=sha256:84513ef0aeb8bf8f4ea11a2e8a7ac163ec5288aa115e649a59b470ac5c3107df \
+    --hash=sha256:9d2b5c97daf61688c2ad1803ca851800feaba50fb68d5821779e9ea5880d968c \
+    --hash=sha256:a4fbb3662f9f59d199d61338e54b4bcc11d07ebbb1aeb3540dacb2be9c521cb7 \
+    --hash=sha256:acd5c57b4bd3e56e246e2731a3a3a6825a7a7d89b7e3b761ba80bc521710f04b \
+    --hash=sha256:bf501c40b72d2d5c8623620210430e9cac1ce47a46e45b34107b70a1557d46b0 \
+    --hash=sha256:c64a0713180ff16829d4e7f39a658b77ea11443af4e1aa46523692943c9b1414 \
+    --hash=sha256:cded33237c77caeef62944d32aa9a7ef42bdce2b3497e18d137e072a8c4be438 \
+    --hash=sha256:d3407fb7b9c4d75dd68850ffd7180bc0a5d2dbaf0762d888e612f31fec3f9c6b \
+    --hash=sha256:ddedfd4b3b4be6be24ff6ca645c4a37fddfd305f6f3e354c54cf10b715c48215 \
+    --hash=sha256:de536665495cb4b409d25bade41963f801aff4225c19a6b804b048f7d14e34c7 \
+    --hash=sha256:e05ab7baf7f47b406a95fea6f3b0a484b2ddcd9e1d14b68844c457eb755085a3 \
+    --hash=sha256:e88646b8580c5ad7f4361477f1298e9cc01771a1ee9aecfe32c47b8ff614cc38 \
+    --hash=sha256:ec82e80e65a862275b97c3d90b7a523df8d9519ee48aeb4e9625b2cc909274e0 \
+    --hash=sha256:efa3d7318406b4d115dce61ad5061953f1f44b128e79c020ce4615d763e23b6e
+    # via transformers
+torch-xla==2.9.0 \
+    --hash=sha256:e85ba39f12758a8b3465df28389f6899e31522bc64cc58dd108fbbfeedafb776
+torchao==0.18.0 \
+    --hash=sha256:5c2b4485341bf28b7fed2c4fc95b9f298e209f41685350f067de85527a05585e \
+    --hash=sha256:6540b148e40ba81cbd4de86392225a076a1591146e9cebb099b3b234ba9feebe
+tqdm==4.70.1 \
+    --hash=sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73 \
+    --hash=sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4
+    # via
+    #   huggingface-hub
+    #   transformers
+transformers==5.20.1 \
+    --hash=sha256:25b30879e4c243f00630c1079171ede6d4f6bf76e68f978acd47628a640bc8a0 \
+    --hash=sha256:5f51ce444193e5d0a751e3fa489ddbc2224ff12f6675a5cfefd4044b09757c19
+    # via flash-linear-attention
+triton==3.8.0 \
+    --hash=sha256:1b84e7d512490ba529111260fa6f7cad8b254a6bb5fbdf41d5ef9a5e57f52d0a \
+    --hash=sha256:1f0497218e26b7d79773ad9c2a3fa3b539ee69f587a13fac2e552b1d322a8015 \
+    --hash=sha256:1f6b48d0591929a3867973acac3dccd4e058585f91bfb41022de496c9ffab304 \
+    --hash=sha256:387dae4cb0089a7b6ba1a428ae0782b65c4c58f57d94617cb22ca8593d8ccbca \
+    --hash=sha256:74217bb56ed8692759227758e4c4b3bd2d608a209c1a7a081bf361fb4c2c1bf9 \
+    --hash=sha256:b7004666652f500ed854a86988e4b3d69d247188b5d2092b5df1e44f4a954099
+    # via liger-kernel
+typer==0.27.3 \
+    --hash=sha256:d0396f770a560ab1b0a8504e13b5f254b728cedb05c61cf0359e944e50ce8901 \
+    --hash=sha256:e50022f28b82a86313e54501317a1db64bf8f8d036ff8cfe5ca7e47675454aff
+    # via transformers
+wheel==0.48.0 \
+    --hash=sha256:3217dcc807155e45db462d7ef2431f5ddda0d7273b700d05a67b271ceb1287ab \
+    --hash=sha256:94800765601e9171bf5d58d066e640662842bcedcbab982b2c90787a2c987322
+    # via astunparse
+wrapt==2.5.1 \
+    --hash=sha256:03aa7d2256309b57ddbf317bff2cae5f47e50ea9ae8d582780ebe0b554347b42 \
+    --hash=sha256:051220e5071fdfb1a6678707c8abb7bbf4824d40f99758394b2b4d64855fb284 \
+    --hash=sha256:0591e6eace0d186c9ef1ecd1244be5a04e98041424cfca425b684ffe4f0d8030 \
+    --hash=sha256:094b847491b813b6e6c1775e03770930d75078c0821adf929ac712830951ef25 \
+    --hash=sha256:09b1893ee4063706574c1813abf479b8b51926633fbdb6f96aab8dc7b0976668 \
+    --hash=sha256:0a526227efe17dd94bd16b123d170f879bce42c15f10eb92495a745f54caa943 \
+    --hash=sha256:0c9480bdee340a1602cae5a777146ab4be3e384fdcb569fffdf8721032314645 \
+    --hash=sha256:129cab3c7b21e68e693c2819a95c47f3b1c41a834b931154688c83b6aef6bdab \
+    --hash=sha256:12d3d2b9d6553df6e2421ab99e1cc5413509076788f57fcb3169f5ce100a19d1 \
+    --hash=sha256:1425fcf0e70b27053bd610d57bae975856e7897e3f6ba1456d2b80b9d7fd15d1 \
+    --hash=sha256:183bf0bb893f783c9d22f953cb01fababb9f618e098763f8e66337b575b0647a \
+    --hash=sha256:25eb4d928a9abeaf70ca786a35861b46d1ab37cc4ce49ea70a070dacdead4dfe \
+    --hash=sha256:25ed8b1b39234140d5b5c6a273130c7595e0abece417c3ca3cb378fcea5cd0fe \
+    --hash=sha256:26313f38d18d40a9975123a4ebff9da125ec63ab9ece4f05320a3d8d37d2c1fe \
+    --hash=sha256:26d8ea2ec6818aeb656bd8a9e745a6f1fb0edfcd8f54291ccd94f62eb5f5e3bd \
+    --hash=sha256:36d7d0ad593c4f1a651e4032de834db59aee1a929ee396cd483895b673328e51 \
+    --hash=sha256:380f72610181883f66b41442cfc7c0f7552b42169efb2113def26e6380013d37 \
+    --hash=sha256:3cf273b7e8d2038abb7f0a8c6550aff4f617b9d486a9965c8e8acc96a3a04de9 \
+    --hash=sha256:4b3f410c416752e1dba53d361e2e6562f22c2c3ec855740dfa5836e061b22571 \
+    --hash=sha256:521bd5ef2a33171fac08a0a302d51a983c19c3519406c1ee8da7ce29285488da \
+    --hash=sha256:5b53000b424dc2133eaaf22838a2352d3497f5d7c2e7d9a2acfe675ab7225bb1 \
+    --hash=sha256:5be9816d9de88f02fce23cf55f392403411d9bd9c7ae57fdc965a43b22e2de5e \
+    --hash=sha256:6201c7e122f40060a9b50696d80deec8f93b1a235ec0443f51d7a8a42f7044a6 \
+    --hash=sha256:6405ff2160af9d59132ebb076eda0304db44d9d09809582932412ef7c0788a36 \
+    --hash=sha256:69fd0fbb3daf7c8c6f5e062847a0061f880f347374d74cf1daba57220fb64cd0 \
+    --hash=sha256:711e73da3d7983547fc9dd208973b6b0c52640822f5d477910ba24622df6ba64 \
+    --hash=sha256:736c1de0230c6d24327b14684794214167b2c5ebb6332e28a10f504641b600df \
+    --hash=sha256:76f230a9b07e3cb66646d265398f579abb6128b1bb4cb97c74b1ae5d09e96f31 \
+    --hash=sha256:8078186f719a92693199f1e06c4ec72e1e6d374c2e459da18ed5c39d6966d727 \
+    --hash=sha256:89d9a8607b7028054bb6fd01d437f205534a5d59d53c3665d15949a99a2fce0d \
+    --hash=sha256:8a7c078323e6e1534968cb85488c5eb7ee2b9bbd0f8a291095213a763da40dab \
+    --hash=sha256:9a34640eb6295f33ca23462977de275fe8f3a50ab339b8918b96d69a7451e2e1 \
+    --hash=sha256:9bad4dbb4e61624fcce5f301e37f9e743ecae4f1259a3777b3207eb7eba3dccd \
+    --hash=sha256:9f0750cbc2e29e4f3c9529d3587d4e7ed8f60638ceafb80b87a95833b0c5acd9 \
+    --hash=sha256:9f437dd704abc4ee1bd03bb2d796d362d0e75915e8f3113a7900b3b7ec5f8b47 \
+    --hash=sha256:a18e63910252eb75d8806b4baefbc3a03612502f63eab042e3741b00b719f043 \
+    --hash=sha256:a1e823aecb3746b8f9e0aee2e1413887871ee2f5c502a3e0ef8d466dbd4adde1 \
+    --hash=sha256:a424e8a9776c06aef6313af1d0e3fe6e0838af4241d0c09eb0a3b46f2c9a5ff3 \
+    --hash=sha256:ab6db7d2a18d366cc57c2228253cf26443190aba0a6dd0939b3c1e8ac6e29e2c \
+    --hash=sha256:ad81bf81b0a0b6c6ec74169638202851962843e86749570c463eecc55072f93b \
+    --hash=sha256:b238e955ba34ef2b8897f358b7b868b41b9a02ffd338014b62985fa91898cc4a \
+    --hash=sha256:bc5c0203d383403043fb86c964bd0bab4fcbfb26004ff4bb9c6d02ebc1d608ae \
+    --hash=sha256:bde5d1b37101b1e9dd3da1f35072e2e7028e9c5e3511f7d76d3fdd4d071b7663 \
+    --hash=sha256:c6e6c226b1ca5402d7ae5fb34a0d21f1b49124fe4200e5884d1e19e53c47ac1d \
+    --hash=sha256:cef2a8f006410b6134a0d273ec037fea8cc7a6a914f1bd7555ad9788ad788c6e \
+    --hash=sha256:cf63fffcdcd8c60f223d3967bb92cc4fc2e8b46f09e75b67a6a75e6f47c0fc43 \
+    --hash=sha256:d5b665a43fe0d3b390cbdd3c003d61c92fa07bd5e3fb1ed3f47920c2d03cd9fd \
+    --hash=sha256:d800c7689154622b0ba2922ceca44a3cf2ef61c3b9a4c4eeb1d8b3050d7ededa \
+    --hash=sha256:da847332447db5505162759a4cd5ac374eb8b74841fe97a98ef3de14edd2586d \
+    --hash=sha256:dc401274fcc7b15b3b2c12df2ff34024a11925243a7d3daee91c6d7d14f9addf \
+    --hash=sha256:df6e3a36170cda0d313be50fe5065948e7f12f3a181b38cbc262e9f2ee4824e1 \
+    --hash=sha256:f1630201b0e2a96bb26304b7adfbd91a4ef486abb5a4c48377444a0bed749f37 \
+    --hash=sha256:f280c115ea64eff3dcbd68a668ce3f63476a4ba386bbabb318017e286196ea2c \
+    --hash=sha256:f595bb0185aab3e9dc31950c95d914f56ea8278810c3b928f3426e12ed6d27bc \
+    --hash=sha256:f98eaf784cd12bc69c77af398084174531007cd81849c962163ccfc6e791f3ea \
+    --hash=sha256:fcccaa1484f7dd1091602970988ab741491f9f974013c844f70e45ac1196b80d \
+    --hash=sha256:fd3f878a4aac3c262447ddf43c5f4c18fc67dfc3ba69c4fb1c7a4c4af96abe7e
+    # via tensorflow
+xformers==0.0.35 \
+    --hash=sha256:57381ce3cbb79b593e6b62cb20a937885345fad2796de2aa6fbb66c033601179 \
+    --hash=sha256:ccc73c7db9890224ab05f5fb60e2034f9e6c8672a10be0cf00e95cbbae3eda7c \
+    --hash=sha256:f7fc183a58e4bf0e2ae339a18fb1b1d4a37854c0f2545b4f360fef001646ab76

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:547868b9fdc9e916c5530467a6bc1353a08a7e1934b0671873b0d64caa9f5bfd
+size 447723

tokenizer.py

@@ -0,0 +1,432 @@
+"""
+tokenizer.py — RustBPETokenizer adapter.
+
+Drop-in replacement for the previous SentencePiece-based TokenizerWrapper.
+Exposes the SAME public API so train.py, _tok_worker.py, and finetune.py
+work unchanged. The only pipeline change required is that the on-disk
+tokenizer artifact is now a directory containing `tokenizer.pkl` (a
+pickled tiktoken.Encoding), rather than a single `tokenizer.model` file.
+
+Set TOKENIZER_MODEL_PATH in train.py to the directory (or to the pickle
+file directly — both are accepted).
+
+API compatibility
+-----------------
+Public attributes:
+    model_path      str   absolute path to the .pkl (hashed into fingerprint)
+    vocab_size      int   total vocab size, specials included
+    bos_id          int   id of <|bos|>
+    eos_id          int   same as bos_id (see Notes)
+    pad_id          int   same as bos_id (see Notes)
+    unk_id          int   same as bos_id (see Notes)
+
+Public methods:
+    encode(text, add_bos=True, add_eos=False)                -> List[int]
+    encode_large_text(text, add_bos, add_eos, chunk_chars)   -> List[int]
+    iter_encode_chunks(text, add_bos, add_eos, chunk_chars)  -> Iterator[np.ndarray]
+    encode_batch(texts, add_bos, add_eos, skip_errors)       -> List[List[int]]
+    decode(ids, skip_special_tokens=True)                    -> str
+    decode_batch(batch_ids, skip_special_tokens=True)        -> List[str]
+    save_config(path)
+    from_config(model_path, config_path=None)                -> TokenizerWrapper
+
+Notes
+-----
+pad_id / eos_id / unk_id all map to bos_id because:
+  - tiktoken is byte-level, so <unk> is never emitted
+    - rustbpe's SPECIAL_TOKENS list has no dedicated <|eos>; this adapter
+        treats each input to encode() as one document. For iter_encode_chunks(),
+        one BOS is placed at the start of the file and one EOS-equivalent BOS is
+        placed at the end, matching the flat SP-era stream contract rather than
+        nanochat's per-document stream.
+  - finetune.py needs SOME valid id for padding; <|bos|> is the standard
+    choice and is filtered by decode(skip_special_tokens=True).
+
+encode() uses tiktoken's encode_ordinary() fast path, which does NOT
+interpret "<|user_start|>" etc. as special tokens. SFT rendering must
+call encode_special_id() for control tokens and encode_ordinary() only
+for content. This is the same contract as nanochat's trainer.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import pickle
+from pathlib import Path
+from typing import Iterator, List, Optional, Sequence, Union
+
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_CHUNK_CHARS = 500_000
+_TIKTOKEN_THREADS = int(os.environ.get(
+    "TIKTOKEN_NUM_THREADS",
+    str(max(1, min(8, os.cpu_count() or 1))),
+))
+
+
+# ---------------------------------------------------------------------------
+# Helper: locate the pickle given a directory or a file path
+# ---------------------------------------------------------------------------
+
+def _resolve_pickle_path(model_path: Union[str, Path]) -> Path:
+    p = Path(model_path)
+    if p.is_dir():
+        candidate = p / "tokenizer.pkl"
+        if not candidate.exists():
+            raise FileNotFoundError(
+                f"{p} is a directory but contains no tokenizer.pkl "
+                f"(expected {candidate})"
+            )
+        return candidate
+    if not p.exists():
+        raise FileNotFoundError(str(p))
+    return p
+
+
+# ---------------------------------------------------------------------------
+# TokenizerWrapper
+# ---------------------------------------------------------------------------
+
+class TokenizerWrapper:
+    """Adapter exposing a pickled tiktoken.Encoding behind the SP-era API."""
+
+    def __init__(self, model_path: Union[str, Path]):
+        pickle_path = _resolve_pickle_path(model_path)
+        # Note: pickled tiktoken.Encoding. Only load pickles you created.
+        try:
+            with open(pickle_path, "rb") as f:
+                self.enc = pickle.load(f)
+        except Exception as exc:
+            raise ValueError(
+                f"Failed to unpickle {pickle_path}: {exc!r}. "
+                "This adapter expects a pickled tiktoken.Encoding "
+                "produced by RustBPETokenizer.save(). If you have an old "
+                "SentencePiece .model file, train a new tokenizer with rustbpe."
+            ) from exc
+
+        # Validate it looks like a tiktoken Encoding before trusting it.
+        for attr in ("n_vocab", "encode_ordinary", "encode_ordinary_batch",
+                     "decode", "encode_single_token", "special_tokens_set"):
+            if not hasattr(self.enc, attr):
+                raise TypeError(
+                    f"{pickle_path}: loaded object is not a tiktoken.Encoding "
+                    f"(missing attribute {attr!r}). Got {type(self.enc).__name__}."
+                )
+
+        self.model_path = str(pickle_path)
+        self.vocab_size = int(self.enc.n_vocab)
+
+        # BOS is required. If the vocab lacks it, that's a hard error.
+        self.bos_id = int(self.enc.encode_single_token("<|bos|>"))
+        # EOS/PAD/UNK reuse BOS — see module docstring.
+        self.eos_id = self.bos_id
+        self.pad_id = self.bos_id
+        self.unk_id = self.bos_id
+
+        # Everything tiktoken labels as special — used by decode() to drop
+        # control tokens when skip_special_tokens=True.
+        self._special_ids = set()
+        for name in self.enc.special_tokens_set:
+            try:
+                self._special_ids.add(int(self.enc.encode_single_token(name)))
+            except Exception:
+                # If a special name isn't encodable (shouldn't happen for a
+                # well-formed Encoding), skip it rather than crash.
+                pass
+        # Always include BOS even if special_tokens_set was empty for some
+        # reason.
+        self._special_ids.add(self.bos_id)
+
+    # -- single-sequence encode ---------------------------------------------
+
+    def encode(
+        self,
+        text: str,
+        add_bos: bool = True,
+        add_eos: bool = False,
+    ) -> List[int]:
+        if text is None:
+            raise ValueError("encode() received None")
+        if text == "":
+            ids: List[int] = []
+        else:
+            ids = self.enc.encode_ordinary(text)
+        if add_bos:
+            ids = [self.bos_id] + ids
+        if add_eos:
+            ids = ids + [self.eos_id]
+        return ids
+
+    # -- chunked streaming --------------------------------------------------
+
+    @staticmethod
+    def _iter_chunks(text: str, chunk_chars: int) -> Iterator[str]:
+        """Yield whitespace-aligned chunks. Same boundaries as the SP-era
+        wrapper, so the resulting token stream is comparable."""
+        if chunk_chars <= 0:
+            raise ValueError(f"chunk_chars must be > 0, got {chunk_chars}")
+        pos, n = 0, len(text)
+        while pos < n:
+            end = min(pos + chunk_chars, n)
+            is_eof = (end >= n)
+            window = text[pos:end]
+            if is_eof:
+                piece = window
+                pos = end
+            else:
+                cut = max(window.rfind(" "), window.rfind("\n"))
+                if cut <= 0:
+                    piece = window
+                    pos = end
+                else:
+                    piece = window[:cut]
+                    pos += cut
+            if piece:
+                yield piece
+            elif not is_eof:
+                pos += 1
+
+    def iter_encode_chunks(
+        self,
+        text: str,
+        add_bos: bool = True,
+        add_eos: bool = True,
+        chunk_chars: int = DEFAULT_CHUNK_CHARS,
+        batch_size: int | None = None,
+    ) -> Iterator[np.ndarray]:
+        """Yield per-chunk int32 arrays with BOS on first, EOS on last."""
+        if text is None:
+            raise ValueError("iter_encode_chunks() received None")
+
+        if text == "":
+            ids: List[int] = []
+            if add_bos:
+                ids.append(self.bos_id)
+            if add_eos:
+                ids.append(self.eos_id)
+            yield np.asarray(ids, dtype=np.int32)
+            return
+
+        if batch_size is None:
+            memory_budget_bytes = 32_000_000
+            chunks_per_batch = memory_budget_bytes // max(1, chunk_chars)
+            batch_size = max(1, min(16, chunks_per_batch))
+        if batch_size <= 0:
+            raise ValueError(f"batch_size must be > 0, got {batch_size}")
+        first_emitted = False
+        pending: List[str] = []
+        current: List[str] | None = None
+
+        def encode_batch(
+            chunks: List[str], is_last: bool, add_bos_here: bool,
+        ) -> Iterator[np.ndarray]:
+            encodings = self.enc.encode_ordinary_batch(
+                chunks, num_threads=_TIKTOKEN_THREADS,
+            )
+            for i, ids in enumerate(encodings):
+                if add_bos_here and i == 0:
+                    ids = [self.bos_id] + ids
+                if add_eos and is_last and i == len(encodings) - 1:
+                    ids = ids + [self.eos_id]
+                yield np.asarray(ids, dtype=np.int32)
+
+        for piece in self._iter_chunks(text, chunk_chars):
+            pending.append(piece)
+            if len(pending) < batch_size:
+                continue
+            if current is not None:
+                yield from encode_batch(current, False, add_bos and not first_emitted)
+                first_emitted = True
+            current, pending = pending, []
+
+        if current is None:
+            current = pending
+        elif pending:
+            yield from encode_batch(current, False, add_bos and not first_emitted)
+            first_emitted = True
+            current = pending
+
+        if current:
+            yield from encode_batch(current, True, add_bos and not first_emitted)
+        else:
+            # Input was all whitespace.
+            ids = []
+            if add_bos:
+                ids.append(self.bos_id)
+            if add_eos:
+                ids.append(self.eos_id)
+            yield np.asarray(ids, dtype=np.int32)
+
+    def iter_encode_file_chunks(
+        self,
+        file_path: Union[str, Path],
+        add_bos: bool = True,
+        add_eos: bool = True,
+        chunk_chars: int = DEFAULT_CHUNK_CHARS,
+    ) -> Iterator[np.ndarray]:
+        """Encode a text file incrementally without retaining the file or tokens."""
+        if chunk_chars <= 0:
+            raise ValueError(f"chunk_chars must be > 0, got {chunk_chars}")
+
+        def iter_file_text() -> Iterator[str]:
+            with open(file_path, "r", encoding="utf-8") as file:
+                buffered = ""
+                while True:
+                    block = file.read(chunk_chars)
+                    if not block:
+                        if buffered:
+                            yield buffered
+                        return
+
+                    buffered += block
+                    cut = max(buffered.rfind(" "), buffered.rfind("\n"))
+                    if cut > 0:
+                        yield buffered[:cut]
+                        buffered = buffered[cut:]
+                    elif len(buffered) >= chunk_chars * 2:
+                        yield buffered[:chunk_chars]
+                        buffered = buffered[chunk_chars:]
+
+        pieces = iter(iter_file_text())
+        try:
+            piece = next(pieces)
+        except StopIteration:
+            return
+
+        first = True
+        while True:
+            try:
+                next_piece = next(pieces)
+            except StopIteration:
+                next_piece = None
+            yield from self.iter_encode_chunks(
+                piece,
+                add_bos=add_bos and first,
+                add_eos=add_eos and next_piece is None,
+                chunk_chars=chunk_chars,
+            )
+            if next_piece is None:
+                return
+            piece = next_piece
+            first = False
+
+    def encode_large_text(
+        self,
+        text: str,
+        add_bos: bool = True,
+        add_eos: bool = False,
+        chunk_chars: int = DEFAULT_CHUNK_CHARS,
+    ) -> List[int]:
+        if text is None:
+            raise ValueError("encode_large_text() received None")
+        parts = list(self.iter_encode_chunks(
+            text, add_bos=add_bos, add_eos=add_eos, chunk_chars=chunk_chars,
+        ))
+        if not parts:
+            return []
+        return np.concatenate(parts, axis=0).astype(np.int32).tolist()
+
+    # -- batch ------------------------------------------------------------
+
+    def encode_batch(
+        self,
+        texts: Sequence[str],
+        add_bos: bool = True,
+        add_eos: bool = False,
+        skip_errors: bool = False,
+    ) -> List[List[int]]:
+        texts = list(texts)
+        if skip_errors:
+            out: List[List[int]] = []
+            for i, t in enumerate(texts):
+                try:
+                    out.append(self.encode(t, add_bos=add_bos, add_eos=add_eos))
+                except Exception as e:
+                    logger.warning(f"encode_batch: skipping item {i} ({e})")
+            return out
+
+        for t in texts:
+            if t is None:
+                raise ValueError("encode_batch() received None")
+        if not texts:
+            return []
+
+        encodings = self.enc.encode_ordinary_batch(
+            texts, num_threads=_TIKTOKEN_THREADS,
+        )
+        out = []
+        for ids in encodings:
+            if add_bos:
+                ids = [self.bos_id] + ids
+            if add_eos:
+                ids = ids + [self.eos_id]
+            out.append(ids)
+        return out
+
+    # -- decode -----------------------------------------------------------
+
+    def decode(
+        self,
+        ids: Sequence[int],
+        skip_special_tokens: bool = True,
+    ) -> str:
+        # Filter out-of-range ids first: PyTorch's ignore_index=-1 convention
+        # for masked positions means callers frequently hand us raw label
+        # tensors. tiktoken.decode() raises on any id < 0 or >= vocab_size.
+        clean: List[int] = []
+        for i in ids:
+            iv = int(i)
+            if 0 <= iv < self.vocab_size:
+                clean.append(iv)
+        if skip_special_tokens:
+            clean = [i for i in clean if i not in self._special_ids]
+        if not clean:
+            return ""
+        return self.enc.decode(clean)
+
+    def decode_batch(
+        self,
+        batch_ids: Sequence[Sequence[int]],
+        skip_special_tokens: bool = True,
+    ) -> List[str]:
+        return [self.decode(ids, skip_special_tokens=skip_special_tokens)
+                for ids in batch_ids]
+
+    # -- special-token helpers (extra, not in the SP wrapper) -------------
+
+    def encode_special_id(self, name: str) -> int:
+        """Look up the id of a named special token (e.g. '<|user_start|>')."""
+        return int(self.enc.encode_single_token(name))
+
+    # -- config round-trip ------------------------------------------------
+
+    def save_config(self, path: str) -> None:
+        Path(path).write_text(json.dumps({
+            "vocab_size": self.vocab_size,
+            "pad_id": self.pad_id,
+            "unk_id": self.unk_id,
+            "bos_id": self.bos_id,
+            "eos_id": self.eos_id,
+        }, indent=2), encoding="utf-8")
+
+    @classmethod
+    def from_config(
+        cls,
+        model_path: str,
+        config_path: Optional[str] = None,
+    ) -> "TokenizerWrapper":
+        tok = cls(model_path)
+        if config_path and Path(config_path).exists():
+            cfg = json.loads(Path(config_path).read_text(encoding="utf-8"))
+            mismatches = {
+                k: (cfg[k], getattr(tok, k))
+                for k in ("vocab_size", "pad_id", "unk_id", "bos_id", "eos_id")
+                if k in cfg and cfg[k] != getattr(tok, k)
+            }
+            if mismatches:
+                raise ValueError(f"Tokenizer/config mismatch: {mismatches}")
+        return tok

tokenizer.vocab

@@ -0,0 +1,512 @@
+<unk>	0
+<pad>	0
+<s>	0
+</s>	0
+<0x00>	0
+<0x01>	0
+<0x02>	0
+<0x03>	0
+<0x04>	0
+<0x05>	0
+<0x06>	0
+<0x07>	0
+<0x08>	0
+<0x09>	0
+<0x0A>	0
+<0x0B>	0
+<0x0C>	0
+<0x0D>	0
+<0x0E>	0
+<0x0F>	0
+<0x10>	0
+<0x11>	0
+<0x12>	0
+<0x13>	0
+<0x14>	0
+<0x15>	0
+<0x16>	0
+<0x17>	0
+<0x18>	0
+<0x19>	0
+<0x1A>	0
+<0x1B>	0
+<0x1C>	0
+<0x1D>	0
+<0x1E>	0
+<0x1F>	0
+<0x20>	0
+<0x21>	0
+<0x22>	0
+<0x23>	0
+<0x24>	0
+<0x25>	0
+<0x26>	0
+<0x27>	0
+<0x28>	0
+<0x29>	0
+<0x2A>	0
+<0x2B>	0
+<0x2C>	0
+<0x2D>	0
+<0x2E>	0
+<0x2F>	0
+<0x30>	0
+<0x31>	0
+<0x32>	0
+<0x33>	0
+<0x34>	0
+<0x35>	0
+<0x36>	0
+<0x37>	0
+<0x38>	0
+<0x39>	0
+<0x3A>	0
+<0x3B>	0
+<0x3C>	0
+<0x3D>	0
+<0x3E>	0
+<0x3F>	0
+<0x40>	0
+<0x41>	0
+<0x42>	0
+<0x43>	0
+<0x44>	0
+<0x45>	0
+<0x46>	0
+<0x47>	0
+<0x48>	0
+<0x49>	0
+<0x4A>	0
+<0x4B>	0
+<0x4C>	0
+<0x4D>	0
+<0x4E>	0
+<0x4F>	0
+<0x50>	0
+<0x51>	0
+<0x52>	0
+<0x53>	0
+<0x54>	0
+<0x55>	0
+<0x56>	0
+<0x57>	0
+<0x58>	0
+<0x59>	0
+<0x5A>	0
+<0x5B>	0
+<0x5C>	0
+<0x5D>	0
+<0x5E>	0
+<0x5F>	0
+<0x60>	0
+<0x61>	0
+<0x62>	0
+<0x63>	0
+<0x64>	0
+<0x65>	0
+<0x66>	0
+<0x67>	0
+<0x68>	0
+<0x69>	0
+<0x6A>	0
+<0x6B>	0
+<0x6C>	0
+<0x6D>	0
+<0x6E>	0
+<0x6F>	0
+<0x70>	0
+<0x71>	0
+<0x72>	0
+<0x73>	0
+<0x74>	0
+<0x75>	0
+<0x76>	0
+<0x77>	0
+<0x78>	0
+<0x79>	0
+<0x7A>	0
+<0x7B>	0
+<0x7C>	0
+<0x7D>	0
+<0x7E>	0
+<0x7F>	0
+<0x80>	0
+<0x81>	0
+<0x82>	0
+<0x83>	0
+<0x84>	0
+<0x85>	0
+<0x86>	0
+<0x87>	0
+<0x88>	0
+<0x89>	0
+<0x8A>	0
+<0x8B>	0
+<0x8C>	0
+<0x8D>	0
+<0x8E>	0
+<0x8F>	0
+<0x90>	0
+<0x91>	0
+<0x92>	0
+<0x93>	0
+<0x94>	0
+<0x95>	0
+<0x96>	0
+<0x97>	0
+<0x98>	0
+<0x99>	0
+<0x9A>	0
+<0x9B>	0
+<0x9C>	0
+<0x9D>	0
+<0x9E>	0
+<0x9F>	0
+<0xA0>	0
+<0xA1>	0
+<0xA2>	0
+<0xA3>	0
+<0xA4>	0
+<0xA5>	0
+<0xA6>	0
+<0xA7>	0
+<0xA8>	0
+<0xA9>	0
+<0xAA>	0
+<0xAB>	0
+<0xAC>	0
+<0xAD>	0
+<0xAE>	0
+<0xAF>	0
+<0xB0>	0
+<0xB1>	0
+<0xB2>	0
+<0xB3>	0
+<0xB4>	0
+<0xB5>	0
+<0xB6>	0
+<0xB7>	0
+<0xB8>	0
+<0xB9>	0
+<0xBA>	0
+<0xBB>	0
+<0xBC>	0
+<0xBD>	0
+<0xBE>	0
+<0xBF>	0
+<0xC0>	0
+<0xC1>	0
+<0xC2>	0
+<0xC3>	0
+<0xC4>	0
+<0xC5>	0
+<0xC6>	0
+<0xC7>	0
+<0xC8>	0
+<0xC9>	0
+<0xCA>	0
+<0xCB>	0
+<0xCC>	0
+<0xCD>	0
+<0xCE>	0
+<0xCF>	0
+<0xD0>	0
+<0xD1>	0
+<0xD2>	0
+<0xD3>	0
+<0xD4>	0
+<0xD5>	0
+<0xD6>	0
+<0xD7>	0
+<0xD8>	0
+<0xD9>	0
+<0xDA>	0
+<0xDB>	0
+<0xDC>	0
+<0xDD>	0
+<0xDE>	0
+<0xDF>	0
+<0xE0>	0
+<0xE1>	0
+<0xE2>	0
+<0xE3>	0
+<0xE4>	0
+<0xE5>	0
+<0xE6>	0
+<0xE7>	0
+<0xE8>	0
+<0xE9>	0
+<0xEA>	0
+<0xEB>	0
+<0xEC>	0
+<0xED>	0
+<0xEE>	0
+<0xEF>	0
+<0xF0>	0
+<0xF1>	0
+<0xF2>	0
+<0xF3>	0
+<0xF4>	0
+<0xF5>	0
+<0xF6>	0
+<0xF7>	0
+<0xF8>	0
+<0xF9>	0
+<0xFA>	0
+<0xFB>	0
+<0xFC>	0
+<0xFD>	0
+<0xFE>	0
+<0xFF>	0
+he	-0
+▁t	-1
+an	-2
+▁s	-3
+in	-4
+▁w	-5
+ed	-6
+▁the	-7
+al	-8
+on	-9
+▁c	-10
+er	-11
+▁f	-12
+and	-13
+at	-14
+▁b	-15
+is	-16
+▁a	-17
+re	-18
+it	-19
+▁and	-20
+ar	-21
+▁o	-22
+▁h	-23
+es	-24
+en	-25
+▁l	-26
+nd	-27
+▁r	-28
+▁m	-29
+▁T	-30
+▁p	-31
+▁of	-32
+▁is	-33
+ing	-34
+▁in	-35
+▁I	-36
+as	-37
+ou	-38
+or	-39
+▁g	-40
+am	-41
+▁The	-42
+ic	-43
+▁d	-44
+el	-45
+her	-46
+et	-47
+▁Ind	-48
+▁n	-49
+▁Indi	-50
+ig	-51
+ol	-52
+il	-53
+ot	-54
+▁to	-55
+all	-56
+ith	-57
+ver	-58
+▁India	-59
+ay	-60
+ow	-61
+ac	-62
+om	-63
+▁st	-64
+▁with	-65
+ht	-66
+▁A	-67
+▁cl	-68
+▁on	-69
+ad	-70
+ul	-71
+▁M	-72
+▁R	-73
+ang	-74
+ion	-75
+▁S	-76
+est	-77
+ight	-78
+ft	-79
+ain	-80
+▁br	-81
+▁th	-82
+▁his	-83
+ly	-84
+▁G	-85
+ent	-86
+▁fl	-87
+▁fr	-88
+▁gre	-89
+ut	-90
+we	-91
+▁C	-92
+▁D	-93
+▁e	-94
+res	-95
+ation	-96
+im	-97
+le	-98
+un	-99
+▁sp	-100
+weet	-101
+▁wat	-102
+▁sweet	-103
+av	-104
+ir	-105
+pp	-106
+▁B	-107
+▁H	-108
+ful	-109
+old	-110
+ong	-111
+▁Ch	-112
+▁by	-113
+▁co	-114
+ational	-115
+ab	-116
+ag	-117
+ak	-118
+ur	-119
+arm	-120
+ill	-121
+ite	-122
+its	-123
+one	-124
+ous	-125
+▁an	-126
+▁tr	-127
+▁wh	-128
+alled	-129
+▁green	-130
+▁called	-131
+▁national	-132
+ch	-133
+iv	-134
+lu	-135
+▁N	-136
+▁k	-137
+ers	-138
+ick	-139
+rou	-140
+▁for	-141
+▁tre	-142
+▁was	-143
+▁warm	-144
+ok	-145
+th	-146
+to	-147
+us	-148
+▁W	-149
+▁y	-150
+ell	-151
+hed	-152
+ter	-153
+▁gr	-154
+▁ri	-155
+▁sh	-156
+▁sm	-157
+▁so	-158
+pped	-159
+▁fam	-160
+▁from	-161
+▁bright	-162
+ds	-163
+ld	-164
+tu	-165
+um	-166
+ace	-167
+ade	-168
+lue	-169
+oon	-170
+ore	-171
+und	-172
+▁al	-173
+▁be	-174
+▁pe	-175
+▁ro	-176
+ands	-177
+ated	-178
+very	-179
+▁its	-180
+▁pot	-181
+▁whe	-182
+other	-183
+▁blue	-184
+▁water	-185
+▁famous	-186
+ew	-187
+op	-188
+os	-189
+ov	-190
+▁P	-191
+▁v	-192
+ack	-193
+anc	-194
+any	-195
+▁	-196
+e	-197
+a	-198
+t	-199
+n	-200
+i	-201
+o	-202
+r	-203
+s	-204
+h	-205
+l	-206
+d	-207
+c	-208
+g	-209
+f	-210
+u	-211
+w	-212
+m	-213
+y	-214
+p	-215
+b	-216
+.	-217
+,	-218
+k	-219
+v	-220
+T	-221
+I	-222
+A	-223
+M	-224
+R	-225
+S	-226
+G	-227
+C	-228
+D	-229
+j	-230
+B	-231
+H	-232
+N	-233
+W	-234
+P	-235
+E	-236
+K	-237
+!	-238
+J	-239
+O	-240
+2	-241
+:	-242
+F	-243
+L	-244
+q	-245
+x	-246
+z	-247
+5	-248
+8	-249
+'	-250
+3	-251

tokenizer/tokenizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ca760f09eb2dd9e980654f61e75867c04f1a9afd9245a031a0fa50025b2bc88
+size 147083

tokenizer/tokenizer_config.json

@@ -0,0 +1,7 @@
+{
+  "vocab_size": 12288,
+  "bos_id": 12287,
+  "eos_id": 12287,
+  "pad_id": 12287,
+  "unk_id": 12287
+}

tokenizer_1.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:547868b9fdc9e916c5530467a6bc1353a08a7e1934b0671873b0d64caa9f5bfd
+size 447723

tokenizer_1.py

@@ -0,0 +1,432 @@
+"""
+tokenizer.py — RustBPETokenizer adapter.
+
+Drop-in replacement for the previous SentencePiece-based TokenizerWrapper.
+Exposes the SAME public API so train.py, _tok_worker.py, and finetune.py
+work unchanged. The only pipeline change required is that the on-disk
+tokenizer artifact is now a directory containing `tokenizer.pkl` (a
+pickled tiktoken.Encoding), rather than a single `tokenizer.model` file.
+
+Set TOKENIZER_MODEL_PATH in train.py to the directory (or to the pickle
+file directly — both are accepted).
+
+API compatibility
+-----------------
+Public attributes:
+    model_path      str   absolute path to the .pkl (hashed into fingerprint)
+    vocab_size      int   total vocab size, specials included
+    bos_id          int   id of <|bos|>
+    eos_id          int   same as bos_id (see Notes)
+    pad_id          int   same as bos_id (see Notes)
+    unk_id          int   same as bos_id (see Notes)
+
+Public methods:
+    encode(text, add_bos=True, add_eos=False)                -> List[int]
+    encode_large_text(text, add_bos, add_eos, chunk_chars)   -> List[int]
+    iter_encode_chunks(text, add_bos, add_eos, chunk_chars)  -> Iterator[np.ndarray]
+    encode_batch(texts, add_bos, add_eos, skip_errors)       -> List[List[int]]
+    decode(ids, skip_special_tokens=True)                    -> str
+    decode_batch(batch_ids, skip_special_tokens=True)        -> List[str]
+    save_config(path)
+    from_config(model_path, config_path=None)                -> TokenizerWrapper
+
+Notes
+-----
+pad_id / eos_id / unk_id all map to bos_id because:
+  - tiktoken is byte-level, so <unk> is never emitted
+    - rustbpe's SPECIAL_TOKENS list has no dedicated <|eos>; this adapter
+        treats each input to encode() as one document. For iter_encode_chunks(),
+        one BOS is placed at the start of the file and one EOS-equivalent BOS is
+        placed at the end, matching the flat SP-era stream contract rather than
+        nanochat's per-document stream.
+  - finetune.py needs SOME valid id for padding; <|bos|> is the standard
+    choice and is filtered by decode(skip_special_tokens=True).
+
+encode() uses tiktoken's encode_ordinary() fast path, which does NOT
+interpret "<|user_start|>" etc. as special tokens. SFT rendering must
+call encode_special_id() for control tokens and encode_ordinary() only
+for content. This is the same contract as nanochat's trainer.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import pickle
+from pathlib import Path
+from typing import Iterator, List, Optional, Sequence, Union
+
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_CHUNK_CHARS = 500_000
+_TIKTOKEN_THREADS = int(os.environ.get(
+    "TIKTOKEN_NUM_THREADS",
+    str(max(1, min(8, os.cpu_count() or 1))),
+))
+
+
+# ---------------------------------------------------------------------------
+# Helper: locate the pickle given a directory or a file path
+# ---------------------------------------------------------------------------
+
+def _resolve_pickle_path(model_path: Union[str, Path]) -> Path:
+    p = Path(model_path)
+    if p.is_dir():
+        candidate = p / "tokenizer.pkl"
+        if not candidate.exists():
+            raise FileNotFoundError(
+                f"{p} is a directory but contains no tokenizer.pkl "
+                f"(expected {candidate})"
+            )
+        return candidate
+    if not p.exists():
+        raise FileNotFoundError(str(p))
+    return p
+
+
+# ---------------------------------------------------------------------------
+# TokenizerWrapper
+# ---------------------------------------------------------------------------
+
+class TokenizerWrapper:
+    """Adapter exposing a pickled tiktoken.Encoding behind the SP-era API."""
+
+    def __init__(self, model_path: Union[str, Path]):
+        pickle_path = _resolve_pickle_path(model_path)
+        # Note: pickled tiktoken.Encoding. Only load pickles you created.
+        try:
+            with open(pickle_path, "rb") as f:
+                self.enc = pickle.load(f)
+        except Exception as exc:
+            raise ValueError(
+                f"Failed to unpickle {pickle_path}: {exc!r}. "
+                "This adapter expects a pickled tiktoken.Encoding "
+                "produced by RustBPETokenizer.save(). If you have an old "
+                "SentencePiece .model file, train a new tokenizer with rustbpe."
+            ) from exc
+
+        # Validate it looks like a tiktoken Encoding before trusting it.
+        for attr in ("n_vocab", "encode_ordinary", "encode_ordinary_batch",
+                     "decode", "encode_single_token", "special_tokens_set"):
+            if not hasattr(self.enc, attr):
+                raise TypeError(
+                    f"{pickle_path}: loaded object is not a tiktoken.Encoding "
+                    f"(missing attribute {attr!r}). Got {type(self.enc).__name__}."
+                )
+
+        self.model_path = str(pickle_path)
+        self.vocab_size = int(self.enc.n_vocab)
+
+        # BOS is required. If the vocab lacks it, that's a hard error.
+        self.bos_id = int(self.enc.encode_single_token("<|bos|>"))
+        # EOS/PAD/UNK reuse BOS — see module docstring.
+        self.eos_id = self.bos_id
+        self.pad_id = self.bos_id
+        self.unk_id = self.bos_id
+
+        # Everything tiktoken labels as special — used by decode() to drop
+        # control tokens when skip_special_tokens=True.
+        self._special_ids = set()
+        for name in self.enc.special_tokens_set:
+            try:
+                self._special_ids.add(int(self.enc.encode_single_token(name)))
+            except Exception:
+                # If a special name isn't encodable (shouldn't happen for a
+                # well-formed Encoding), skip it rather than crash.
+                pass
+        # Always include BOS even if special_tokens_set was empty for some
+        # reason.
+        self._special_ids.add(self.bos_id)
+
+    # -- single-sequence encode ---------------------------------------------
+
+    def encode(
+        self,
+        text: str,
+        add_bos: bool = True,
+        add_eos: bool = False,
+    ) -> List[int]:
+        if text is None:
+            raise ValueError("encode() received None")
+        if text == "":
+            ids: List[int] = []
+        else:
+            ids = self.enc.encode_ordinary(text)
+        if add_bos:
+            ids = [self.bos_id] + ids
+        if add_eos:
+            ids = ids + [self.eos_id]
+        return ids
+
+    # -- chunked streaming --------------------------------------------------
+
+    @staticmethod
+    def _iter_chunks(text: str, chunk_chars: int) -> Iterator[str]:
+        """Yield whitespace-aligned chunks. Same boundaries as the SP-era
+        wrapper, so the resulting token stream is comparable."""
+        if chunk_chars <= 0:
+            raise ValueError(f"chunk_chars must be > 0, got {chunk_chars}")
+        pos, n = 0, len(text)
+        while pos < n:
+            end = min(pos + chunk_chars, n)
+            is_eof = (end >= n)
+            window = text[pos:end]
+            if is_eof:
+                piece = window
+                pos = end
+            else:
+                cut = max(window.rfind(" "), window.rfind("\n"))
+                if cut <= 0:
+                    piece = window
+                    pos = end
+                else:
+                    piece = window[:cut]
+                    pos += cut
+            if piece:
+                yield piece
+            elif not is_eof:
+                pos += 1
+
+    def iter_encode_chunks(
+        self,
+        text: str,
+        add_bos: bool = True,
+        add_eos: bool = True,
+        chunk_chars: int = DEFAULT_CHUNK_CHARS,
+        batch_size: int | None = None,
+    ) -> Iterator[np.ndarray]:
+        """Yield per-chunk int32 arrays with BOS on first, EOS on last."""
+        if text is None:
+            raise ValueError("iter_encode_chunks() received None")
+
+        if text == "":
+            ids: List[int] = []
+            if add_bos:
+                ids.append(self.bos_id)
+            if add_eos:
+                ids.append(self.eos_id)
+            yield np.asarray(ids, dtype=np.int32)
+            return
+
+        if batch_size is None:
+            memory_budget_bytes = 32_000_000
+            chunks_per_batch = memory_budget_bytes // max(1, chunk_chars)
+            batch_size = max(1, min(16, chunks_per_batch))
+        if batch_size <= 0:
+            raise ValueError(f"batch_size must be > 0, got {batch_size}")
+        first_emitted = False
+        pending: List[str] = []
+        current: List[str] | None = None
+
+        def encode_batch(
+            chunks: List[str], is_last: bool, add_bos_here: bool,
+        ) -> Iterator[np.ndarray]:
+            encodings = self.enc.encode_ordinary_batch(
+                chunks, num_threads=_TIKTOKEN_THREADS,
+            )
+            for i, ids in enumerate(encodings):
+                if add_bos_here and i == 0:
+                    ids = [self.bos_id] + ids
+                if add_eos and is_last and i == len(encodings) - 1:
+                    ids = ids + [self.eos_id]
+                yield np.asarray(ids, dtype=np.int32)
+
+        for piece in self._iter_chunks(text, chunk_chars):
+            pending.append(piece)
+            if len(pending) < batch_size:
+                continue
+            if current is not None:
+                yield from encode_batch(current, False, add_bos and not first_emitted)
+                first_emitted = True
+            current, pending = pending, []
+
+        if current is None:
+            current = pending
+        elif pending:
+            yield from encode_batch(current, False, add_bos and not first_emitted)
+            first_emitted = True
+            current = pending
+
+        if current:
+            yield from encode_batch(current, True, add_bos and not first_emitted)
+        else:
+            # Input was all whitespace.
+            ids = []
+            if add_bos:
+                ids.append(self.bos_id)
+            if add_eos:
+                ids.append(self.eos_id)
+            yield np.asarray(ids, dtype=np.int32)
+
+    def iter_encode_file_chunks(
+        self,
+        file_path: Union[str, Path],
+        add_bos: bool = True,
+        add_eos: bool = True,
+        chunk_chars: int = DEFAULT_CHUNK_CHARS,
+    ) -> Iterator[np.ndarray]:
+        """Encode a text file incrementally without retaining the file or tokens."""
+        if chunk_chars <= 0:
+            raise ValueError(f"chunk_chars must be > 0, got {chunk_chars}")
+
+        def iter_file_text() -> Iterator[str]:
+            with open(file_path, "r", encoding="utf-8") as file:
+                buffered = ""
+                while True:
+                    block = file.read(chunk_chars)
+                    if not block:
+                        if buffered:
+                            yield buffered
+                        return
+
+                    buffered += block
+                    cut = max(buffered.rfind(" "), buffered.rfind("\n"))
+                    if cut > 0:
+                        yield buffered[:cut]
+                        buffered = buffered[cut:]
+                    elif len(buffered) >= chunk_chars * 2:
+                        yield buffered[:chunk_chars]
+                        buffered = buffered[chunk_chars:]
+
+        pieces = iter(iter_file_text())
+        try:
+            piece = next(pieces)
+        except StopIteration:
+            return
+
+        first = True
+        while True:
+            try:
+                next_piece = next(pieces)
+            except StopIteration:
+                next_piece = None
+            yield from self.iter_encode_chunks(
+                piece,
+                add_bos=add_bos and first,
+                add_eos=add_eos and next_piece is None,
+                chunk_chars=chunk_chars,
+            )
+            if next_piece is None:
+                return
+            piece = next_piece
+            first = False
+
+    def encode_large_text(
+        self,
+        text: str,
+        add_bos: bool = True,
+        add_eos: bool = False,
+        chunk_chars: int = DEFAULT_CHUNK_CHARS,
+    ) -> List[int]:
+        if text is None:
+            raise ValueError("encode_large_text() received None")
+        parts = list(self.iter_encode_chunks(
+            text, add_bos=add_bos, add_eos=add_eos, chunk_chars=chunk_chars,
+        ))
+        if not parts:
+            return []
+        return np.concatenate(parts, axis=0).astype(np.int32).tolist()
+
+    # -- batch ------------------------------------------------------------
+
+    def encode_batch(
+        self,
+        texts: Sequence[str],
+        add_bos: bool = True,
+        add_eos: bool = False,
+        skip_errors: bool = False,
+    ) -> List[List[int]]:
+        texts = list(texts)
+        if skip_errors:
+            out: List[List[int]] = []
+            for i, t in enumerate(texts):
+                try:
+                    out.append(self.encode(t, add_bos=add_bos, add_eos=add_eos))
+                except Exception as e:
+                    logger.warning(f"encode_batch: skipping item {i} ({e})")
+            return out
+
+        for t in texts:
+            if t is None:
+                raise ValueError("encode_batch() received None")
+        if not texts:
+            return []
+
+        encodings = self.enc.encode_ordinary_batch(
+            texts, num_threads=_TIKTOKEN_THREADS,
+        )
+        out = []
+        for ids in encodings:
+            if add_bos:
+                ids = [self.bos_id] + ids
+            if add_eos:
+                ids = ids + [self.eos_id]
+            out.append(ids)
+        return out
+
+    # -- decode -----------------------------------------------------------
+
+    def decode(
+        self,
+        ids: Sequence[int],
+        skip_special_tokens: bool = True,
+    ) -> str:
+        # Filter out-of-range ids first: PyTorch's ignore_index=-1 convention
+        # for masked positions means callers frequently hand us raw label
+        # tensors. tiktoken.decode() raises on any id < 0 or >= vocab_size.
+        clean: List[int] = []
+        for i in ids:
+            iv = int(i)
+            if 0 <= iv < self.vocab_size:
+                clean.append(iv)
+        if skip_special_tokens:
+            clean = [i for i in clean if i not in self._special_ids]
+        if not clean:
+            return ""
+        return self.enc.decode(clean)
+
+    def decode_batch(
+        self,
+        batch_ids: Sequence[Sequence[int]],
+        skip_special_tokens: bool = True,
+    ) -> List[str]:
+        return [self.decode(ids, skip_special_tokens=skip_special_tokens)
+                for ids in batch_ids]
+
+    # -- special-token helpers (extra, not in the SP wrapper) -------------
+
+    def encode_special_id(self, name: str) -> int:
+        """Look up the id of a named special token (e.g. '<|user_start|>')."""
+        return int(self.enc.encode_single_token(name))
+
+    # -- config round-trip ------------------------------------------------
+
+    def save_config(self, path: str) -> None:
+        Path(path).write_text(json.dumps({
+            "vocab_size": self.vocab_size,
+            "pad_id": self.pad_id,
+            "unk_id": self.unk_id,
+            "bos_id": self.bos_id,
+            "eos_id": self.eos_id,
+        }, indent=2), encoding="utf-8")
+
+    @classmethod
+    def from_config(
+        cls,
+        model_path: str,
+        config_path: Optional[str] = None,
+    ) -> "TokenizerWrapper":
+        tok = cls(model_path)
+        if config_path and Path(config_path).exists():
+            cfg = json.loads(Path(config_path).read_text(encoding="utf-8"))
+            mismatches = {
+                k: (cfg[k], getattr(tok, k))
+                for k in ("vocab_size", "pad_id", "unk_id", "bos_id", "eos_id")
+                if k in cfg and cfg[k] != getattr(tok, k)
+            }
+            if mismatches:
+                raise ValueError(f"Tokenizer/config mismatch: {mismatches}")
+        return tok

tokenizer_1.vocab

@@ -0,0 +1,512 @@
+<unk>	0
+<pad>	0
+<s>	0
+</s>	0
+<0x00>	0
+<0x01>	0
+<0x02>	0
+<0x03>	0
+<0x04>	0
+<0x05>	0
+<0x06>	0
+<0x07>	0
+<0x08>	0
+<0x09>	0
+<0x0A>	0
+<0x0B>	0
+<0x0C>	0
+<0x0D>	0
+<0x0E>	0
+<0x0F>	0
+<0x10>	0
+<0x11>	0
+<0x12>	0
+<0x13>	0
+<0x14>	0
+<0x15>	0
+<0x16>	0
+<0x17>	0
+<0x18>	0
+<0x19>	0
+<0x1A>	0
+<0x1B>	0
+<0x1C>	0
+<0x1D>	0
+<0x1E>	0
+<0x1F>	0
+<0x20>	0
+<0x21>	0
+<0x22>	0
+<0x23>	0
+<0x24>	0
+<0x25>	0
+<0x26>	0
+<0x27>	0
+<0x28>	0
+<0x29>	0
+<0x2A>	0
+<0x2B>	0
+<0x2C>	0
+<0x2D>	0
+<0x2E>	0
+<0x2F>	0
+<0x30>	0
+<0x31>	0
+<0x32>	0
+<0x33>	0
+<0x34>	0
+<0x35>	0
+<0x36>	0
+<0x37>	0
+<0x38>	0
+<0x39>	0
+<0x3A>	0
+<0x3B>	0
+<0x3C>	0
+<0x3D>	0
+<0x3E>	0
+<0x3F>	0
+<0x40>	0
+<0x41>	0
+<0x42>	0
+<0x43>	0
+<0x44>	0
+<0x45>	0
+<0x46>	0
+<0x47>	0
+<0x48>	0
+<0x49>	0
+<0x4A>	0
+<0x4B>	0
+<0x4C>	0
+<0x4D>	0
+<0x4E>	0
+<0x4F>	0
+<0x50>	0
+<0x51>	0
+<0x52>	0
+<0x53>	0
+<0x54>	0
+<0x55>	0
+<0x56>	0
+<0x57>	0
+<0x58>	0
+<0x59>	0
+<0x5A>	0
+<0x5B>	0
+<0x5C>	0
+<0x5D>	0
+<0x5E>	0
+<0x5F>	0
+<0x60>	0
+<0x61>	0
+<0x62>	0
+<0x63>	0
+<0x64>	0
+<0x65>	0
+<0x66>	0
+<0x67>	0
+<0x68>	0
+<0x69>	0
+<0x6A>	0
+<0x6B>	0
+<0x6C>	0
+<0x6D>	0
+<0x6E>	0
+<0x6F>	0
+<0x70>	0
+<0x71>	0
+<0x72>	0
+<0x73>	0
+<0x74>	0
+<0x75>	0
+<0x76>	0
+<0x77>	0
+<0x78>	0
+<0x79>	0
+<0x7A>	0
+<0x7B>	0
+<0x7C>	0
+<0x7D>	0
+<0x7E>	0
+<0x7F>	0
+<0x80>	0
+<0x81>	0
+<0x82>	0
+<0x83>	0
+<0x84>	0
+<0x85>	0
+<0x86>	0
+<0x87>	0
+<0x88>	0
+<0x89>	0
+<0x8A>	0
+<0x8B>	0
+<0x8C>	0
+<0x8D>	0
+<0x8E>	0
+<0x8F>	0
+<0x90>	0
+<0x91>	0
+<0x92>	0
+<0x93>	0
+<0x94>	0
+<0x95>	0
+<0x96>	0
+<0x97>	0
+<0x98>	0
+<0x99>	0
+<0x9A>	0
+<0x9B>	0
+<0x9C>	0
+<0x9D>	0
+<0x9E>	0
+<0x9F>	0
+<0xA0>	0
+<0xA1>	0
+<0xA2>	0
+<0xA3>	0
+<0xA4>	0
+<0xA5>	0
+<0xA6>	0
+<0xA7>	0
+<0xA8>	0
+<0xA9>	0
+<0xAA>	0
+<0xAB>	0
+<0xAC>	0
+<0xAD>	0
+<0xAE>	0
+<0xAF>	0
+<0xB0>	0
+<0xB1>	0
+<0xB2>	0
+<0xB3>	0
+<0xB4>	0
+<0xB5>	0
+<0xB6>	0
+<0xB7>	0
+<0xB8>	0
+<0xB9>	0
+<0xBA>	0
+<0xBB>	0
+<0xBC>	0
+<0xBD>	0
+<0xBE>	0
+<0xBF>	0
+<0xC0>	0
+<0xC1>	0
+<0xC2>	0
+<0xC3>	0
+<0xC4>	0
+<0xC5>	0
+<0xC6>	0
+<0xC7>	0
+<0xC8>	0
+<0xC9>	0
+<0xCA>	0
+<0xCB>	0
+<0xCC>	0
+<0xCD>	0
+<0xCE>	0
+<0xCF>	0
+<0xD0>	0
+<0xD1>	0
+<0xD2>	0
+<0xD3>	0
+<0xD4>	0
+<0xD5>	0
+<0xD6>	0
+<0xD7>	0
+<0xD8>	0
+<0xD9>	0
+<0xDA>	0
+<0xDB>	0
+<0xDC>	0
+<0xDD>	0
+<0xDE>	0
+<0xDF>	0
+<0xE0>	0
+<0xE1>	0
+<0xE2>	0
+<0xE3>	0
+<0xE4>	0
+<0xE5>	0
+<0xE6>	0
+<0xE7>	0
+<0xE8>	0
+<0xE9>	0
+<0xEA>	0
+<0xEB>	0
+<0xEC>	0
+<0xED>	0
+<0xEE>	0
+<0xEF>	0
+<0xF0>	0
+<0xF1>	0
+<0xF2>	0
+<0xF3>	0
+<0xF4>	0
+<0xF5>	0
+<0xF6>	0
+<0xF7>	0
+<0xF8>	0
+<0xF9>	0
+<0xFA>	0
+<0xFB>	0
+<0xFC>	0
+<0xFD>	0
+<0xFE>	0
+<0xFF>	0
+he	-0
+▁t	-1
+an	-2
+▁s	-3
+in	-4
+▁w	-5
+ed	-6
+▁the	-7
+al	-8
+on	-9
+▁c	-10
+er	-11
+▁f	-12
+and	-13
+at	-14
+▁b	-15
+is	-16
+▁a	-17
+re	-18
+it	-19
+▁and	-20
+ar	-21
+▁o	-22
+▁h	-23
+es	-24
+en	-25
+▁l	-26
+nd	-27
+▁r	-28
+▁m	-29
+▁T	-30
+▁p	-31
+▁of	-32
+▁is	-33
+ing	-34
+▁in	-35
+▁I	-36
+as	-37
+ou	-38
+or	-39
+▁g	-40
+am	-41
+▁The	-42
+ic	-43
+▁d	-44
+el	-45
+her	-46
+et	-47
+▁Ind	-48
+▁n	-49
+▁Indi	-50
+ig	-51
+ol	-52
+il	-53
+ot	-54
+▁to	-55
+all	-56
+ith	-57
+ver	-58
+▁India	-59
+ay	-60
+ow	-61
+ac	-62
+om	-63
+▁st	-64
+▁with	-65
+ht	-66
+▁A	-67
+▁cl	-68
+▁on	-69
+ad	-70
+ul	-71
+▁M	-72
+▁R	-73
+ang	-74
+ion	-75
+▁S	-76
+est	-77
+ight	-78
+ft	-79
+ain	-80
+▁br	-81
+▁th	-82
+▁his	-83
+ly	-84
+▁G	-85
+ent	-86
+▁fl	-87
+▁fr	-88
+▁gre	-89
+ut	-90
+we	-91
+▁C	-92
+▁D	-93
+▁e	-94
+res	-95
+ation	-96
+im	-97
+le	-98
+un	-99
+▁sp	-100
+weet	-101
+▁wat	-102
+▁sweet	-103
+av	-104
+ir	-105
+pp	-106
+▁B	-107
+▁H	-108
+ful	-109
+old	-110
+ong	-111
+▁Ch	-112
+▁by	-113
+▁co	-114
+ational	-115
+ab	-116
+ag	-117
+ak	-118
+ur	-119
+arm	-120
+ill	-121
+ite	-122
+its	-123
+one	-124
+ous	-125
+▁an	-126
+▁tr	-127
+▁wh	-128
+alled	-129
+▁green	-130
+▁called	-131
+▁national	-132
+ch	-133
+iv	-134
+lu	-135
+▁N	-136
+▁k	-137
+ers	-138
+ick	-139
+rou	-140
+▁for	-141
+▁tre	-142
+▁was	-143
+▁warm	-144
+ok	-145
+th	-146
+to	-147
+us	-148
+▁W	-149
+▁y	-150
+ell	-151
+hed	-152
+ter	-153
+▁gr	-154
+▁ri	-155
+▁sh	-156
+▁sm	-157
+▁so	-158
+pped	-159
+▁fam	-160
+▁from	-161
+▁bright	-162
+ds	-163
+ld	-164
+tu	-165
+um	-166
+ace	-167
+ade	-168
+lue	-169
+oon	-170
+ore	-171
+und	-172
+▁al	-173
+▁be	-174
+▁pe	-175
+▁ro	-176
+ands	-177
+ated	-178
+very	-179
+▁its	-180
+▁pot	-181
+▁whe	-182
+other	-183
+▁blue	-184
+▁water	-185
+▁famous	-186
+ew	-187
+op	-188
+os	-189
+ov	-190
+▁P	-191
+▁v	-192
+ack	-193
+anc	-194
+any	-195
+▁	-196
+e	-197
+a	-198
+t	-199
+n	-200
+i	-201
+o	-202
+r	-203
+s	-204
+h	-205
+l	-206
+d	-207
+c	-208
+g	-209
+f	-210
+u	-211
+w	-212
+m	-213
+y	-214
+p	-215
+b	-216
+.	-217
+,	-218
+k	-219
+v	-220
+T	-221
+I	-222
+A	-223
+M	-224
+R	-225
+S	-226
+G	-227
+C	-228
+D	-229
+j	-230
+B	-231
+H	-232
+N	-233
+W	-234
+P	-235
+E	-236
+K	-237
+!	-238
+J	-239
+O	-240
+2	-241
+:	-242
+F	-243
+L	-244
+q	-245
+x	-246
+z	-247
+5	-248
+8	-249
+'	-250
+3	-251