EPFL ALiBi and training provenance 251-300 of 1184

.gitattributes

@@ -223,3 +223,7 @@ epfl_final/wandb/run-20260513_052440-i0f0xwcq/run-i0f0xwcq.wandb filter=lfs diff
 epfl_final/wandb/run-20260513_151725-qudv4nmq/run-qudv4nmq.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/wandb/run-20260513_164613-40cb81kv/run-40cb81kv.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/wandb/run-20260517_023932-hi2z3jh6/run-hi2z3jh6.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/wandb/run-20260517_042355-hi2z3jh6/run-hi2z3jh6.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/wandb/run-20260517_162018-hi2z3jh6/run-hi2z3jh6.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/wandb/run-20260518_025234-hi2z3jh6/run-hi2z3jh6.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/wandb/run-20260519_212833-oeak7lgl/run-oeak7lgl.wandb filter=lfs diff=lfs merge=lfs -text

epfl_final/wandb/run-20260517_041523-2uawripc/files/output.log

@@ -0,0 +1,110 @@
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1024
+  num_hidden_layers: 28
+  num_attention_heads: 16
+  num_key_value_heads: 8
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 18
+  sparse_mask_pool_size: 2000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 596,049,920
+  Trainable parameters: 596,049,920
+  Non-trainable parameters: 0
+  Model size (BF16): 1.19 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:2055: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [3, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [3, 8192, 1024]           --                        True
+│    └─Embedding: 2-1                                        [3, 8192]                 [3, 8192, 1024]           155,582,464               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    └─Qwen3RMSNorm: 2-3                                     [3, 8192, 1024]           [3, 8192, 1024]           1,024                     True
+├─Linear: 1-2                                                [3, 8192, 1024]           [3, 8192, 151936]         155,582,464               True
+================================================================================================================================================================
+Total params: 751,632,384
+Trainable params: 751,632,384
+Non-trainable params: 0
+Total mult-adds (G): 2.25
+================================================================================================================================================================
+Input size (MB): 0.20
+Forward/backward pass size (MB): 63052.97
+Params size (MB): 1503.26
+Estimated Total Size (MB): 64556.43
+================================================================================================================================================================
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003)
+Dataset split: 1017602 train, 254401 validation
+Training examples (packed, train split):  1017602
+Validation examples (packed, val split):  254401
+Per-device batch size:                    3
+World size (GPUs):                        1
+Gradient accumulation steps:              8
+Effective batch size (seqs/step):         24
+Sequence length:                          8192
+Tokens per optimizer step:                196,608
+Micro-batches per epoch (per rank):       339200
+Optimizer steps per epoch:                42400
+Total optimizer steps:                    20005  (capped by max_steps; 0.47 epochs)
+Loading checkpoint from checkpoints/run_20260517_023855/checkpoint-1000...
+Resumed from checkpoint at step 999 (epoch 0, micro_step 8000)
+Resuming training from step 999
+Starting training...
+Step 1000/20005 | Loss: 0.3998 | LR: 4.98e-04 | 82882.0ms/step | 0.3 samples/s

epfl_final/wandb/run-20260517_041523-2uawripc/files/requirements.txt

@@ -0,0 +1,97 @@
+pytz==2026.2
+nvidia-cusparselt-cu13==0.8.1
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.7.0
+tzdata==2026.2
+typing_extensions==4.15.0
+triton==3.7.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+safetensors==0.7.0
+regex==2026.5.9
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.5.2
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.29.7
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+numpy==2.2.6
+networkx==3.4.2
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.15
+hf-xet==1.5.0
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.4.0
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+async-timeout==5.0.1
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+typing-inspection==0.4.2
+sentry-sdk==2.60.0
+requests==2.34.2
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.4
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cublas==13.1.1.3
+multiprocess==0.70.19
+multidict==6.7.1
+markdown-it-py==4.2.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+exceptiongroup==1.3.1
+cuda-bindings==13.2.0
+aiosignal==1.4.0
+yarl==1.23.0
+rich==15.0.0
+pydantic==2.13.4
+pandas==2.3.3
+nvidia-cusolver==12.0.4.66
+nvidia-cudnn-cu13==9.20.0.48
+GitPython==3.1.50
+anyio==4.13.0
+wandb==0.27.0
+typer==0.25.1
+httpx==0.28.1
+aiohttp==3.13.5
+torch==2.12.0
+huggingface_hub==1.15.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.8.1
+PyGObject==3.42.1
+dbus-python==1.2.18
+distro==1.7.0
+pip==22.0.2
+setuptools==59.6.0
+ssh-import-id==5.11
+wheel==0.37.1

epfl_final/wandb/run-20260517_041523-2uawripc/files/wandb-metadata.json

@@ -0,0 +1,36 @@
+{
+  "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.12",
+  "startedAt": "2026-05-17T04:15:23.993873Z",
+  "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath": "SparseAtt/train.py",
+  "codePathLocal": "SparseAtt/train.py",
+  "email": "ruben.navasardyan@mail.utoronto.ca",
+  "root": "/mloscratch/homes/navasard/sparse",
+  "host": "h100-train-14-0-4",
+  "executable": "/usr/bin/python3",
+  "cpu_count": 96,
+  "cpu_count_logical": 192,
+  "gpu": "NVIDIA H100 80GB HBM3",
+  "gpu_count": 1,
+  "disk": {
+    "/": {
+      "total": "9374931091456",
+      "used": "6127603781632"
+    }
+  },
+  "memory": {
+    "total": "1623005847552"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-c65885f7-52dc-6f23-a3dd-6addf8b518a2"
+    }
+  ],
+  "cudaVersion": "13.0",
+  "writerId": "8w4wjnqrb1k0w9tyzw6wur36lmxhqkaj"
+}

epfl_final/wandb/run-20260517_041523-2uawripc/logs/debug-internal.log

@@ -0,0 +1,41 @@
+{"time":"2026-05-17T04:15:24.252665781Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-05-17T04:15:24.257600012Z","level":"INFO","msg":"stream: starting","core version":"0.27.0"}
+{"time":"2026-05-17T04:15:24.570111423Z","level":"INFO","msg":"stream: created new stream","id":"2uawripc"}
+{"time":"2026-05-17T04:15:24.570247927Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-05-17T04:15:24.57281714Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-05-17T04:15:24.572923179Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-05-17T04:15:24.572908137Z","level":"INFO","msg":"writer: started","stream_id":"2uawripc"}
+{"time":"2026-05-17T04:15:25.073444832Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":3}
+{"time":"2026-05-17T04:15:25.30138877Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:15:40.074549543Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":1,"console_offset":3,"console_lines":103,"uploaded_len":2}
+{"time":"2026-05-17T04:15:40.313093177Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:15:55.074283924Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":1,"events_lines":2,"console_offset":106,"console_lines":3}
+{"time":"2026-05-17T04:15:55.304141013Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:16:10.07434938Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":3,"events_lines":2}
+{"time":"2026-05-17T04:16:10.299733218Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:16:25.073800767Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":5,"events_lines":2}
+{"time":"2026-05-17T04:16:25.330648372Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:16:40.073844101Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":7,"events_lines":2}
+{"time":"2026-05-17T04:16:40.488519928Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:16:55.073500735Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":9,"events_lines":2}
+{"time":"2026-05-17T04:16:55.444103545Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:17:10.073993671Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":11,"events_lines":2,"console_offset":109,"console_lines":1}
+{"time":"2026-05-17T04:17:10.330392053Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:17:25.074098646Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":13,"events_lines":2}
+{"time":"2026-05-17T04:17:25.285529803Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:17:40.074184382Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":15,"events_lines":2}
+{"time":"2026-05-17T04:17:40.327792631Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:17:55.074296228Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":17,"events_lines":2}
+{"time":"2026-05-17T04:17:55.280187627Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:18:10.074068245Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":19,"events_lines":2}
+{"time":"2026-05-17T04:18:10.291555801Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:18:25.073661751Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":21,"events_lines":2}
+{"time":"2026-05-17T04:18:25.289334673Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:18:40.073769286Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":23,"events_lines":2}
+{"time":"2026-05-17T04:18:40.305416747Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:18:55.073921279Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":25,"events_lines":2}
+{"time":"2026-05-17T04:18:55.280846634Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:19:10.074083756Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":27,"events_lines":2}
+{"time":"2026-05-17T04:19:10.320163107Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:19:25.073778855Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":29,"events_lines":2}
+{"time":"2026-05-17T04:19:25.29088467Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

epfl_final/wandb/run-20260517_041523-2uawripc/logs/debug.log

@@ -0,0 +1,19 @@
+2026-05-17 04:15:24,012 INFO    MainThread:4155 [wandb_setup.py:_flush():81] Current SDK version is 0.27.0
+2026-05-17 04:15:24,012 INFO    MainThread:4155 [wandb_setup.py:_flush():81] Configure stats pid to 4155
+2026-05-17 04:15:24,013 INFO    MainThread:4155 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-17 04:15:24,014 INFO    MainThread:4155 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260517_041523-2uawripc/logs/debug.log
+2026-05-17 04:15:24,014 INFO    MainThread:4155 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260517_041523-2uawripc/logs/debug-internal.log
+2026-05-17 04:15:24,015 INFO    MainThread:4155 [wandb_init.py:init():851] calling init triggers
+2026-05-17 04:15:24,015 INFO    MainThread:4155 [wandb_init.py:init():856] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': 'checkpoints/run_20260517_023855', 'resume_from_checkpoint': 'checkpoints/run_20260517_023855/checkpoint-1000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'sparse 30%', 'wandb_tags': ['qwen3', '0.6b'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 18, 'sparse_mask_pool_size': 2000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_19', 'dense_mode': False, '_wandb': {}}
+2026-05-17 04:15:24,016 INFO    MainThread:4155 [wandb_init.py:init():899] starting backend
+2026-05-17 04:15:24,242 INFO    MainThread:4155 [wandb_init.py:init():914] sending inform_init request
+2026-05-17 04:15:24,573 INFO    MainThread:4155 [wandb_init.py:init():919] backend started and connected
+2026-05-17 04:15:24,579 INFO    MainThread:4155 [wandb_init.py:init():989] updated telemetry
+2026-05-17 04:15:24,580 INFO    MainThread:4155 [wandb_init.py:init():1012] communicating run to backend with 90.0 second timeout
+2026-05-17 04:15:24,976 INFO    MainThread:4155 [wandb_init.py:init():1057] starting run threads in backend
+2026-05-17 04:15:25,065 INFO    MainThread:4155 [wandb_run.py:_console_start():2509] atexit reg
+2026-05-17 04:15:25,066 INFO    MainThread:4155 [wandb_run.py:_redirect():2359] redirect: wrap_raw
+2026-05-17 04:15:25,066 INFO    MainThread:4155 [wandb_run.py:_redirect():2428] Wrapping output streams.
+2026-05-17 04:15:25,066 INFO    MainThread:4155 [wandb_run.py:_redirect():2451] Redirects installed.
+2026-05-17 04:15:25,069 INFO    MainThread:4155 [wandb_init.py:init():1095] run started, returning control to user process

epfl_final/wandb/run-20260517_041954-4vpsvlj0/files/output.log

@@ -0,0 +1,159 @@
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "checkpoints/run_20260517_023855",
+  "resume_from_checkpoint": "checkpoints/run_20260517_023855/checkpoint-1000",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 20005,
+  "per_device_batch_size": 3,
+  "gradient_accumulation_steps": 8,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 250,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 200,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "sparse 30%",
+  "wandb_tags": [
+    "qwen3",
+    "0.6b"
+  ],
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 18,
+  "sparse_mask_pool_size": 2000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_19",
+  "dense_mode": false
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: checkpoints/run_20260517_023855/checkpoint-1000
+Checkpoint activations: False
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1024
+  num_hidden_layers: 28
+  num_attention_heads: 16
+  num_key_value_heads: 8
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 18
+  sparse_mask_pool_size: 2000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 596,049,920
+  Trainable parameters: 596,049,920
+  Non-trainable parameters: 0
+  Model size (BF16): 1.19 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:2055: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [3, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [3, 8192, 1024]           --                        True
+│    └─Embedding: 2-1                                        [3, 8192]                 [3, 8192, 1024]           155,582,464               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    └─Qwen3RMSNorm: 2-3                                     [3, 8192, 1024]           [3, 8192, 1024]           1,024                     True
+├─Linear: 1-2                                                [3, 8192, 1024]           [3, 8192, 151936]         155,582,464               True
+================================================================================================================================================================
+Total params: 751,632,384
+Trainable params: 751,632,384
+Non-trainable params: 0
+Total mult-adds (G): 2.25
+================================================================================================================================================================
+Input size (MB): 0.20
+Forward/backward pass size (MB): 63052.97
+Params size (MB): 1503.26
+Estimated Total Size (MB): 64556.43
+================================================================================================================================================================
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003)
+Dataset split: 1017602 train, 254401 validation
+Training examples (packed, train split):  1017602
+Validation examples (packed, val split):  254401
+Per-device batch size:                    3
+World size (GPUs):                        1
+Gradient accumulation steps:              8
+Effective batch size (seqs/step):         24
+Sequence length:                          8192
+Tokens per optimizer step:                196,608
+Micro-batches per epoch (per rank):       339200
+Optimizer steps per epoch:                42400
+Total optimizer steps:                    20005  (capped by max_steps; 0.47 epochs)
+Loading checkpoint from checkpoints/run_20260517_023855/checkpoint-1000...
+Resumed from checkpoint at step 999 (epoch 0, micro_step 8000)
+Resuming training from step 999
+Starting training...
+Step 1000/20005 | Loss: 0.3998 | LR: 4.98e-04 | 35334.6ms/step | 0.7 samples/s

epfl_final/wandb/run-20260517_041954-4vpsvlj0/files/requirements.txt

@@ -0,0 +1,97 @@
+pytz==2026.2
+nvidia-cusparselt-cu13==0.8.1
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.7.0
+tzdata==2026.2
+typing_extensions==4.15.0
+triton==3.7.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+safetensors==0.7.0
+regex==2026.5.9
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.5.2
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.29.7
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+numpy==2.2.6
+networkx==3.4.2
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.15
+hf-xet==1.5.0
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.4.0
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+async-timeout==5.0.1
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+typing-inspection==0.4.2
+sentry-sdk==2.60.0
+requests==2.34.2
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.4
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cublas==13.1.1.3
+multiprocess==0.70.19
+multidict==6.7.1
+markdown-it-py==4.2.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+exceptiongroup==1.3.1
+cuda-bindings==13.2.0
+aiosignal==1.4.0
+yarl==1.23.0
+rich==15.0.0
+pydantic==2.13.4
+pandas==2.3.3
+nvidia-cusolver==12.0.4.66
+nvidia-cudnn-cu13==9.20.0.48
+GitPython==3.1.50
+anyio==4.13.0
+wandb==0.27.0
+typer==0.25.1
+httpx==0.28.1
+aiohttp==3.13.5
+torch==2.12.0
+huggingface_hub==1.15.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.8.1
+PyGObject==3.42.1
+dbus-python==1.2.18
+distro==1.7.0
+pip==22.0.2
+setuptools==59.6.0
+ssh-import-id==5.11
+wheel==0.37.1

epfl_final/wandb/run-20260517_041954-4vpsvlj0/files/wandb-metadata.json

@@ -0,0 +1,36 @@
+{
+  "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.12",
+  "startedAt": "2026-05-17T04:19:54.895094Z",
+  "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath": "SparseAtt/train.py",
+  "codePathLocal": "SparseAtt/train.py",
+  "email": "ruben.navasardyan@mail.utoronto.ca",
+  "root": "/mloscratch/homes/navasard/sparse",
+  "host": "h100-train-14-0-4",
+  "executable": "/usr/bin/python3",
+  "cpu_count": 96,
+  "cpu_count_logical": 192,
+  "gpu": "NVIDIA H100 80GB HBM3",
+  "gpu_count": 1,
+  "disk": {
+    "/": {
+      "total": "9374931091456",
+      "used": "6127724277760"
+    }
+  },
+  "memory": {
+    "total": "1623005847552"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-c65885f7-52dc-6f23-a3dd-6addf8b518a2"
+    }
+  ],
+  "cudaVersion": "13.0",
+  "writerId": "z3axhgap8xikpsr5hv0b0u2lf52cqs4j"
+}

epfl_final/wandb/run-20260517_041954-4vpsvlj0/logs/debug-internal.log

@@ -0,0 +1,39 @@
+{"time":"2026-05-17T04:19:55.145235254Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-05-17T04:19:55.145816614Z","level":"INFO","msg":"stream: starting","core version":"0.27.0"}
+{"time":"2026-05-17T04:19:55.475141122Z","level":"INFO","msg":"stream: created new stream","id":"4vpsvlj0"}
+{"time":"2026-05-17T04:19:55.475321842Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-05-17T04:19:55.477702894Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-05-17T04:19:55.477777796Z","level":"INFO","msg":"writer: started","stream_id":"4vpsvlj0"}
+{"time":"2026-05-17T04:19:55.477822163Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-05-17T04:19:55.881210021Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-05-17T04:19:56.20983354Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:20:10.882030463Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":1,"console_offset":1,"console_lines":48,"uploaded_len":2}
+{"time":"2026-05-17T04:20:11.169239905Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:20:25.882265429Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":1,"events_lines":2,"console_offset":49,"console_lines":92}
+{"time":"2026-05-17T04:20:26.10811153Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:20:40.882024564Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":3,"events_lines":2,"console_offset":141,"console_lines":17}
+{"time":"2026-05-17T04:20:41.102750135Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:20:55.881471191Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":5,"events_lines":2}
+{"time":"2026-05-17T04:20:56.093505559Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:21:10.881967177Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":7,"events_lines":2,"console_offset":158,"console_lines":1}
+{"time":"2026-05-17T04:21:11.094617738Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:21:25.882022414Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":9,"events_lines":2}
+{"time":"2026-05-17T04:21:26.085035177Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:21:40.882199013Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":11,"events_lines":2}
+{"time":"2026-05-17T04:21:41.10469963Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:21:55.881804718Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":13,"events_lines":2}
+{"time":"2026-05-17T04:21:56.087255618Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:22:10.881489952Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":15,"events_lines":2}
+{"time":"2026-05-17T04:22:11.108194718Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:22:25.881744587Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":17,"events_lines":2}
+{"time":"2026-05-17T04:22:26.088739347Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:22:40.88189659Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":19,"events_lines":2}
+{"time":"2026-05-17T04:22:41.0966243Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:22:55.882170282Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":21,"events_lines":2}
+{"time":"2026-05-17T04:22:56.095754582Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:23:10.881248806Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":23,"events_lines":2}
+{"time":"2026-05-17T04:23:11.094472456Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:23:25.881793674Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":25,"events_lines":2}
+{"time":"2026-05-17T04:23:26.098646534Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T04:23:40.88184299Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":27,"events_lines":2}
+{"time":"2026-05-17T04:23:41.11330076Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

epfl_final/wandb/run-20260517_041954-4vpsvlj0/logs/debug.log

@@ -0,0 +1,19 @@
+2026-05-17 04:19:54,912 INFO    MainThread:6565 [wandb_setup.py:_flush():81] Current SDK version is 0.27.0
+2026-05-17 04:19:54,912 INFO    MainThread:6565 [wandb_setup.py:_flush():81] Configure stats pid to 6565
+2026-05-17 04:19:54,912 INFO    MainThread:6565 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-17 04:19:54,913 INFO    MainThread:6565 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260517_041954-4vpsvlj0/logs/debug.log
+2026-05-17 04:19:54,914 INFO    MainThread:6565 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260517_041954-4vpsvlj0/logs/debug-internal.log
+2026-05-17 04:19:54,914 INFO    MainThread:6565 [wandb_init.py:init():851] calling init triggers
+2026-05-17 04:19:54,915 INFO    MainThread:6565 [wandb_init.py:init():856] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': 'checkpoints/run_20260517_023855', 'resume_from_checkpoint': 'checkpoints/run_20260517_023855/checkpoint-1000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'sparse 30%', 'wandb_tags': ['qwen3', '0.6b'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 18, 'sparse_mask_pool_size': 2000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_19', 'dense_mode': False, '_wandb': {}}
+2026-05-17 04:19:54,916 INFO    MainThread:6565 [wandb_init.py:init():899] starting backend
+2026-05-17 04:19:55,134 INFO    MainThread:6565 [wandb_init.py:init():914] sending inform_init request
+2026-05-17 04:19:55,478 INFO    MainThread:6565 [wandb_init.py:init():919] backend started and connected
+2026-05-17 04:19:55,481 INFO    MainThread:6565 [wandb_init.py:init():989] updated telemetry
+2026-05-17 04:19:55,482 INFO    MainThread:6565 [wandb_init.py:init():1012] communicating run to backend with 90.0 second timeout
+2026-05-17 04:19:55,788 INFO    MainThread:6565 [wandb_init.py:init():1057] starting run threads in backend
+2026-05-17 04:19:55,874 INFO    MainThread:6565 [wandb_run.py:_console_start():2509] atexit reg
+2026-05-17 04:19:55,875 INFO    MainThread:6565 [wandb_run.py:_redirect():2359] redirect: wrap_raw
+2026-05-17 04:19:55,875 INFO    MainThread:6565 [wandb_run.py:_redirect():2428] Wrapping output streams.
+2026-05-17 04:19:55,875 INFO    MainThread:6565 [wandb_run.py:_redirect():2451] Redirects installed.
+2026-05-17 04:19:55,877 INFO    MainThread:6565 [wandb_init.py:init():1095] run started, returning control to user process

epfl_final/wandb/run-20260517_042355-hi2z3jh6/files/output.log

@@ -0,0 +1,824 @@
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "checkpoints/run_20260517_023855",
+  "resume_from_checkpoint": "checkpoints/run_20260517_023855/checkpoint-1000",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 20005,
+  "per_device_batch_size": 3,
+  "gradient_accumulation_steps": 8,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 250,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 200,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "sparse 30%",
+  "wandb_tags": [
+    "qwen3",
+    "0.6b"
+  ],
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 18,
+  "sparse_mask_pool_size": 2000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_19",
+  "dense_mode": false
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: checkpoints/run_20260517_023855/checkpoint-1000
+Checkpoint activations: False
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1024
+  num_hidden_layers: 28
+  num_attention_heads: 16
+  num_key_value_heads: 8
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 18
+  sparse_mask_pool_size: 2000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 596,049,920
+  Trainable parameters: 596,049,920
+  Non-trainable parameters: 0
+  Model size (BF16): 1.19 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:2055: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [3, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [3, 8192, 1024]           --                        True
+│    └─Embedding: 2-1                                        [3, 8192]                 [3, 8192, 1024]           155,582,464               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    └─Qwen3RMSNorm: 2-3                                     [3, 8192, 1024]           [3, 8192, 1024]           1,024                     True
+├─Linear: 1-2                                                [3, 8192, 1024]           [3, 8192, 151936]         155,582,464               True
+================================================================================================================================================================
+Total params: 751,632,384
+Trainable params: 751,632,384
+Non-trainable params: 0
+Total mult-adds (G): 2.25
+================================================================================================================================================================
+Input size (MB): 0.20
+Forward/backward pass size (MB): 63052.97
+Params size (MB): 1503.26
+Estimated Total Size (MB): 64556.43
+================================================================================================================================================================
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003)
+Dataset split: 1017602 train, 254401 validation
+Training examples (packed, train split):  1017602
+Validation examples (packed, val split):  254401
+Per-device batch size:                    3
+World size (GPUs):                        1
+Gradient accumulation steps:              8
+Effective batch size (seqs/step):         24
+Sequence length:                          8192
+Tokens per optimizer step:                196,608
+Micro-batches per epoch (per rank):       339200
+Optimizer steps per epoch:                42400
+Total optimizer steps:                    20005  (capped by max_steps; 0.47 epochs)
+Loading checkpoint from checkpoints/run_20260517_023855/checkpoint-1000...
+Resumed from checkpoint at step 999 (epoch 0, micro_step 8000)
+Resuming training from step 999
+Starting training...
+Step 1000/20005 | Loss: 0.3998 | LR: 4.98e-04 | 35522.8ms/step | 0.7 samples/s
+wandb: WARNING Tried to log to step 1000 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Validation at step 1000: Loss = 3.9606, Perplexity = 52.49
+Saved checkpoint to checkpoints/run_20260517_023855/checkpoint-1000
+wandb: WARNING Tried to log to step 1000 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1010/20005 | Loss: 3.9276 | LR: 4.98e-04 | 19084.3ms/step | 1.3 samples/s
+wandb: WARNING Tried to log to step 1010 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1020/20005 | Loss: 3.9534 | LR: 4.98e-04 | 2297.9ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1020 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1030/20005 | Loss: 3.9302 | LR: 4.98e-04 | 2299.8ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1030 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1040/20005 | Loss: 3.9131 | LR: 4.98e-04 | 2299.3ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1040 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1050/20005 | Loss: 3.9269 | LR: 4.98e-04 | 2298.4ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1050 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1060/20005 | Loss: 3.9039 | LR: 4.98e-04 | 2300.1ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1060 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1070/20005 | Loss: 3.9059 | LR: 4.98e-04 | 2298.5ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1070 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1080/20005 | Loss: 3.9133 | LR: 4.98e-04 | 2299.1ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1080 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1090/20005 | Loss: 3.9251 | LR: 4.98e-04 | 2298.1ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1090 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1100/20005 | Loss: 3.8744 | LR: 4.98e-04 | 2298.5ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1100 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1110/20005 | Loss: 3.8858 | LR: 4.98e-04 | 2298.7ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1110 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1120/20005 | Loss: 3.8987 | LR: 4.98e-04 | 2299.2ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1120 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1130/20005 | Loss: 3.8878 | LR: 4.98e-04 | 2299.6ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1130 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1140/20005 | Loss: 3.8660 | LR: 4.97e-04 | 2297.3ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1140 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1150/20005 | Loss: 3.8729 | LR: 4.97e-04 | 2299.0ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1150 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1160/20005 | Loss: 3.8615 | LR: 4.97e-04 | 2298.6ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1160 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1170/20005 | Loss: 3.8681 | LR: 4.97e-04 | 2299.1ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1170 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1180/20005 | Loss: 3.8544 | LR: 4.97e-04 | 2297.3ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1180 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1190/20005 | Loss: 3.8422 | LR: 4.97e-04 | 2298.8ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1190 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1200/20005 | Loss: 3.8391 | LR: 4.97e-04 | 2297.6ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1200 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1210/20005 | Loss: 3.8169 | LR: 4.97e-04 | 2298.4ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1210 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1220/20005 | Loss: 3.8116 | LR: 4.97e-04 | 2296.3ms/step | 10.5 samples/s
+wandb: WARNING Tried to log to step 1220 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1230/20005 | Loss: 3.8485 | LR: 4.97e-04 | 2297.8ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1230 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1240/20005 | Loss: 3.7997 | LR: 4.97e-04 | 2298.6ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1240 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1250/20005 | Loss: 3.8127 | LR: 4.97e-04 | 2299.3ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1250 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Validation at step 1250: Loss = 3.8110, Perplexity = 45.20
+wandb: WARNING Tried to log to step 1250 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1260/20005 | Loss: 3.8067 | LR: 4.97e-04 | 17398.2ms/step | 1.4 samples/s
+wandb: WARNING Tried to log to step 1260 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1270/20005 | Loss: 3.8315 | LR: 4.97e-04 | 2301.8ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1270 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1280/20005 | Loss: 3.7741 | LR: 4.97e-04 | 2300.6ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1280 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1290/20005 | Loss: 3.7474 | LR: 4.97e-04 | 2302.2ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1290 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1300/20005 | Loss: 3.7980 | LR: 4.97e-04 | 2299.2ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1300 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1310/20005 | Loss: 3.8003 | LR: 4.96e-04 | 2300.2ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1310 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1320/20005 | Loss: 3.7876 | LR: 4.96e-04 | 2297.9ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 1320 that is less than the current step 1321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 1330/20005 | Loss: 3.7562 | LR: 4.96e-04 | 2294.7ms/step | 10.5 samples/s
+Step 1340/20005 | Loss: 3.7852 | LR: 4.96e-04 | 2299.3ms/step | 10.4 samples/s
+Step 1350/20005 | Loss: 3.7429 | LR: 4.96e-04 | 2298.3ms/step | 10.4 samples/s
+Step 1360/20005 | Loss: 3.7881 | LR: 4.96e-04 | 2298.8ms/step | 10.4 samples/s
+Step 1370/20005 | Loss: 3.7557 | LR: 4.96e-04 | 2297.1ms/step | 10.4 samples/s
+Step 1380/20005 | Loss: 3.7201 | LR: 4.96e-04 | 2300.2ms/step | 10.4 samples/s
+Step 1390/20005 | Loss: 3.7457 | LR: 4.96e-04 | 2299.3ms/step | 10.4 samples/s
+Step 1400/20005 | Loss: 3.7639 | LR: 4.96e-04 | 2298.2ms/step | 10.4 samples/s
+Step 1410/20005 | Loss: 3.7374 | LR: 4.96e-04 | 2297.1ms/step | 10.4 samples/s
+Step 1420/20005 | Loss: 3.7420 | LR: 4.96e-04 | 2299.7ms/step | 10.4 samples/s
+Step 1430/20005 | Loss: 3.7419 | LR: 4.96e-04 | 2299.0ms/step | 10.4 samples/s
+Step 1440/20005 | Loss: 3.7317 | LR: 4.96e-04 | 2298.7ms/step | 10.4 samples/s
+Step 1450/20005 | Loss: 3.7051 | LR: 4.95e-04 | 2298.6ms/step | 10.4 samples/s
+Step 1460/20005 | Loss: 3.7073 | LR: 4.95e-04 | 2296.4ms/step | 10.5 samples/s
+Step 1470/20005 | Loss: 3.7299 | LR: 4.95e-04 | 2298.8ms/step | 10.4 samples/s
+Step 1480/20005 | Loss: 3.7257 | LR: 4.95e-04 | 2300.6ms/step | 10.4 samples/s
+Step 1490/20005 | Loss: 3.6955 | LR: 4.95e-04 | 2297.8ms/step | 10.4 samples/s
+Step 1500/20005 | Loss: 3.7397 | LR: 4.95e-04 | 2298.1ms/step | 10.4 samples/s
+Validation at step 1500: Loss = 3.7066, Perplexity = 40.72
+Step 1510/20005 | Loss: 3.7021 | LR: 4.95e-04 | 17291.7ms/step | 1.4 samples/s
+Step 1520/20005 | Loss: 3.6917 | LR: 4.95e-04 | 2298.2ms/step | 10.4 samples/s
+Step 1530/20005 | Loss: 3.7007 | LR: 4.95e-04 | 2297.6ms/step | 10.4 samples/s
+Step 1540/20005 | Loss: 3.6938 | LR: 4.95e-04 | 2298.7ms/step | 10.4 samples/s
+Step 1550/20005 | Loss: 3.6861 | LR: 4.95e-04 | 2297.7ms/step | 10.4 samples/s
+Step 1560/20005 | Loss: 3.6639 | LR: 4.95e-04 | 2297.6ms/step | 10.4 samples/s
+Step 1570/20005 | Loss: 3.6871 | LR: 4.95e-04 | 2299.1ms/step | 10.4 samples/s
+Step 1580/20005 | Loss: 3.6975 | LR: 4.94e-04 | 2294.8ms/step | 10.5 samples/s
+Step 1590/20005 | Loss: 3.6826 | LR: 4.94e-04 | 2297.7ms/step | 10.4 samples/s
+Step 1600/20005 | Loss: 3.6514 | LR: 4.94e-04 | 2296.3ms/step | 10.5 samples/s
+Step 1610/20005 | Loss: 3.6636 | LR: 4.94e-04 | 2296.7ms/step | 10.4 samples/s
+Step 1620/20005 | Loss: 3.6964 | LR: 4.94e-04 | 2295.4ms/step | 10.5 samples/s
+Step 1630/20005 | Loss: 3.6471 | LR: 4.94e-04 | 2295.9ms/step | 10.5 samples/s
+Step 1640/20005 | Loss: 3.6661 | LR: 4.94e-04 | 2295.6ms/step | 10.5 samples/s
+Step 1650/20005 | Loss: 3.6637 | LR: 4.94e-04 | 2295.7ms/step | 10.5 samples/s
+Step 1660/20005 | Loss: 3.6418 | LR: 4.94e-04 | 2296.2ms/step | 10.5 samples/s
+Step 1670/20005 | Loss: 3.6548 | LR: 4.94e-04 | 2294.6ms/step | 10.5 samples/s
+Step 1680/20005 | Loss: 3.6420 | LR: 4.94e-04 | 2296.5ms/step | 10.5 samples/s
+Step 1690/20005 | Loss: 3.6254 | LR: 4.93e-04 | 2297.6ms/step | 10.4 samples/s
+Step 1700/20005 | Loss: 3.6323 | LR: 4.93e-04 | 2296.9ms/step | 10.4 samples/s
+Step 1710/20005 | Loss: 3.6316 | LR: 4.93e-04 | 2296.6ms/step | 10.5 samples/s
+Step 1720/20005 | Loss: 3.6314 | LR: 4.93e-04 | 2296.2ms/step | 10.5 samples/s
+Step 1730/20005 | Loss: 3.6171 | LR: 4.93e-04 | 2295.2ms/step | 10.5 samples/s
+Step 1740/20005 | Loss: 3.6403 | LR: 4.93e-04 | 2296.8ms/step | 10.4 samples/s
+Step 1750/20005 | Loss: 3.6444 | LR: 4.93e-04 | 2295.6ms/step | 10.5 samples/s
+Validation at step 1750: Loss = 3.6232, Perplexity = 37.46
+Step 1760/20005 | Loss: 3.6255 | LR: 4.93e-04 | 17278.0ms/step | 1.4 samples/s
+Step 1770/20005 | Loss: 3.6134 | LR: 4.93e-04 | 2296.0ms/step | 10.5 samples/s
+Step 1780/20005 | Loss: 3.6407 | LR: 4.93e-04 | 2297.9ms/step | 10.4 samples/s
+Step 1790/20005 | Loss: 3.6292 | LR: 4.93e-04 | 2296.9ms/step | 10.4 samples/s
+Step 1800/20005 | Loss: 3.6005 | LR: 4.92e-04 | 2296.2ms/step | 10.5 samples/s
+Step 1810/20005 | Loss: 3.5947 | LR: 4.92e-04 | 2297.7ms/step | 10.4 samples/s
+Step 1820/20005 | Loss: 3.6164 | LR: 4.92e-04 | 2435.3ms/step | 9.9 samples/s
+Step 1830/20005 | Loss: 3.5960 | LR: 4.92e-04 | 2298.3ms/step | 10.4 samples/s
+Step 1840/20005 | Loss: 3.5775 | LR: 4.92e-04 | 2298.1ms/step | 10.4 samples/s
+Step 1850/20005 | Loss: 3.6085 | LR: 4.92e-04 | 2299.3ms/step | 10.4 samples/s
+Step 1860/20005 | Loss: 3.6199 | LR: 4.92e-04 | 2296.4ms/step | 10.5 samples/s
+Step 1870/20005 | Loss: 3.6121 | LR: 4.92e-04 | 2299.1ms/step | 10.4 samples/s
+Step 1880/20005 | Loss: 3.5697 | LR: 4.92e-04 | 2297.5ms/step | 10.4 samples/s
+Step 1890/20005 | Loss: 3.5982 | LR: 4.92e-04 | 2299.2ms/step | 10.4 samples/s
+Step 1900/20005 | Loss: 3.5975 | LR: 4.91e-04 | 2300.0ms/step | 10.4 samples/s
+Step 1910/20005 | Loss: 3.5777 | LR: 4.91e-04 | 2297.8ms/step | 10.4 samples/s
+Step 1920/20005 | Loss: 3.5641 | LR: 4.91e-04 | 2297.9ms/step | 10.4 samples/s
+Step 1930/20005 | Loss: 3.5782 | LR: 4.91e-04 | 2298.2ms/step | 10.4 samples/s
+Step 1940/20005 | Loss: 3.5899 | LR: 4.91e-04 | 2299.5ms/step | 10.4 samples/s
+Step 1950/20005 | Loss: 3.5848 | LR: 4.91e-04 | 2300.4ms/step | 10.4 samples/s
+Step 1960/20005 | Loss: 3.5971 | LR: 4.91e-04 | 2296.3ms/step | 10.5 samples/s
+Step 1970/20005 | Loss: 3.5468 | LR: 4.91e-04 | 2297.4ms/step | 10.4 samples/s
+Step 1980/20005 | Loss: 3.5792 | LR: 4.91e-04 | 2314.5ms/step | 10.4 samples/s
+Step 1990/20005 | Loss: 3.5597 | LR: 4.90e-04 | 2307.3ms/step | 10.4 samples/s
+Step 2000/20005 | Loss: 3.5800 | LR: 4.90e-04 | 2309.2ms/step | 10.4 samples/s
+Validation at step 2000: Loss = 3.5615, Perplexity = 35.22
+Saved checkpoint to checkpoints/run_20260517_023855/checkpoint-2000
+Step 2010/20005 | Loss: 3.5639 | LR: 4.90e-04 | 17812.0ms/step | 1.3 samples/s
+Step 2020/20005 | Loss: 3.5651 | LR: 4.90e-04 | 2295.4ms/step | 10.5 samples/s
+Step 2030/20005 | Loss: 3.5786 | LR: 4.90e-04 | 2296.8ms/step | 10.4 samples/s
+Step 2040/20005 | Loss: 3.5739 | LR: 4.90e-04 | 2298.7ms/step | 10.4 samples/s
+Step 2050/20005 | Loss: 3.5705 | LR: 4.90e-04 | 2297.1ms/step | 10.4 samples/s
+Step 2060/20005 | Loss: 3.5291 | LR: 4.90e-04 | 2299.3ms/step | 10.4 samples/s
+Step 2070/20005 | Loss: 3.5556 | LR: 4.90e-04 | 2296.6ms/step | 10.5 samples/s
+Step 2080/20005 | Loss: 3.5463 | LR: 4.89e-04 | 2297.6ms/step | 10.4 samples/s
+Step 2090/20005 | Loss: 3.5461 | LR: 4.89e-04 | 2297.9ms/step | 10.4 samples/s
+Step 2100/20005 | Loss: 3.5473 | LR: 4.89e-04 | 2297.6ms/step | 10.4 samples/s
+Step 2110/20005 | Loss: 3.5402 | LR: 4.89e-04 | 2298.5ms/step | 10.4 samples/s
+Step 2120/20005 | Loss: 3.5312 | LR: 4.89e-04 | 2297.6ms/step | 10.4 samples/s
+Step 2130/20005 | Loss: 3.5331 | LR: 4.89e-04 | 2297.2ms/step | 10.4 samples/s
+Step 2140/20005 | Loss: 3.5563 | LR: 4.89e-04 | 2298.4ms/step | 10.4 samples/s
+Step 2150/20005 | Loss: 3.5249 | LR: 4.89e-04 | 2297.0ms/step | 10.4 samples/s
+Step 2160/20005 | Loss: 3.5221 | LR: 4.89e-04 | 2297.4ms/step | 10.4 samples/s
+Step 2170/20005 | Loss: 3.5136 | LR: 4.88e-04 | 2297.8ms/step | 10.4 samples/s
+Step 2180/20005 | Loss: 3.4931 | LR: 4.88e-04 | 2296.4ms/step | 10.5 samples/s
+Step 2190/20005 | Loss: 3.5300 | LR: 4.88e-04 | 2297.9ms/step | 10.4 samples/s
+Step 2200/20005 | Loss: 3.5344 | LR: 4.88e-04 | 2298.0ms/step | 10.4 samples/s
+Step 2210/20005 | Loss: 3.5477 | LR: 4.88e-04 | 2296.8ms/step | 10.4 samples/s
+Step 2220/20005 | Loss: 3.5295 | LR: 4.88e-04 | 2297.9ms/step | 10.4 samples/s
+Step 2230/20005 | Loss: 3.5041 | LR: 4.88e-04 | 2300.2ms/step | 10.4 samples/s
+Step 2240/20005 | Loss: 3.5213 | LR: 4.88e-04 | 2299.0ms/step | 10.4 samples/s
+Step 2250/20005 | Loss: 3.4978 | LR: 4.87e-04 | 2298.2ms/step | 10.4 samples/s
+Validation at step 2250: Loss = 3.5080, Perplexity = 33.38
+Step 2260/20005 | Loss: 3.4940 | LR: 4.87e-04 | 17284.2ms/step | 1.4 samples/s
+Step 2270/20005 | Loss: 3.5003 | LR: 4.87e-04 | 2299.3ms/step | 10.4 samples/s
+Step 2280/20005 | Loss: 3.5254 | LR: 4.87e-04 | 2297.6ms/step | 10.4 samples/s
+Step 2290/20005 | Loss: 3.5128 | LR: 4.87e-04 | 2297.7ms/step | 10.4 samples/s
+Step 2300/20005 | Loss: 3.4993 | LR: 4.87e-04 | 2297.8ms/step | 10.4 samples/s
+Step 2310/20005 | Loss: 3.5023 | LR: 4.87e-04 | 2300.1ms/step | 10.4 samples/s
+Step 2320/20005 | Loss: 3.4985 | LR: 4.87e-04 | 2299.3ms/step | 10.4 samples/s
+Step 2330/20005 | Loss: 3.4773 | LR: 4.86e-04 | 2299.2ms/step | 10.4 samples/s
+Step 2340/20005 | Loss: 3.4738 | LR: 4.86e-04 | 2298.0ms/step | 10.4 samples/s
+Step 2350/20005 | Loss: 3.4968 | LR: 4.86e-04 | 2299.1ms/step | 10.4 samples/s
+Step 2360/20005 | Loss: 3.4823 | LR: 4.86e-04 | 2298.0ms/step | 10.4 samples/s
+Step 2370/20005 | Loss: 3.4963 | LR: 4.86e-04 | 2299.8ms/step | 10.4 samples/s
+Step 2380/20005 | Loss: 3.4879 | LR: 4.86e-04 | 2296.7ms/step | 10.4 samples/s
+Step 2390/20005 | Loss: 3.4928 | LR: 4.86e-04 | 2299.2ms/step | 10.4 samples/s
+Step 2400/20005 | Loss: 3.5022 | LR: 4.86e-04 | 2298.4ms/step | 10.4 samples/s
+Step 2410/20005 | Loss: 3.4629 | LR: 4.85e-04 | 2299.3ms/step | 10.4 samples/s
+Step 2420/20005 | Loss: 3.4705 | LR: 4.85e-04 | 2295.4ms/step | 10.5 samples/s
+Step 2430/20005 | Loss: 3.4690 | LR: 4.85e-04 | 2298.1ms/step | 10.4 samples/s
+Step 2440/20005 | Loss: 3.4900 | LR: 4.85e-04 | 2297.4ms/step | 10.4 samples/s
+Step 2450/20005 | Loss: 3.4738 | LR: 4.85e-04 | 2299.0ms/step | 10.4 samples/s
+Step 2460/20005 | Loss: 3.4627 | LR: 4.85e-04 | 2296.2ms/step | 10.5 samples/s
+Step 2470/20005 | Loss: 3.4742 | LR: 4.85e-04 | 2298.2ms/step | 10.4 samples/s
+Step 2480/20005 | Loss: 3.4923 | LR: 4.84e-04 | 2298.2ms/step | 10.4 samples/s
+Step 2490/20005 | Loss: 3.4722 | LR: 4.84e-04 | 2296.2ms/step | 10.5 samples/s
+Step 2500/20005 | Loss: 3.4673 | LR: 4.84e-04 | 2298.8ms/step | 10.4 samples/s
+Validation at step 2500: Loss = 3.4651, Perplexity = 31.98
+Step 2510/20005 | Loss: 3.4436 | LR: 4.84e-04 | 17273.2ms/step | 1.4 samples/s
+Step 2520/20005 | Loss: 3.5013 | LR: 4.84e-04 | 2296.1ms/step | 10.5 samples/s
+Step 2530/20005 | Loss: 3.4774 | LR: 4.84e-04 | 2298.5ms/step | 10.4 samples/s
+Step 2540/20005 | Loss: 3.4287 | LR: 4.84e-04 | 2296.7ms/step | 10.4 samples/s
+Step 2550/20005 | Loss: 3.4625 | LR: 4.83e-04 | 2296.3ms/step | 10.5 samples/s
+Step 2560/20005 | Loss: 3.4592 | LR: 4.83e-04 | 2296.0ms/step | 10.5 samples/s
+Step 2570/20005 | Loss: 3.4703 | LR: 4.83e-04 | 2296.8ms/step | 10.4 samples/s
+Step 2580/20005 | Loss: 3.4668 | LR: 4.83e-04 | 2296.4ms/step | 10.5 samples/s
+Step 2590/20005 | Loss: 3.4395 | LR: 4.83e-04 | 2296.4ms/step | 10.5 samples/s
+Step 2600/20005 | Loss: 3.4420 | LR: 4.83e-04 | 2297.7ms/step | 10.4 samples/s
+Step 2610/20005 | Loss: 3.4448 | LR: 4.83e-04 | 2298.2ms/step | 10.4 samples/s
+Step 2620/20005 | Loss: 3.4322 | LR: 4.82e-04 | 2296.2ms/step | 10.5 samples/s
+Step 2630/20005 | Loss: 3.4073 | LR: 4.82e-04 | 2297.8ms/step | 10.4 samples/s
+Step 2640/20005 | Loss: 3.4410 | LR: 4.82e-04 | 2296.1ms/step | 10.5 samples/s
+Step 2650/20005 | Loss: 3.4573 | LR: 4.82e-04 | 2298.2ms/step | 10.4 samples/s
+Step 2660/20005 | Loss: 3.4249 | LR: 4.82e-04 | 2296.7ms/step | 10.4 samples/s
+Step 2670/20005 | Loss: 3.4524 | LR: 4.82e-04 | 2297.9ms/step | 10.4 samples/s
+Step 2680/20005 | Loss: 3.4188 | LR: 4.82e-04 | 2298.2ms/step | 10.4 samples/s
+Step 2690/20005 | Loss: 3.4476 | LR: 4.81e-04 | 2296.6ms/step | 10.5 samples/s
+Step 2700/20005 | Loss: 3.4084 | LR: 4.81e-04 | 2297.2ms/step | 10.4 samples/s
+Step 2710/20005 | Loss: 3.4404 | LR: 4.81e-04 | 2298.4ms/step | 10.4 samples/s
+Step 2720/20005 | Loss: 3.4263 | LR: 4.81e-04 | 2296.2ms/step | 10.5 samples/s
+Step 2730/20005 | Loss: 3.4218 | LR: 4.81e-04 | 2298.1ms/step | 10.4 samples/s
+Step 2740/20005 | Loss: 3.4188 | LR: 4.81e-04 | 2298.2ms/step | 10.4 samples/s
+Step 2750/20005 | Loss: 3.4393 | LR: 4.80e-04 | 2297.3ms/step | 10.4 samples/s
+Validation at step 2750: Loss = 3.4264, Perplexity = 30.77
+Step 2760/20005 | Loss: 3.4406 | LR: 4.80e-04 | 17403.2ms/step | 1.4 samples/s
+Step 2770/20005 | Loss: 3.4319 | LR: 4.80e-04 | 2297.7ms/step | 10.4 samples/s
+Step 2780/20005 | Loss: 3.4174 | LR: 4.80e-04 | 2297.8ms/step | 10.4 samples/s
+Step 2790/20005 | Loss: 3.4255 | LR: 4.80e-04 | 2297.7ms/step | 10.4 samples/s
+Step 2800/20005 | Loss: 3.3918 | LR: 4.80e-04 | 2296.7ms/step | 10.4 samples/s
+Step 2810/20005 | Loss: 3.4305 | LR: 4.80e-04 | 2297.6ms/step | 10.4 samples/s
+Step 2820/20005 | Loss: 3.4067 | LR: 4.79e-04 | 2298.6ms/step | 10.4 samples/s
+Step 2830/20005 | Loss: 3.4237 | LR: 4.79e-04 | 2297.7ms/step | 10.4 samples/s
+Step 2840/20005 | Loss: 3.4003 | LR: 4.79e-04 | 2298.7ms/step | 10.4 samples/s
+Step 2850/20005 | Loss: 3.4233 | LR: 4.79e-04 | 2297.7ms/step | 10.4 samples/s
+Step 2860/20005 | Loss: 3.3977 | LR: 4.79e-04 | 2299.1ms/step | 10.4 samples/s
+Step 2870/20005 | Loss: 3.3976 | LR: 4.79e-04 | 2300.8ms/step | 10.4 samples/s
+Step 2880/20005 | Loss: 3.3941 | LR: 4.78e-04 | 2299.5ms/step | 10.4 samples/s
+Step 2890/20005 | Loss: 3.3950 | LR: 4.78e-04 | 2296.6ms/step | 10.5 samples/s
+Step 2900/20005 | Loss: 3.4301 | LR: 4.78e-04 | 2297.1ms/step | 10.4 samples/s
+Step 2910/20005 | Loss: 3.3844 | LR: 4.78e-04 | 2297.7ms/step | 10.4 samples/s
+Step 2920/20005 | Loss: 3.4093 | LR: 4.78e-04 | 2298.8ms/step | 10.4 samples/s
+Step 2930/20005 | Loss: 3.3665 | LR: 4.78e-04 | 2296.4ms/step | 10.5 samples/s
+Step 2940/20005 | Loss: 3.3739 | LR: 4.77e-04 | 2297.2ms/step | 10.4 samples/s
+Step 2950/20005 | Loss: 3.4081 | LR: 4.77e-04 | 2295.6ms/step | 10.5 samples/s
+Step 2960/20005 | Loss: 3.3755 | LR: 4.77e-04 | 2296.6ms/step | 10.5 samples/s
+Step 2970/20005 | Loss: 3.4158 | LR: 4.77e-04 | 2296.6ms/step | 10.5 samples/s
+Step 2980/20005 | Loss: 3.4059 | LR: 4.77e-04 | 2296.8ms/step | 10.4 samples/s
+Step 2990/20005 | Loss: 3.3629 | LR: 4.77e-04 | 2297.7ms/step | 10.4 samples/s
+Step 3000/20005 | Loss: 3.3750 | LR: 4.76e-04 | 2298.4ms/step | 10.4 samples/s
+Validation at step 3000: Loss = 3.3943, Perplexity = 29.79
+Saved checkpoint to checkpoints/run_20260517_023855/checkpoint-3000
+Step 3010/20005 | Loss: 3.4043 | LR: 4.76e-04 | 17802.0ms/step | 1.3 samples/s
+Step 3020/20005 | Loss: 3.3785 | LR: 4.76e-04 | 2309.0ms/step | 10.4 samples/s
+Step 3030/20005 | Loss: 3.3743 | LR: 4.76e-04 | 2310.7ms/step | 10.4 samples/s
+Step 3040/20005 | Loss: 3.3921 | LR: 4.76e-04 | 2310.2ms/step | 10.4 samples/s
+Step 3050/20005 | Loss: 3.3710 | LR: 4.76e-04 | 2309.6ms/step | 10.4 samples/s
+Step 3060/20005 | Loss: 3.3760 | LR: 4.75e-04 | 2311.2ms/step | 10.4 samples/s
+Step 3070/20005 | Loss: 3.3840 | LR: 4.75e-04 | 2315.9ms/step | 10.4 samples/s
+Step 3080/20005 | Loss: 3.3580 | LR: 4.75e-04 | 2319.1ms/step | 10.3 samples/s
+Step 3090/20005 | Loss: 3.3722 | LR: 4.75e-04 | 2316.4ms/step | 10.4 samples/s
+Step 3100/20005 | Loss: 3.3822 | LR: 4.75e-04 | 2317.5ms/step | 10.4 samples/s
+Step 3110/20005 | Loss: 3.3517 | LR: 4.75e-04 | 2310.4ms/step | 10.4 samples/s
+Step 3120/20005 | Loss: 3.3717 | LR: 4.74e-04 | 2310.2ms/step | 10.4 samples/s
+Step 3130/20005 | Loss: 3.3568 | LR: 4.74e-04 | 2311.8ms/step | 10.4 samples/s
+Step 3140/20005 | Loss: 3.3643 | LR: 4.74e-04 | 2313.7ms/step | 10.4 samples/s
+Step 3150/20005 | Loss: 3.3959 | LR: 4.74e-04 | 2316.4ms/step | 10.4 samples/s
+Step 3160/20005 | Loss: 3.3789 | LR: 4.74e-04 | 2314.1ms/step | 10.4 samples/s
+Step 3170/20005 | Loss: 3.3773 | LR: 4.74e-04 | 2310.9ms/step | 10.4 samples/s
+Step 3180/20005 | Loss: 3.3774 | LR: 4.73e-04 | 2311.0ms/step | 10.4 samples/s
+Step 3190/20005 | Loss: 3.3797 | LR: 4.73e-04 | 2310.7ms/step | 10.4 samples/s
+Step 3200/20005 | Loss: 3.3721 | LR: 4.73e-04 | 2312.0ms/step | 10.4 samples/s
+Step 3210/20005 | Loss: 3.3825 | LR: 4.73e-04 | 2309.9ms/step | 10.4 samples/s
+Step 3220/20005 | Loss: 3.3641 | LR: 4.73e-04 | 2310.5ms/step | 10.4 samples/s
+Step 3230/20005 | Loss: 3.3721 | LR: 4.72e-04 | 2310.3ms/step | 10.4 samples/s
+Step 3240/20005 | Loss: 3.3833 | LR: 4.72e-04 | 2311.2ms/step | 10.4 samples/s
+Step 3250/20005 | Loss: 3.3775 | LR: 4.72e-04 | 2311.4ms/step | 10.4 samples/s
+Validation at step 3250: Loss = 3.3670, Perplexity = 28.99
+Step 3260/20005 | Loss: 3.3747 | LR: 4.72e-04 | 17408.7ms/step | 1.4 samples/s
+Step 3270/20005 | Loss: 3.3522 | LR: 4.72e-04 | 2301.9ms/step | 10.4 samples/s
+Step 3280/20005 | Loss: 3.3638 | LR: 4.72e-04 | 2305.6ms/step | 10.4 samples/s
+Step 3290/20005 | Loss: 3.3459 | LR: 4.71e-04 | 2306.4ms/step | 10.4 samples/s
+Step 3300/20005 | Loss: 3.3603 | LR: 4.71e-04 | 2305.7ms/step | 10.4 samples/s
+Step 3310/20005 | Loss: 3.3736 | LR: 4.71e-04 | 2307.6ms/step | 10.4 samples/s
+Step 3320/20005 | Loss: 3.3552 | LR: 4.71e-04 | 2304.0ms/step | 10.4 samples/s
+Step 3330/20005 | Loss: 3.3894 | LR: 4.71e-04 | 2302.3ms/step | 10.4 samples/s
+Step 3340/20005 | Loss: 3.3773 | LR: 4.70e-04 | 2304.4ms/step | 10.4 samples/s
+Step 3350/20005 | Loss: 3.3649 | LR: 4.70e-04 | 2302.7ms/step | 10.4 samples/s
+Step 3360/20005 | Loss: 3.3615 | LR: 4.70e-04 | 2302.7ms/step | 10.4 samples/s
+Step 3370/20005 | Loss: 3.3461 | LR: 4.70e-04 | 2306.5ms/step | 10.4 samples/s
+Step 3380/20005 | Loss: 3.3689 | LR: 4.70e-04 | 2305.3ms/step | 10.4 samples/s
+Step 3390/20005 | Loss: 3.3048 | LR: 4.69e-04 | 2303.9ms/step | 10.4 samples/s
+Step 3400/20005 | Loss: 3.3553 | LR: 4.69e-04 | 2305.4ms/step | 10.4 samples/s
+Step 3410/20005 | Loss: 3.3456 | LR: 4.69e-04 | 2302.3ms/step | 10.4 samples/s
+Step 3420/20005 | Loss: 3.3120 | LR: 4.69e-04 | 2303.5ms/step | 10.4 samples/s
+Step 3430/20005 | Loss: 3.3585 | LR: 4.69e-04 | 2302.8ms/step | 10.4 samples/s
+Step 3440/20005 | Loss: 3.3637 | LR: 4.68e-04 | 2304.2ms/step | 10.4 samples/s
+Step 3450/20005 | Loss: 3.3342 | LR: 4.68e-04 | 2304.6ms/step | 10.4 samples/s
+Step 3460/20005 | Loss: 3.3428 | LR: 4.68e-04 | 2304.1ms/step | 10.4 samples/s
+Step 3470/20005 | Loss: 3.3394 | LR: 4.68e-04 | 2305.0ms/step | 10.4 samples/s
+Step 3480/20005 | Loss: 3.3323 | LR: 4.68e-04 | 2306.2ms/step | 10.4 samples/s
+Step 3490/20005 | Loss: 3.3316 | LR: 4.68e-04 | 2302.7ms/step | 10.4 samples/s
+Step 3500/20005 | Loss: 3.3446 | LR: 4.67e-04 | 2304.1ms/step | 10.4 samples/s
+Validation at step 3500: Loss = 3.3386, Perplexity = 28.18
+Step 3510/20005 | Loss: 3.3336 | LR: 4.67e-04 | 17351.5ms/step | 1.4 samples/s
+Step 3520/20005 | Loss: 3.3429 | LR: 4.67e-04 | 2300.2ms/step | 10.4 samples/s
+Step 3530/20005 | Loss: 3.3552 | LR: 4.67e-04 | 2301.5ms/step | 10.4 samples/s
+Step 3540/20005 | Loss: 3.3469 | LR: 4.67e-04 | 2302.0ms/step | 10.4 samples/s
+Step 3550/20005 | Loss: 3.3403 | LR: 4.66e-04 | 2300.2ms/step | 10.4 samples/s
+Step 3560/20005 | Loss: 3.3176 | LR: 4.66e-04 | 2301.8ms/step | 10.4 samples/s
+Step 3570/20005 | Loss: 3.3144 | LR: 4.66e-04 | 2300.3ms/step | 10.4 samples/s
+Step 3580/20005 | Loss: 3.3298 | LR: 4.66e-04 | 2300.2ms/step | 10.4 samples/s
+Step 3590/20005 | Loss: 3.3344 | LR: 4.66e-04 | 2300.9ms/step | 10.4 samples/s
+Step 3600/20005 | Loss: 3.3482 | LR: 4.65e-04 | 2299.9ms/step | 10.4 samples/s
+Step 3610/20005 | Loss: 3.3315 | LR: 4.65e-04 | 2301.1ms/step | 10.4 samples/s
+Step 3620/20005 | Loss: 3.3413 | LR: 4.65e-04 | 2300.4ms/step | 10.4 samples/s
+Step 3630/20005 | Loss: 3.3199 | LR: 4.65e-04 | 2301.1ms/step | 10.4 samples/s
+Step 3640/20005 | Loss: 3.3406 | LR: 4.65e-04 | 2301.8ms/step | 10.4 samples/s
+Step 3650/20005 | Loss: 3.3107 | LR: 4.64e-04 | 2301.4ms/step | 10.4 samples/s
+Step 3660/20005 | Loss: 3.3099 | LR: 4.64e-04 | 2302.6ms/step | 10.4 samples/s
+Step 3670/20005 | Loss: 3.3660 | LR: 4.64e-04 | 2301.2ms/step | 10.4 samples/s
+Step 3680/20005 | Loss: 3.3217 | LR: 4.64e-04 | 2301.3ms/step | 10.4 samples/s
+Step 3690/20005 | Loss: 3.3204 | LR: 4.63e-04 | 2300.8ms/step | 10.4 samples/s
+Step 3700/20005 | Loss: 3.3263 | LR: 4.63e-04 | 2301.4ms/step | 10.4 samples/s
+Step 3710/20005 | Loss: 3.2963 | LR: 4.63e-04 | 2487.3ms/step | 9.6 samples/s
+Step 3720/20005 | Loss: 3.3083 | LR: 4.63e-04 | 2310.4ms/step | 10.4 samples/s
+Step 3730/20005 | Loss: 3.3284 | LR: 4.63e-04 | 2311.4ms/step | 10.4 samples/s
+Step 3740/20005 | Loss: 3.3077 | LR: 4.62e-04 | 2310.5ms/step | 10.4 samples/s
+Step 3750/20005 | Loss: 3.3083 | LR: 4.62e-04 | 2312.0ms/step | 10.4 samples/s
+Validation at step 3750: Loss = 3.3157, Perplexity = 27.54
+Step 3760/20005 | Loss: 3.3224 | LR: 4.62e-04 | 17399.8ms/step | 1.4 samples/s
+Step 3770/20005 | Loss: 3.3415 | LR: 4.62e-04 | 2309.7ms/step | 10.4 samples/s
+Step 3780/20005 | Loss: 3.3247 | LR: 4.62e-04 | 2309.3ms/step | 10.4 samples/s
+Step 3790/20005 | Loss: 3.3185 | LR: 4.61e-04 | 2310.9ms/step | 10.4 samples/s
+Step 3800/20005 | Loss: 3.2931 | LR: 4.61e-04 | 2309.3ms/step | 10.4 samples/s
+Step 3810/20005 | Loss: 3.3120 | LR: 4.61e-04 | 2307.3ms/step | 10.4 samples/s
+Step 3820/20005 | Loss: 3.3149 | LR: 4.61e-04 | 2310.4ms/step | 10.4 samples/s
+Step 3830/20005 | Loss: 3.2968 | LR: 4.61e-04 | 2311.2ms/step | 10.4 samples/s
+Step 3840/20005 | Loss: 3.2989 | LR: 4.60e-04 | 2307.5ms/step | 10.4 samples/s
+Step 3850/20005 | Loss: 3.3332 | LR: 4.60e-04 | 2310.0ms/step | 10.4 samples/s
+Step 3860/20005 | Loss: 3.3292 | LR: 4.60e-04 | 2309.5ms/step | 10.4 samples/s
+Step 3870/20005 | Loss: 3.2892 | LR: 4.60e-04 | 2308.4ms/step | 10.4 samples/s
+Step 3880/20005 | Loss: 3.3084 | LR: 4.59e-04 | 2309.1ms/step | 10.4 samples/s
+Step 3890/20005 | Loss: 3.3052 | LR: 4.59e-04 | 2309.6ms/step | 10.4 samples/s
+Step 3900/20005 | Loss: 3.2974 | LR: 4.59e-04 | 2309.6ms/step | 10.4 samples/s
+Step 3910/20005 | Loss: 3.2934 | LR: 4.59e-04 | 2310.1ms/step | 10.4 samples/s
+Step 3920/20005 | Loss: 3.2869 | LR: 4.59e-04 | 2310.2ms/step | 10.4 samples/s
+Step 3930/20005 | Loss: 3.2909 | LR: 4.58e-04 | 2309.9ms/step | 10.4 samples/s
+Step 3940/20005 | Loss: 3.3000 | LR: 4.58e-04 | 2309.2ms/step | 10.4 samples/s
+Step 3950/20005 | Loss: 3.3116 | LR: 4.58e-04 | 2309.5ms/step | 10.4 samples/s
+Step 3960/20005 | Loss: 3.2894 | LR: 4.58e-04 | 2308.4ms/step | 10.4 samples/s
+Step 3970/20005 | Loss: 3.3043 | LR: 4.57e-04 | 2310.3ms/step | 10.4 samples/s
+Step 3980/20005 | Loss: 3.3012 | LR: 4.57e-04 | 2310.0ms/step | 10.4 samples/s
+Step 3990/20005 | Loss: 3.2945 | LR: 4.57e-04 | 2310.0ms/step | 10.4 samples/s
+Step 4000/20005 | Loss: 3.2997 | LR: 4.57e-04 | 2311.1ms/step | 10.4 samples/s
+Validation at step 4000: Loss = 3.2948, Perplexity = 26.97
+Saved checkpoint to checkpoints/run_20260517_023855/checkpoint-4000
+Step 4010/20005 | Loss: 3.3060 | LR: 4.57e-04 | 17880.7ms/step | 1.3 samples/s
+Step 4020/20005 | Loss: 3.2992 | LR: 4.56e-04 | 2309.9ms/step | 10.4 samples/s
+Step 4030/20005 | Loss: 3.2725 | LR: 4.56e-04 | 2311.4ms/step | 10.4 samples/s
+Step 4040/20005 | Loss: 3.3014 | LR: 4.56e-04 | 2310.7ms/step | 10.4 samples/s
+Step 4050/20005 | Loss: 3.2515 | LR: 4.56e-04 | 2309.1ms/step | 10.4 samples/s
+Step 4060/20005 | Loss: 3.2812 | LR: 4.55e-04 | 2309.0ms/step | 10.4 samples/s
+Step 4070/20005 | Loss: 3.2827 | LR: 4.55e-04 | 2310.9ms/step | 10.4 samples/s
+Step 4080/20005 | Loss: 3.2731 | LR: 4.55e-04 | 2311.5ms/step | 10.4 samples/s
+Step 4090/20005 | Loss: 3.2971 | LR: 4.55e-04 | 2310.2ms/step | 10.4 samples/s
+Step 4100/20005 | Loss: 3.3017 | LR: 4.55e-04 | 2309.7ms/step | 10.4 samples/s
+Step 4110/20005 | Loss: 3.2878 | LR: 4.54e-04 | 2310.2ms/step | 10.4 samples/s
+Step 4120/20005 | Loss: 3.2913 | LR: 4.54e-04 | 2311.2ms/step | 10.4 samples/s
+Step 4130/20005 | Loss: 3.2752 | LR: 4.54e-04 | 2309.3ms/step | 10.4 samples/s
+Step 4140/20005 | Loss: 3.2851 | LR: 4.54e-04 | 2310.0ms/step | 10.4 samples/s
+Step 4150/20005 | Loss: 3.2679 | LR: 4.53e-04 | 2309.3ms/step | 10.4 samples/s
+Step 4160/20005 | Loss: 3.2765 | LR: 4.53e-04 | 2310.0ms/step | 10.4 samples/s
+Step 4170/20005 | Loss: 3.2707 | LR: 4.53e-04 | 2314.3ms/step | 10.4 samples/s
+Step 4180/20005 | Loss: 3.2619 | LR: 4.53e-04 | 2310.2ms/step | 10.4 samples/s
+Step 4190/20005 | Loss: 3.2897 | LR: 4.52e-04 | 2310.3ms/step | 10.4 samples/s
+Step 4200/20005 | Loss: 3.2867 | LR: 4.52e-04 | 2311.5ms/step | 10.4 samples/s
+Step 4210/20005 | Loss: 3.2853 | LR: 4.52e-04 | 2307.4ms/step | 10.4 samples/s
+Step 4220/20005 | Loss: 3.2685 | LR: 4.52e-04 | 2309.8ms/step | 10.4 samples/s
+Step 4230/20005 | Loss: 3.2637 | LR: 4.52e-04 | 2311.1ms/step | 10.4 samples/s
+Step 4240/20005 | Loss: 3.2898 | LR: 4.51e-04 | 2310.4ms/step | 10.4 samples/s
+Step 4250/20005 | Loss: 3.2819 | LR: 4.51e-04 | 2307.3ms/step | 10.4 samples/s
+Validation at step 4250: Loss = 3.2750, Perplexity = 26.44
+Step 4260/20005 | Loss: 3.2865 | LR: 4.51e-04 | 17398.1ms/step | 1.4 samples/s
+Step 4270/20005 | Loss: 3.2660 | LR: 4.51e-04 | 2311.6ms/step | 10.4 samples/s
+Step 4280/20005 | Loss: 3.2667 | LR: 4.50e-04 | 2308.6ms/step | 10.4 samples/s
+Step 4290/20005 | Loss: 3.2558 | LR: 4.50e-04 | 2313.2ms/step | 10.4 samples/s
+Step 4300/20005 | Loss: 3.2841 | LR: 4.50e-04 | 2310.4ms/step | 10.4 samples/s
+Step 4310/20005 | Loss: 3.2431 | LR: 4.50e-04 | 2313.0ms/step | 10.4 samples/s
+Step 4320/20005 | Loss: 3.2726 | LR: 4.49e-04 | 2310.3ms/step | 10.4 samples/s
+Step 4330/20005 | Loss: 3.2797 | LR: 4.49e-04 | 2311.5ms/step | 10.4 samples/s
+Step 4340/20005 | Loss: 3.2878 | LR: 4.49e-04 | 2311.1ms/step | 10.4 samples/s
+Step 4350/20005 | Loss: 3.2529 | LR: 4.49e-04 | 2310.0ms/step | 10.4 samples/s
+Step 4360/20005 | Loss: 3.2584 | LR: 4.48e-04 | 2311.0ms/step | 10.4 samples/s
+Step 4370/20005 | Loss: 3.3119 | LR: 4.48e-04 | 2311.7ms/step | 10.4 samples/s
+Step 4380/20005 | Loss: 3.2548 | LR: 4.48e-04 | 2311.3ms/step | 10.4 samples/s
+Step 4390/20005 | Loss: 3.2647 | LR: 4.48e-04 | 2310.6ms/step | 10.4 samples/s
+Step 4400/20005 | Loss: 3.2836 | LR: 4.47e-04 | 2310.8ms/step | 10.4 samples/s
+Step 4410/20005 | Loss: 3.2683 | LR: 4.47e-04 | 2311.6ms/step | 10.4 samples/s
+Step 4420/20005 | Loss: 3.2429 | LR: 4.47e-04 | 2308.5ms/step | 10.4 samples/s
+Step 4430/20005 | Loss: 3.2781 | LR: 4.47e-04 | 2311.9ms/step | 10.4 samples/s
+Step 4440/20005 | Loss: 3.2903 | LR: 4.46e-04 | 2311.0ms/step | 10.4 samples/s
+Step 4450/20005 | Loss: 3.2413 | LR: 4.46e-04 | 2311.3ms/step | 10.4 samples/s
+Step 4460/20005 | Loss: 3.2837 | LR: 4.46e-04 | 2309.3ms/step | 10.4 samples/s
+Step 4470/20005 | Loss: 3.2438 | LR: 4.46e-04 | 2310.5ms/step | 10.4 samples/s
+Step 4480/20005 | Loss: 3.2249 | LR: 4.46e-04 | 2311.7ms/step | 10.4 samples/s
+Step 4490/20005 | Loss: 3.2353 | LR: 4.45e-04 | 2312.0ms/step | 10.4 samples/s
+Step 4500/20005 | Loss: 3.2586 | LR: 4.45e-04 | 2307.8ms/step | 10.4 samples/s
+Validation at step 4500: Loss = 3.2564, Perplexity = 25.95
+Step 4510/20005 | Loss: 3.2608 | LR: 4.45e-04 | 17396.8ms/step | 1.4 samples/s
+Step 4520/20005 | Loss: 3.2450 | LR: 4.45e-04 | 2311.3ms/step | 10.4 samples/s
+Step 4530/20005 | Loss: 3.2396 | LR: 4.44e-04 | 2309.7ms/step | 10.4 samples/s
+Step 4540/20005 | Loss: 3.2508 | LR: 4.44e-04 | 2312.1ms/step | 10.4 samples/s
+Step 4550/20005 | Loss: 3.2650 | LR: 4.44e-04 | 2309.1ms/step | 10.4 samples/s
+Step 4560/20005 | Loss: 3.2451 | LR: 4.44e-04 | 2309.3ms/step | 10.4 samples/s
+Step 4570/20005 | Loss: 3.2474 | LR: 4.43e-04 | 2310.6ms/step | 10.4 samples/s
+Step 4580/20005 | Loss: 3.2540 | LR: 4.43e-04 | 2310.1ms/step | 10.4 samples/s
+Step 4590/20005 | Loss: 3.2663 | LR: 4.43e-04 | 2308.4ms/step | 10.4 samples/s
+Step 4600/20005 | Loss: 3.2718 | LR: 4.43e-04 | 2309.3ms/step | 10.4 samples/s
+Step 4610/20005 | Loss: 3.2498 | LR: 4.42e-04 | 2309.9ms/step | 10.4 samples/s
+Step 4620/20005 | Loss: 3.2728 | LR: 4.42e-04 | 2309.7ms/step | 10.4 samples/s
+Step 4630/20005 | Loss: 3.2559 | LR: 4.42e-04 | 2307.9ms/step | 10.4 samples/s
+Step 4640/20005 | Loss: 3.2258 | LR: 4.41e-04 | 2311.5ms/step | 10.4 samples/s
+Step 4650/20005 | Loss: 3.2357 | LR: 4.41e-04 | 2446.6ms/step | 9.8 samples/s
+Step 4660/20005 | Loss: 3.2495 | LR: 4.41e-04 | 2298.4ms/step | 10.4 samples/s
+Step 4670/20005 | Loss: 3.2349 | LR: 4.41e-04 | 2298.9ms/step | 10.4 samples/s
+Step 4680/20005 | Loss: 3.2487 | LR: 4.40e-04 | 2298.3ms/step | 10.4 samples/s
+Step 4690/20005 | Loss: 3.2556 | LR: 4.40e-04 | 2298.3ms/step | 10.4 samples/s
+Step 4700/20005 | Loss: 3.2504 | LR: 4.40e-04 | 2298.6ms/step | 10.4 samples/s
+Step 4710/20005 | Loss: 3.2082 | LR: 4.40e-04 | 2300.4ms/step | 10.4 samples/s
+Step 4720/20005 | Loss: 3.2445 | LR: 4.39e-04 | 2299.2ms/step | 10.4 samples/s
+Step 4730/20005 | Loss: 3.2601 | LR: 4.39e-04 | 2299.9ms/step | 10.4 samples/s
+Step 4740/20005 | Loss: 3.2301 | LR: 4.39e-04 | 2298.0ms/step | 10.4 samples/s
+Step 4750/20005 | Loss: 3.2471 | LR: 4.39e-04 | 2298.1ms/step | 10.4 samples/s
+Validation at step 4750: Loss = 3.2398, Perplexity = 25.53
+Step 4760/20005 | Loss: 3.2145 | LR: 4.38e-04 | 17316.5ms/step | 1.4 samples/s
+Step 4770/20005 | Loss: 3.2513 | LR: 4.38e-04 | 2299.2ms/step | 10.4 samples/s
+Step 4780/20005 | Loss: 3.2256 | LR: 4.38e-04 | 2298.1ms/step | 10.4 samples/s
+Step 4790/20005 | Loss: 3.2478 | LR: 4.38e-04 | 2298.8ms/step | 10.4 samples/s
+Step 4800/20005 | Loss: 3.2303 | LR: 4.37e-04 | 2299.6ms/step | 10.4 samples/s
+Step 4810/20005 | Loss: 3.2441 | LR: 4.37e-04 | 2299.5ms/step | 10.4 samples/s
+Step 4820/20005 | Loss: 3.2271 | LR: 4.37e-04 | 2298.3ms/step | 10.4 samples/s
+Step 4830/20005 | Loss: 3.2275 | LR: 4.37e-04 | 2297.8ms/step | 10.4 samples/s
+Step 4840/20005 | Loss: 3.2064 | LR: 4.36e-04 | 2300.4ms/step | 10.4 samples/s
+Step 4850/20005 | Loss: 3.2338 | LR: 4.36e-04 | 2297.6ms/step | 10.4 samples/s
+Step 4860/20005 | Loss: 3.2094 | LR: 4.36e-04 | 2299.8ms/step | 10.4 samples/s
+Step 4870/20005 | Loss: 3.2546 | LR: 4.35e-04 | 2299.8ms/step | 10.4 samples/s
+Step 4880/20005 | Loss: 3.2223 | LR: 4.35e-04 | 2298.7ms/step | 10.4 samples/s
+Step 4890/20005 | Loss: 3.2384 | LR: 4.35e-04 | 2299.7ms/step | 10.4 samples/s
+Step 4900/20005 | Loss: 3.2163 | LR: 4.35e-04 | 2300.7ms/step | 10.4 samples/s
+Step 4910/20005 | Loss: 3.2368 | LR: 4.34e-04 | 2299.9ms/step | 10.4 samples/s
+Step 4920/20005 | Loss: 3.2488 | LR: 4.34e-04 | 2301.9ms/step | 10.4 samples/s
+Step 4930/20005 | Loss: 3.2331 | LR: 4.34e-04 | 2298.5ms/step | 10.4 samples/s
+Step 4940/20005 | Loss: 3.2078 | LR: 4.34e-04 | 2298.5ms/step | 10.4 samples/s
+Step 4950/20005 | Loss: 3.2136 | LR: 4.33e-04 | 2301.1ms/step | 10.4 samples/s
+Step 4960/20005 | Loss: 3.2227 | LR: 4.33e-04 | 2298.7ms/step | 10.4 samples/s
+Step 4970/20005 | Loss: 3.2171 | LR: 4.33e-04 | 2299.1ms/step | 10.4 samples/s
+Step 4980/20005 | Loss: 3.1890 | LR: 4.33e-04 | 2299.5ms/step | 10.4 samples/s
+Step 4990/20005 | Loss: 3.2246 | LR: 4.32e-04 | 2299.3ms/step | 10.4 samples/s
+Step 5000/20005 | Loss: 3.2077 | LR: 4.32e-04 | 2300.0ms/step | 10.4 samples/s
+Validation at step 5000: Loss = 3.2249, Perplexity = 25.15
+Saved checkpoint to checkpoints/run_20260517_023855/checkpoint-5000
+Step 5010/20005 | Loss: 3.2205 | LR: 4.32e-04 | 17832.7ms/step | 1.3 samples/s
+Step 5020/20005 | Loss: 3.2215 | LR: 4.31e-04 | 2299.8ms/step | 10.4 samples/s
+Step 5030/20005 | Loss: 3.2351 | LR: 4.31e-04 | 2297.7ms/step | 10.4 samples/s
+Step 5040/20005 | Loss: 3.2114 | LR: 4.31e-04 | 2299.1ms/step | 10.4 samples/s
+Step 5050/20005 | Loss: 3.2217 | LR: 4.31e-04 | 2299.1ms/step | 10.4 samples/s
+Step 5060/20005 | Loss: 3.2202 | LR: 4.30e-04 | 2298.9ms/step | 10.4 samples/s
+Step 5070/20005 | Loss: 3.2287 | LR: 4.30e-04 | 2299.1ms/step | 10.4 samples/s
+Step 5080/20005 | Loss: 3.2043 | LR: 4.30e-04 | 2299.6ms/step | 10.4 samples/s
+Step 5090/20005 | Loss: 3.2417 | LR: 4.30e-04 | 2298.6ms/step | 10.4 samples/s
+Step 5100/20005 | Loss: 3.2167 | LR: 4.29e-04 | 2299.2ms/step | 10.4 samples/s
+Step 5110/20005 | Loss: 3.1993 | LR: 4.29e-04 | 2299.7ms/step | 10.4 samples/s
+Step 5120/20005 | Loss: 3.2129 | LR: 4.29e-04 | 2297.9ms/step | 10.4 samples/s
+Step 5130/20005 | Loss: 3.2258 | LR: 4.28e-04 | 2299.5ms/step | 10.4 samples/s
+Step 5140/20005 | Loss: 3.2265 | LR: 4.28e-04 | 2300.9ms/step | 10.4 samples/s
+Step 5150/20005 | Loss: 3.1969 | LR: 4.28e-04 | 2299.2ms/step | 10.4 samples/s
+Step 5160/20005 | Loss: 3.2422 | LR: 4.28e-04 | 2299.0ms/step | 10.4 samples/s
+Step 5170/20005 | Loss: 3.1901 | LR: 4.27e-04 | 2298.5ms/step | 10.4 samples/s
+Step 5180/20005 | Loss: 3.1797 | LR: 4.27e-04 | 2299.5ms/step | 10.4 samples/s
+Step 5190/20005 | Loss: 3.2050 | LR: 4.27e-04 | 2301.1ms/step | 10.4 samples/s
+Step 5200/20005 | Loss: 3.2266 | LR: 4.26e-04 | 2299.6ms/step | 10.4 samples/s
+Step 5210/20005 | Loss: 3.2000 | LR: 4.26e-04 | 2299.9ms/step | 10.4 samples/s
+Step 5220/20005 | Loss: 3.2097 | LR: 4.26e-04 | 2299.5ms/step | 10.4 samples/s
+Step 5230/20005 | Loss: 3.2039 | LR: 4.26e-04 | 2299.2ms/step | 10.4 samples/s
+Step 5240/20005 | Loss: 3.2095 | LR: 4.25e-04 | 2298.3ms/step | 10.4 samples/s
+Step 5250/20005 | Loss: 3.2310 | LR: 4.25e-04 | 2299.3ms/step | 10.4 samples/s
+Validation at step 5250: Loss = 3.2101, Perplexity = 24.78
+Step 5260/20005 | Loss: 3.2258 | LR: 4.25e-04 | 17310.0ms/step | 1.4 samples/s
+Step 5270/20005 | Loss: 3.2242 | LR: 4.24e-04 | 2299.1ms/step | 10.4 samples/s
+Step 5280/20005 | Loss: 3.2015 | LR: 4.24e-04 | 2301.1ms/step | 10.4 samples/s
+Step 5290/20005 | Loss: 3.2094 | LR: 4.24e-04 | 2299.6ms/step | 10.4 samples/s
+Step 5300/20005 | Loss: 3.1955 | LR: 4.24e-04 | 2299.4ms/step | 10.4 samples/s
+Step 5310/20005 | Loss: 3.2315 | LR: 4.23e-04 | 2297.9ms/step | 10.4 samples/s
+Step 5320/20005 | Loss: 3.1858 | LR: 4.23e-04 | 2298.1ms/step | 10.4 samples/s
+Step 5330/20005 | Loss: 3.1962 | LR: 4.23e-04 | 2299.9ms/step | 10.4 samples/s
+Step 5340/20005 | Loss: 3.1937 | LR: 4.22e-04 | 2298.3ms/step | 10.4 samples/s
+Step 5350/20005 | Loss: 3.2127 | LR: 4.22e-04 | 2300.7ms/step | 10.4 samples/s
+Step 5360/20005 | Loss: 3.2047 | LR: 4.22e-04 | 2301.9ms/step | 10.4 samples/s
+Step 5370/20005 | Loss: 3.2220 | LR: 4.22e-04 | 2298.1ms/step | 10.4 samples/s
+Step 5380/20005 | Loss: 3.2102 | LR: 4.21e-04 | 2299.9ms/step | 10.4 samples/s
+Step 5390/20005 | Loss: 3.1818 | LR: 4.21e-04 | 2296.4ms/step | 10.5 samples/s
+Step 5400/20005 | Loss: 3.2025 | LR: 4.21e-04 | 2298.5ms/step | 10.4 samples/s
+Step 5410/20005 | Loss: 3.1776 | LR: 4.20e-04 | 2300.0ms/step | 10.4 samples/s
+Step 5420/20005 | Loss: 3.2156 | LR: 4.20e-04 | 2297.9ms/step | 10.4 samples/s
+Step 5430/20005 | Loss: 3.2140 | LR: 4.20e-04 | 2299.6ms/step | 10.4 samples/s
+Step 5440/20005 | Loss: 3.1654 | LR: 4.20e-04 | 2298.8ms/step | 10.4 samples/s
+Step 5450/20005 | Loss: 3.2195 | LR: 4.19e-04 | 2301.8ms/step | 10.4 samples/s
+Step 5460/20005 | Loss: 3.1934 | LR: 4.19e-04 | 2299.4ms/step | 10.4 samples/s
+Step 5470/20005 | Loss: 3.1904 | LR: 4.19e-04 | 2301.0ms/step | 10.4 samples/s
+Step 5480/20005 | Loss: 3.1763 | LR: 4.18e-04 | 2300.1ms/step | 10.4 samples/s
+Step 5490/20005 | Loss: 3.1838 | LR: 4.18e-04 | 2297.5ms/step | 10.4 samples/s
+Step 5500/20005 | Loss: 3.1913 | LR: 4.18e-04 | 2300.6ms/step | 10.4 samples/s
+Validation at step 5500: Loss = 3.1960, Perplexity = 24.43
+Step 5510/20005 | Loss: 3.1761 | LR: 4.17e-04 | 17306.9ms/step | 1.4 samples/s
+Step 5520/20005 | Loss: 3.2122 | LR: 4.17e-04 | 2298.0ms/step | 10.4 samples/s
+Step 5530/20005 | Loss: 3.1874 | LR: 4.17e-04 | 2298.5ms/step | 10.4 samples/s
+Step 5540/20005 | Loss: 3.2022 | LR: 4.17e-04 | 2300.2ms/step | 10.4 samples/s
+Step 5550/20005 | Loss: 3.2076 | LR: 4.16e-04 | 2299.6ms/step | 10.4 samples/s
+Step 5560/20005 | Loss: 3.2150 | LR: 4.16e-04 | 2297.4ms/step | 10.4 samples/s
+Step 5570/20005 | Loss: 3.2023 | LR: 4.16e-04 | 2299.4ms/step | 10.4 samples/s
+Step 5580/20005 | Loss: 3.1716 | LR: 4.15e-04 | 2298.3ms/step | 10.4 samples/s
+Step 5590/20005 | Loss: 3.1930 | LR: 4.15e-04 | 2438.2ms/step | 9.8 samples/s
+Step 5600/20005 | Loss: 3.1883 | LR: 4.15e-04 | 2298.3ms/step | 10.4 samples/s
+Step 5610/20005 | Loss: 3.1919 | LR: 4.15e-04 | 2295.7ms/step | 10.5 samples/s
+Step 5620/20005 | Loss: 3.2078 | LR: 4.14e-04 | 2297.0ms/step | 10.4 samples/s
+Step 5630/20005 | Loss: 3.1685 | LR: 4.14e-04 | 2298.5ms/step | 10.4 samples/s
+Step 5640/20005 | Loss: 3.2065 | LR: 4.14e-04 | 2298.9ms/step | 10.4 samples/s
+Step 5650/20005 | Loss: 3.1772 | LR: 4.13e-04 | 2299.9ms/step | 10.4 samples/s
+Step 5660/20005 | Loss: 3.1739 | LR: 4.13e-04 | 2299.5ms/step | 10.4 samples/s
+Step 5670/20005 | Loss: 3.1828 | LR: 4.13e-04 | 2299.4ms/step | 10.4 samples/s
+Step 5680/20005 | Loss: 3.1828 | LR: 4.12e-04 | 2301.4ms/step | 10.4 samples/s
+Step 5690/20005 | Loss: 3.1874 | LR: 4.12e-04 | 2300.8ms/step | 10.4 samples/s
+Step 5700/20005 | Loss: 3.1741 | LR: 4.12e-04 | 2298.7ms/step | 10.4 samples/s
+Step 5710/20005 | Loss: 3.1700 | LR: 4.12e-04 | 2298.5ms/step | 10.4 samples/s
+Step 5720/20005 | Loss: 3.1818 | LR: 4.11e-04 | 2298.1ms/step | 10.4 samples/s
+Step 5730/20005 | Loss: 3.1372 | LR: 4.11e-04 | 2298.7ms/step | 10.4 samples/s
+Step 5740/20005 | Loss: 3.2102 | LR: 4.11e-04 | 2299.3ms/step | 10.4 samples/s
+Step 5750/20005 | Loss: 3.1773 | LR: 4.10e-04 | 2298.6ms/step | 10.4 samples/s
+Validation at step 5750: Loss = 3.1817, Perplexity = 24.09
+Step 5760/20005 | Loss: 3.1909 | LR: 4.10e-04 | 17315.2ms/step | 1.4 samples/s
+Step 5770/20005 | Loss: 3.2037 | LR: 4.10e-04 | 2302.0ms/step | 10.4 samples/s
+Step 5780/20005 | Loss: 3.1790 | LR: 4.09e-04 | 2299.0ms/step | 10.4 samples/s
+Step 5790/20005 | Loss: 3.1456 | LR: 4.09e-04 | 2298.2ms/step | 10.4 samples/s
+Step 5800/20005 | Loss: 3.1684 | LR: 4.09e-04 | 2299.6ms/step | 10.4 samples/s
+Step 5810/20005 | Loss: 3.1825 | LR: 4.08e-04 | 2297.7ms/step | 10.4 samples/s
+Step 5820/20005 | Loss: 3.1442 | LR: 4.08e-04 | 2299.3ms/step | 10.4 samples/s
+Step 5830/20005 | Loss: 3.1798 | LR: 4.08e-04 | 2298.0ms/step | 10.4 samples/s
+Step 5840/20005 | Loss: 3.2001 | LR: 4.08e-04 | 2298.0ms/step | 10.4 samples/s
+Step 5850/20005 | Loss: 3.1666 | LR: 4.07e-04 | 2298.5ms/step | 10.4 samples/s
+Step 5860/20005 | Loss: 3.1840 | LR: 4.07e-04 | 2299.8ms/step | 10.4 samples/s
+Step 5870/20005 | Loss: 3.1763 | LR: 4.07e-04 | 2299.1ms/step | 10.4 samples/s
+Step 5880/20005 | Loss: 3.1817 | LR: 4.06e-04 | 2298.6ms/step | 10.4 samples/s
+Step 5890/20005 | Loss: 3.1568 | LR: 4.06e-04 | 2298.4ms/step | 10.4 samples/s
+Step 5900/20005 | Loss: 3.1609 | LR: 4.06e-04 | 2299.9ms/step | 10.4 samples/s
+Step 5910/20005 | Loss: 3.1954 | LR: 4.05e-04 | 2300.1ms/step | 10.4 samples/s
+Step 5920/20005 | Loss: 3.1442 | LR: 4.05e-04 | 2299.3ms/step | 10.4 samples/s
+Step 5930/20005 | Loss: 3.1578 | LR: 4.05e-04 | 2301.3ms/step | 10.4 samples/s
+Step 5940/20005 | Loss: 3.1838 | LR: 4.04e-04 | 2299.9ms/step | 10.4 samples/s
+Step 5950/20005 | Loss: 3.1736 | LR: 4.04e-04 | 2299.7ms/step | 10.4 samples/s
+Step 5960/20005 | Loss: 3.1779 | LR: 4.04e-04 | 2298.3ms/step | 10.4 samples/s
+Step 5970/20005 | Loss: 3.1718 | LR: 4.03e-04 | 2299.0ms/step | 10.4 samples/s
+Step 5980/20005 | Loss: 3.1570 | LR: 4.03e-04 | 2299.9ms/step | 10.4 samples/s
+Step 5990/20005 | Loss: 3.1823 | LR: 4.03e-04 | 2298.9ms/step | 10.4 samples/s
+Step 6000/20005 | Loss: 3.1593 | LR: 4.03e-04 | 2298.8ms/step | 10.4 samples/s
+Validation at step 6000: Loss = 3.1698, Perplexity = 23.80
+Saved checkpoint to checkpoints/run_20260517_023855/checkpoint-6000
+Step 6010/20005 | Loss: 3.1827 | LR: 4.02e-04 | 17816.3ms/step | 1.3 samples/s
+Step 6020/20005 | Loss: 3.1507 | LR: 4.02e-04 | 2295.9ms/step | 10.5 samples/s
+Step 6030/20005 | Loss: 3.1578 | LR: 4.02e-04 | 2298.2ms/step | 10.4 samples/s
+Step 6040/20005 | Loss: 3.1645 | LR: 4.01e-04 | 2297.9ms/step | 10.4 samples/s
+Step 6050/20005 | Loss: 3.1627 | LR: 4.01e-04 | 2296.0ms/step | 10.5 samples/s
+Step 6060/20005 | Loss: 3.1576 | LR: 4.01e-04 | 2299.7ms/step | 10.4 samples/s
+Step 6070/20005 | Loss: 3.1536 | LR: 4.00e-04 | 2298.4ms/step | 10.4 samples/s
+Step 6080/20005 | Loss: 3.1656 | LR: 4.00e-04 | 2298.1ms/step | 10.4 samples/s
+Step 6090/20005 | Loss: 3.1583 | LR: 4.00e-04 | 2298.6ms/step | 10.4 samples/s
+Step 6100/20005 | Loss: 3.1606 | LR: 3.99e-04 | 2296.9ms/step | 10.4 samples/s
+Step 6110/20005 | Loss: 3.1783 | LR: 3.99e-04 | 2297.6ms/step | 10.4 samples/s
+Step 6120/20005 | Loss: 3.1647 | LR: 3.99e-04 | 2297.8ms/step | 10.4 samples/s
+Step 6130/20005 | Loss: 3.1640 | LR: 3.98e-04 | 2297.8ms/step | 10.4 samples/s
+Step 6140/20005 | Loss: 3.1640 | LR: 3.98e-04 | 2297.7ms/step | 10.4 samples/s
+Step 6150/20005 | Loss: 3.1779 | LR: 3.98e-04 | 2298.6ms/step | 10.4 samples/s
+Step 6160/20005 | Loss: 3.1703 | LR: 3.97e-04 | 2300.6ms/step | 10.4 samples/s
+Step 6170/20005 | Loss: 3.1528 | LR: 3.97e-04 | 2300.2ms/step | 10.4 samples/s
+Step 6180/20005 | Loss: 3.1592 | LR: 3.97e-04 | 2299.6ms/step | 10.4 samples/s
+Step 6190/20005 | Loss: 3.1413 | LR: 3.96e-04 | 2300.0ms/step | 10.4 samples/s
+Step 6200/20005 | Loss: 3.1226 | LR: 3.96e-04 | 2297.7ms/step | 10.4 samples/s
+Step 6210/20005 | Loss: 3.1727 | LR: 3.96e-04 | 2297.7ms/step | 10.4 samples/s
+Step 6220/20005 | Loss: 3.1433 | LR: 3.96e-04 | 2299.8ms/step | 10.4 samples/s
+Step 6230/20005 | Loss: 3.1535 | LR: 3.95e-04 | 2299.0ms/step | 10.4 samples/s
+Step 6240/20005 | Loss: 3.1380 | LR: 3.95e-04 | 2297.9ms/step | 10.4 samples/s
+Step 6250/20005 | Loss: 3.1582 | LR: 3.95e-04 | 2299.7ms/step | 10.4 samples/s
+Validation at step 6250: Loss = 3.1600, Perplexity = 23.57
+Step 6260/20005 | Loss: 3.1675 | LR: 3.94e-04 | 17381.9ms/step | 1.4 samples/s
+Step 6270/20005 | Loss: 3.1477 | LR: 3.94e-04 | 2307.7ms/step | 10.4 samples/s
+Step 6280/20005 | Loss: 3.1233 | LR: 3.94e-04 | 2307.6ms/step | 10.4 samples/s
+Step 6290/20005 | Loss: 3.1457 | LR: 3.93e-04 | 2310.5ms/step | 10.4 samples/s
+Step 6300/20005 | Loss: 3.1708 | LR: 3.93e-04 | 2309.0ms/step | 10.4 samples/s
+Step 6310/20005 | Loss: 3.1623 | LR: 3.93e-04 | 2310.1ms/step | 10.4 samples/s
+Step 6320/20005 | Loss: 3.1580 | LR: 3.92e-04 | 2309.4ms/step | 10.4 samples/s
+Step 6330/20005 | Loss: 3.1641 | LR: 3.92e-04 | 2310.1ms/step | 10.4 samples/s
+Step 6340/20005 | Loss: 3.1829 | LR: 3.92e-04 | 2310.0ms/step | 10.4 samples/s
+Step 6350/20005 | Loss: 3.1605 | LR: 3.91e-04 | 2309.3ms/step | 10.4 samples/s
+Step 6360/20005 | Loss: 3.1638 | LR: 3.91e-04 | 2310.6ms/step | 10.4 samples/s
+Step 6370/20005 | Loss: 3.1825 | LR: 3.91e-04 | 2307.1ms/step | 10.4 samples/s
+Step 6380/20005 | Loss: 3.1334 | LR: 3.90e-04 | 2308.3ms/step | 10.4 samples/s
+Step 6390/20005 | Loss: 3.1387 | LR: 3.90e-04 | 2308.7ms/step | 10.4 samples/s
+Step 6400/20005 | Loss: 3.1453 | LR: 3.90e-04 | 2309.8ms/step | 10.4 samples/s
+Step 6410/20005 | Loss: 3.1380 | LR: 3.89e-04 | 2310.8ms/step | 10.4 samples/s
+Step 6420/20005 | Loss: 3.1172 | LR: 3.89e-04 | 2309.1ms/step | 10.4 samples/s
+Step 6430/20005 | Loss: 3.1523 | LR: 3.89e-04 | 2309.2ms/step | 10.4 samples/s
+Step 6440/20005 | Loss: 3.1687 | LR: 3.88e-04 | 2308.9ms/step | 10.4 samples/s
+Step 6450/20005 | Loss: 3.1506 | LR: 3.88e-04 | 2307.1ms/step | 10.4 samples/s
+Step 6460/20005 | Loss: 3.1600 | LR: 3.88e-04 | 2309.8ms/step | 10.4 samples/s
+Step 6470/20005 | Loss: 3.1243 | LR: 3.87e-04 | 2308.2ms/step | 10.4 samples/s
+Step 6480/20005 | Loss: 3.1792 | LR: 3.87e-04 | 2308.3ms/step | 10.4 samples/s
+Step 6490/20005 | Loss: 3.1123 | LR: 3.87e-04 | 2306.4ms/step | 10.4 samples/s
+Step 6500/20005 | Loss: 3.1190 | LR: 3.86e-04 | 2307.4ms/step | 10.4 samples/s
+Validation at step 6500: Loss = 3.1479, Perplexity = 23.29
+Step 6510/20005 | Loss: 3.1297 | LR: 3.86e-04 | 17384.4ms/step | 1.4 samples/s
+Step 6520/20005 | Loss: 3.1216 | LR: 3.86e-04 | 2307.5ms/step | 10.4 samples/s
+Step 6530/20005 | Loss: 3.1484 | LR: 3.85e-04 | 2311.2ms/step | 10.4 samples/s
+Step 6540/20005 | Loss: 3.1367 | LR: 3.85e-04 | 2448.8ms/step | 9.8 samples/s
+Step 6550/20005 | Loss: 3.1073 | LR: 3.85e-04 | 2310.6ms/step | 10.4 samples/s
+Step 6560/20005 | Loss: 3.1410 | LR: 3.84e-04 | 2309.6ms/step | 10.4 samples/s
+Step 6570/20005 | Loss: 3.1222 | LR: 3.84e-04 | 2309.8ms/step | 10.4 samples/s
+Step 6580/20005 | Loss: 3.1125 | LR: 3.84e-04 | 2307.4ms/step | 10.4 samples/s
+Step 6590/20005 | Loss: 3.1591 | LR: 3.83e-04 | 2309.5ms/step | 10.4 samples/s
+Step 6600/20005 | Loss: 3.1448 | LR: 3.83e-04 | 2309.3ms/step | 10.4 samples/s
+Step 6610/20005 | Loss: 3.1424 | LR: 3.83e-04 | 2315.0ms/step | 10.4 samples/s
+Step 6620/20005 | Loss: 3.1445 | LR: 3.82e-04 | 2311.5ms/step | 10.4 samples/s
+Step 6630/20005 | Loss: 3.1631 | LR: 3.82e-04 | 2311.4ms/step | 10.4 samples/s
+Step 6640/20005 | Loss: 3.1128 | LR: 3.82e-04 | 2310.6ms/step | 10.4 samples/s
+Step 6650/20005 | Loss: 3.1287 | LR: 3.81e-04 | 2309.5ms/step | 10.4 samples/s
+Step 6660/20005 | Loss: 3.1049 | LR: 3.81e-04 | 2309.8ms/step | 10.4 samples/s
+Step 6670/20005 | Loss: 3.1483 | LR: 3.81e-04 | 2308.2ms/step | 10.4 samples/s
+Step 6680/20005 | Loss: 3.1597 | LR: 3.80e-04 | 2306.4ms/step | 10.4 samples/s
+Step 6690/20005 | Loss: 3.1623 | LR: 3.80e-04 | 2309.8ms/step | 10.4 samples/s
+Step 6700/20005 | Loss: 3.1066 | LR: 3.80e-04 | 2307.7ms/step | 10.4 samples/s
+Step 6710/20005 | Loss: 3.1409 | LR: 3.79e-04 | 2306.2ms/step | 10.4 samples/s
+Step 6720/20005 | Loss: 3.1201 | LR: 3.79e-04 | 2310.4ms/step | 10.4 samples/s
+Step 6730/20005 | Loss: 3.1068 | LR: 3.79e-04 | 2304.8ms/step | 10.4 samples/s
+Step 6740/20005 | Loss: 3.1531 | LR: 3.78e-04 | 2307.4ms/step | 10.4 samples/s
+Step 6750/20005 | Loss: 3.1253 | LR: 3.78e-04 | 2307.8ms/step | 10.4 samples/s
+Validation at step 6750: Loss = 3.1378, Perplexity = 23.05
+Step 6760/20005 | Loss: 3.1392 | LR: 3.78e-04 | 17374.3ms/step | 1.4 samples/s
+Step 6770/20005 | Loss: 3.1322 | LR: 3.77e-04 | 2308.2ms/step | 10.4 samples/s
+Step 6780/20005 | Loss: 3.1549 | LR: 3.77e-04 | 2307.2ms/step | 10.4 samples/s
+Step 6790/20005 | Loss: 3.1386 | LR: 3.77e-04 | 2307.5ms/step | 10.4 samples/s
+Step 6800/20005 | Loss: 3.1426 | LR: 3.76e-04 | 2309.3ms/step | 10.4 samples/s
+Step 6810/20005 | Loss: 3.1542 | LR: 3.76e-04 | 2309.5ms/step | 10.4 samples/s
+Step 6820/20005 | Loss: 3.1231 | LR: 3.75e-04 | 2307.5ms/step | 10.4 samples/s
+Step 6830/20005 | Loss: 3.1262 | LR: 3.75e-04 | 2309.7ms/step | 10.4 samples/s
+Step 6840/20005 | Loss: 3.1451 | LR: 3.75e-04 | 2308.5ms/step | 10.4 samples/s
+Step 6850/20005 | Loss: 3.1656 | LR: 3.74e-04 | 2308.2ms/step | 10.4 samples/s
+Step 6860/20005 | Loss: 3.1389 | LR: 3.74e-04 | 2308.5ms/step | 10.4 samples/s
+Step 6870/20005 | Loss: 3.1291 | LR: 3.74e-04 | 2306.9ms/step | 10.4 samples/s
+Step 6880/20005 | Loss: 3.1172 | LR: 3.73e-04 | 2308.1ms/step | 10.4 samples/s
+Step 6890/20005 | Loss: 3.1212 | LR: 3.73e-04 | 2307.3ms/step | 10.4 samples/s
+Step 6900/20005 | Loss: 3.1467 | LR: 3.73e-04 | 2307.8ms/step | 10.4 samples/s
+Step 6910/20005 | Loss: 3.1277 | LR: 3.72e-04 | 2309.5ms/step | 10.4 samples/s
+Step 6920/20005 | Loss: 3.1407 | LR: 3.72e-04 | 2309.1ms/step | 10.4 samples/s
+Step 6930/20005 | Loss: 3.1364 | LR: 3.72e-04 | 2306.6ms/step | 10.4 samples/s
+Step 6940/20005 | Loss: 3.1403 | LR: 3.71e-04 | 2308.1ms/step | 10.4 samples/s
+Step 6950/20005 | Loss: 3.1120 | LR: 3.71e-04 | 2306.7ms/step | 10.4 samples/s
+Step 6960/20005 | Loss: 3.1028 | LR: 3.71e-04 | 2307.2ms/step | 10.4 samples/s
+Step 6970/20005 | Loss: 3.1285 | LR: 3.70e-04 | 2309.0ms/step | 10.4 samples/s
+Step 6980/20005 | Loss: 3.1354 | LR: 3.70e-04 | 2307.6ms/step | 10.4 samples/s
+Step 6990/20005 | Loss: 3.1252 | LR: 3.70e-04 | 2307.1ms/step | 10.4 samples/s
+Step 7000/20005 | Loss: 3.1436 | LR: 3.69e-04 | 2307.7ms/step | 10.4 samples/s

epfl_final/wandb/run-20260517_042355-hi2z3jh6/files/requirements.txt

@@ -0,0 +1,97 @@
+pytz==2026.2
+nvidia-cusparselt-cu13==0.8.1
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.7.0
+tzdata==2026.2
+typing_extensions==4.15.0
+triton==3.7.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+safetensors==0.7.0
+regex==2026.5.9
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.5.2
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.29.7
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+numpy==2.2.6
+networkx==3.4.2
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.15
+hf-xet==1.5.0
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.4.0
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+async-timeout==5.0.1
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+typing-inspection==0.4.2
+sentry-sdk==2.60.0
+requests==2.34.2
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.4
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cublas==13.1.1.3
+multiprocess==0.70.19
+multidict==6.7.1
+markdown-it-py==4.2.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+exceptiongroup==1.3.1
+cuda-bindings==13.2.0
+aiosignal==1.4.0
+yarl==1.23.0
+rich==15.0.0
+pydantic==2.13.4
+pandas==2.3.3
+nvidia-cusolver==12.0.4.66
+nvidia-cudnn-cu13==9.20.0.48
+GitPython==3.1.50
+anyio==4.13.0
+wandb==0.27.0
+typer==0.25.1
+httpx==0.28.1
+aiohttp==3.13.5
+torch==2.12.0
+huggingface_hub==1.15.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.8.1
+PyGObject==3.42.1
+dbus-python==1.2.18
+distro==1.7.0
+pip==22.0.2
+setuptools==59.6.0
+ssh-import-id==5.11
+wheel==0.37.1

epfl_final/wandb/run-20260517_042355-hi2z3jh6/files/wandb-metadata.json

@@ -0,0 +1,36 @@
+{
+  "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.12",
+  "startedAt": "2026-05-17T04:23:55.183961Z",
+  "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath": "SparseAtt/train.py",
+  "codePathLocal": "SparseAtt/train.py",
+  "email": "ruben.navasardyan@mail.utoronto.ca",
+  "root": "/mloscratch/homes/navasard/sparse",
+  "host": "h100-train-14-0-4",
+  "executable": "/usr/bin/python3",
+  "cpu_count": 96,
+  "cpu_count_logical": 192,
+  "gpu": "NVIDIA H100 80GB HBM3",
+  "gpu_count": 1,
+  "disk": {
+    "/": {
+      "total": "9374931091456",
+      "used": "6127724351488"
+    }
+  },
+  "memory": {
+    "total": "1623005847552"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-c65885f7-52dc-6f23-a3dd-6addf8b518a2"
+    }
+  ],
+  "cudaVersion": "13.0",
+  "writerId": "z1jr3ni1v69qz52f2g6uucdpugs932rk"
+}

epfl_final/wandb/run-20260517_042355-hi2z3jh6/logs/debug.log

@@ -0,0 +1,20 @@
+2026-05-17 04:23:55,204 INFO    MainThread:8134 [wandb_setup.py:_flush():81] Current SDK version is 0.27.0
+2026-05-17 04:23:55,204 INFO    MainThread:8134 [wandb_setup.py:_flush():81] Configure stats pid to 8134
+2026-05-17 04:23:55,205 INFO    MainThread:8134 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-17 04:23:55,205 INFO    MainThread:8134 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260517_042355-hi2z3jh6/logs/debug.log
+2026-05-17 04:23:55,206 INFO    MainThread:8134 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260517_042355-hi2z3jh6/logs/debug-internal.log
+2026-05-17 04:23:55,206 INFO    MainThread:8134 [wandb_init.py:init():851] calling init triggers
+2026-05-17 04:23:55,207 INFO    MainThread:8134 [wandb_init.py:init():856] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': 'checkpoints/run_20260517_023855', 'resume_from_checkpoint': 'checkpoints/run_20260517_023855/checkpoint-1000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'sparse 30%', 'wandb_tags': ['qwen3', '0.6b'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 18, 'sparse_mask_pool_size': 2000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_19', 'dense_mode': False, '_wandb': {}}
+2026-05-17 04:23:55,208 INFO    MainThread:8134 [wandb_init.py:init():899] starting backend
+2026-05-17 04:23:55,445 INFO    MainThread:8134 [wandb_init.py:init():914] sending inform_init request
+2026-05-17 04:23:55,776 INFO    MainThread:8134 [wandb_init.py:init():919] backend started and connected
+2026-05-17 04:23:55,780 INFO    MainThread:8134 [wandb_init.py:init():989] updated telemetry
+2026-05-17 04:23:55,781 INFO    MainThread:8134 [wandb_init.py:init():1012] communicating run to backend with 90.0 second timeout
+2026-05-17 04:23:56,241 INFO    MainThread:8134 [wandb_init.py:init():1052] run resumed
+2026-05-17 04:23:56,243 INFO    MainThread:8134 [wandb_init.py:init():1057] starting run threads in backend
+2026-05-17 04:23:56,330 INFO    MainThread:8134 [wandb_run.py:_console_start():2509] atexit reg
+2026-05-17 04:23:56,330 INFO    MainThread:8134 [wandb_run.py:_redirect():2359] redirect: wrap_raw
+2026-05-17 04:23:56,330 INFO    MainThread:8134 [wandb_run.py:_redirect():2428] Wrapping output streams.
+2026-05-17 04:23:56,330 INFO    MainThread:8134 [wandb_run.py:_redirect():2451] Redirects installed.
+2026-05-17 04:23:56,333 INFO    MainThread:8134 [wandb_init.py:init():1095] run started, returning control to user process

epfl_final/wandb/run-20260517_042355-hi2z3jh6/run-hi2z3jh6.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae3e89bf03bd4a61c3c1712b51c22352c0e82748e6879193e2e8e81236e244ee
+size 2916352

epfl_final/wandb/run-20260517_162018-hi2z3jh6/files/requirements.txt

@@ -0,0 +1,97 @@
+pytz==2026.2
+nvidia-cusparselt-cu13==0.8.1
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.7.0
+tzdata==2026.2
+typing_extensions==4.15.0
+triton==3.7.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+safetensors==0.7.0
+regex==2026.5.9
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.5.2
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.29.7
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+numpy==2.2.6
+networkx==3.4.2
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.15
+hf-xet==1.5.0
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.4.0
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+async-timeout==5.0.1
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+typing-inspection==0.4.2
+sentry-sdk==2.60.0
+requests==2.34.2
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.4
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cublas==13.1.1.3
+multiprocess==0.70.19
+multidict==6.7.1
+markdown-it-py==4.2.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+exceptiongroup==1.3.1
+cuda-bindings==13.2.0
+aiosignal==1.4.0
+yarl==1.23.0
+rich==15.0.0
+pydantic==2.13.4
+pandas==2.3.3
+nvidia-cusolver==12.0.4.66
+nvidia-cudnn-cu13==9.20.0.48
+GitPython==3.1.50
+anyio==4.13.0
+wandb==0.27.0
+typer==0.25.1
+httpx==0.28.1
+aiohttp==3.13.5
+torch==2.12.0
+huggingface_hub==1.15.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.8.1
+PyGObject==3.42.1
+dbus-python==1.2.18
+distro==1.7.0
+pip==22.0.2
+setuptools==59.6.0
+ssh-import-id==5.11
+wheel==0.37.1

epfl_final/wandb/run-20260517_162018-hi2z3jh6/files/wandb-metadata.json

@@ -0,0 +1,36 @@
+{
+  "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.12",
+  "startedAt": "2026-05-17T16:20:18.743509Z",
+  "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath": "SparseAtt/train.py",
+  "codePathLocal": "SparseAtt/train.py",
+  "email": "ruben.navasardyan@mail.utoronto.ca",
+  "root": "/mloscratch/homes/navasard/sparse",
+  "host": "h100-node-0-2",
+  "executable": "/usr/bin/python3",
+  "cpu_count": 96,
+  "cpu_count_logical": 192,
+  "gpu": "NVIDIA H100 80GB HBM3",
+  "gpu_count": 1,
+  "disk": {
+    "/": {
+      "total": "9374931091456",
+      "used": "6497144651776"
+    }
+  },
+  "memory": {
+    "total": "1623005847552"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-3217a779-c453-e309-b765-27ba1809f528"
+    }
+  ],
+  "cudaVersion": "13.0",
+  "writerId": "uowzknc26qq4ecxclsrb8r7ghqptzgzq"
+}

epfl_final/wandb/run-20260517_162018-hi2z3jh6/logs/debug.log

@@ -0,0 +1,20 @@
+2026-05-17 16:20:18,759 INFO    MainThread:4559 [wandb_setup.py:_flush():81] Current SDK version is 0.27.0
+2026-05-17 16:20:18,759 INFO    MainThread:4559 [wandb_setup.py:_flush():81] Configure stats pid to 4559
+2026-05-17 16:20:18,759 INFO    MainThread:4559 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-17 16:20:18,760 INFO    MainThread:4559 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260517_162018-hi2z3jh6/logs/debug.log
+2026-05-17 16:20:18,760 INFO    MainThread:4559 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260517_162018-hi2z3jh6/logs/debug-internal.log
+2026-05-17 16:20:18,761 INFO    MainThread:4559 [wandb_init.py:init():851] calling init triggers
+2026-05-17 16:20:18,761 INFO    MainThread:4559 [wandb_init.py:init():856] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': 'checkpoints/run_20260517_023855', 'resume_from_checkpoint': 'checkpoints/run_20260517_023855/checkpoint-6000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'sparse 30%', 'wandb_tags': ['qwen3', '0.6b'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 18, 'sparse_mask_pool_size': 2000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_19', 'dense_mode': False, '_wandb': {}}
+2026-05-17 16:20:18,762 INFO    MainThread:4559 [wandb_init.py:init():899] starting backend
+2026-05-17 16:20:18,994 INFO    MainThread:4559 [wandb_init.py:init():914] sending inform_init request
+2026-05-17 16:20:19,316 INFO    MainThread:4559 [wandb_init.py:init():919] backend started and connected
+2026-05-17 16:20:19,320 INFO    MainThread:4559 [wandb_init.py:init():989] updated telemetry
+2026-05-17 16:20:19,321 INFO    MainThread:4559 [wandb_init.py:init():1012] communicating run to backend with 90.0 second timeout
+2026-05-17 16:20:19,821 INFO    MainThread:4559 [wandb_init.py:init():1052] run resumed
+2026-05-17 16:20:19,824 INFO    MainThread:4559 [wandb_init.py:init():1057] starting run threads in backend
+2026-05-17 16:20:19,910 INFO    MainThread:4559 [wandb_run.py:_console_start():2509] atexit reg
+2026-05-17 16:20:19,910 INFO    MainThread:4559 [wandb_run.py:_redirect():2359] redirect: wrap_raw
+2026-05-17 16:20:19,911 INFO    MainThread:4559 [wandb_run.py:_redirect():2428] Wrapping output streams.
+2026-05-17 16:20:19,911 INFO    MainThread:4559 [wandb_run.py:_redirect():2451] Redirects installed.
+2026-05-17 16:20:19,912 INFO    MainThread:4559 [wandb_init.py:init():1095] run started, returning control to user process

epfl_final/wandb/run-20260517_162018-hi2z3jh6/run-hi2z3jh6.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08eb93eb063fc95436986c3f7952ffed56b5f8e3d0162664f43687a5dabc1114
+size 6094848

epfl_final/wandb/run-20260518_025234-hi2z3jh6/files/output.log

@@ -0,0 +1,356 @@
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "checkpoints/run_20260517_023855",
+  "resume_from_checkpoint": "checkpoints/run_20260517_023855/checkpoint-18000",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 20005,
+  "per_device_batch_size": 3,
+  "gradient_accumulation_steps": 8,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 250,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 200,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "sparse 30%",
+  "wandb_tags": [
+    "qwen3",
+    "0.6b"
+  ],
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 18,
+  "sparse_mask_pool_size": 2000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_19",
+  "dense_mode": false
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: checkpoints/run_20260517_023855/checkpoint-18000
+Checkpoint activations: False
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1024
+  num_hidden_layers: 28
+  num_attention_heads: 16
+  num_key_value_heads: 8
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 18
+  sparse_mask_pool_size: 2000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 596,049,920
+  Trainable parameters: 596,049,920
+  Non-trainable parameters: 0
+  Model size (BF16): 1.19 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:2055: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [3, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [3, 8192, 1024]           --                        True
+│    └─Embedding: 2-1                                        [3, 8192]                 [3, 8192, 1024]           155,582,464               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └���Qwen3DecoderLayer: 3-7                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    └─Qwen3RMSNorm: 2-3                                     [3, 8192, 1024]           [3, 8192, 1024]           1,024                     True
+├─Linear: 1-2                                                [3, 8192, 1024]           [3, 8192, 151936]         155,582,464               True
+================================================================================================================================================================
+Total params: 751,632,384
+Trainable params: 751,632,384
+Non-trainable params: 0
+Total mult-adds (G): 2.25
+================================================================================================================================================================
+Input size (MB): 0.20
+Forward/backward pass size (MB): 63052.97
+Params size (MB): 1503.26
+Estimated Total Size (MB): 64556.43
+================================================================================================================================================================
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003)
+Dataset split: 1017602 train, 254401 validation
+Training examples (packed, train split):  1017602
+Validation examples (packed, val split):  254401
+Per-device batch size:                    3
+World size (GPUs):                        1
+Gradient accumulation steps:              8
+Effective batch size (seqs/step):         24
+Sequence length:                          8192
+Tokens per optimizer step:                196,608
+Micro-batches per epoch (per rank):       339200
+Optimizer steps per epoch:                42400
+Total optimizer steps:                    20005  (capped by max_steps; 0.47 epochs)
+Loading checkpoint from checkpoints/run_20260517_023855/checkpoint-18000...
+Resumed from checkpoint at step 17999 (epoch 0, micro_step 144016)
+Resuming training from step 17999
+Starting training...
+Step 18000/20005 | Loss: 0.2871 | LR: 1.26e-05 | 347868.6ms/step | 0.1 samples/s
+wandb: WARNING Tried to log to step 18000 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Validation at step 18000: Loss = 2.9449, Perplexity = 19.01
+Saved checkpoint to checkpoints/run_20260517_023855/checkpoint-18000
+wandb: WARNING Tried to log to step 18000 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18010/20005 | Loss: 2.9262 | LR: 1.24e-05 | 20258.4ms/step | 1.2 samples/s
+wandb: WARNING Tried to log to step 18010 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18020/20005 | Loss: 2.9380 | LR: 1.23e-05 | 2325.3ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18020 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18030/20005 | Loss: 2.9472 | LR: 1.22e-05 | 2322.8ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18030 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18040/20005 | Loss: 2.9313 | LR: 1.21e-05 | 2315.8ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18040 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18050/20005 | Loss: 2.9340 | LR: 1.19e-05 | 2315.6ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18050 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18060/20005 | Loss: 2.9068 | LR: 1.18e-05 | 2316.8ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18060 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18070/20005 | Loss: 2.8988 | LR: 1.17e-05 | 2315.9ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18070 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18080/20005 | Loss: 2.9077 | LR: 1.16e-05 | 2314.4ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18080 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18090/20005 | Loss: 2.9596 | LR: 1.15e-05 | 2314.3ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18090 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18100/20005 | Loss: 2.9119 | LR: 1.13e-05 | 2316.3ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18100 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18110/20005 | Loss: 2.9399 | LR: 1.12e-05 | 2315.4ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18110 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18120/20005 | Loss: 2.9164 | LR: 1.11e-05 | 2315.4ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18120 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18130/20005 | Loss: 2.9187 | LR: 1.10e-05 | 2321.0ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18130 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18140/20005 | Loss: 2.9256 | LR: 1.09e-05 | 2315.5ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18140 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18150/20005 | Loss: 2.9161 | LR: 1.08e-05 | 2318.5ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18150 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18160/20005 | Loss: 2.9449 | LR: 1.06e-05 | 2323.3ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18160 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18170/20005 | Loss: 2.9103 | LR: 1.05e-05 | 2322.9ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18170 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18180/20005 | Loss: 2.9492 | LR: 1.04e-05 | 2322.2ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18180 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18190/20005 | Loss: 2.9149 | LR: 1.03e-05 | 2318.4ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18190 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18200/20005 | Loss: 2.9357 | LR: 1.02e-05 | 2313.1ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18200 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18210/20005 | Loss: 2.9313 | LR: 1.01e-05 | 2313.5ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18210 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18220/20005 | Loss: 2.9328 | LR: 9.97e-06 | 2310.1ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18220 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18230/20005 | Loss: 2.9211 | LR: 9.86e-06 | 2314.0ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18230 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18240/20005 | Loss: 2.9613 | LR: 9.75e-06 | 2312.3ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18240 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18250/20005 | Loss: 2.9196 | LR: 9.64e-06 | 2310.7ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18250 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Validation at step 18250: Loss = 2.9449, Perplexity = 19.01
+wandb: WARNING Tried to log to step 18250 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18260/20005 | Loss: 2.9467 | LR: 9.53e-06 | 17425.7ms/step | 1.4 samples/s
+wandb: WARNING Tried to log to step 18260 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18270/20005 | Loss: 2.9209 | LR: 9.42e-06 | 2315.3ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18270 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18280/20005 | Loss: 2.9144 | LR: 9.32e-06 | 2314.6ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18280 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18290/20005 | Loss: 2.9284 | LR: 9.21e-06 | 2312.5ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18290 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18300/20005 | Loss: 2.9517 | LR: 9.10e-06 | 2314.0ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18300 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18310/20005 | Loss: 2.9304 | LR: 9.00e-06 | 2315.0ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18310 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18320/20005 | Loss: 2.9358 | LR: 8.89e-06 | 2315.1ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18320 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18330/20005 | Loss: 2.9497 | LR: 8.79e-06 | 2314.7ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18330 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18340/20005 | Loss: 2.9259 | LR: 8.68e-06 | 2313.7ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18340 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18350/20005 | Loss: 2.9214 | LR: 8.58e-06 | 2314.9ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18350 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18360/20005 | Loss: 2.9279 | LR: 8.47e-06 | 2315.6ms/step | 10.4 samples/s
+wandb: WARNING Tried to log to step 18360 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18370/20005 | Loss: 2.9385 | LR: 8.37e-06 | 2482.2ms/step | 9.7 samples/s
+wandb: WARNING Tried to log to step 18370 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18380/20005 | Loss: 2.9436 | LR: 8.27e-06 | 2321.8ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18380 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18390/20005 | Loss: 2.9214 | LR: 8.17e-06 | 2324.1ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18390 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18400/20005 | Loss: 2.9564 | LR: 8.07e-06 | 2323.5ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18400 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18410/20005 | Loss: 2.9666 | LR: 7.97e-06 | 2326.4ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18410 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18420/20005 | Loss: 2.9280 | LR: 7.87e-06 | 2322.6ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18420 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18430/20005 | Loss: 2.9215 | LR: 7.77e-06 | 2324.1ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18430 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18440/20005 | Loss: 2.9387 | LR: 7.67e-06 | 2321.5ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18440 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18450/20005 | Loss: 2.9111 | LR: 7.58e-06 | 2322.5ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18450 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18460/20005 | Loss: 2.9352 | LR: 7.48e-06 | 2323.8ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18460 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18470/20005 | Loss: 2.9359 | LR: 7.38e-06 | 2322.9ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18470 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18480/20005 | Loss: 2.9074 | LR: 7.29e-06 | 2323.3ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18480 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18490/20005 | Loss: 2.9367 | LR: 7.19e-06 | 2324.8ms/step | 10.3 samples/s
+wandb: WARNING Tried to log to step 18490 that is less than the current step 18491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 18500/20005 | Loss: 2.9431 | LR: 7.10e-06 | 2324.3ms/step | 10.3 samples/s
+Validation at step 18500: Loss = 2.9448, Perplexity = 19.01
+Step 18510/20005 | Loss: 2.9130 | LR: 7.00e-06 | 17470.7ms/step | 1.4 samples/s
+Step 18520/20005 | Loss: 2.9038 | LR: 6.91e-06 | 2322.2ms/step | 10.3 samples/s
+Step 18530/20005 | Loss: 2.9199 | LR: 6.82e-06 | 2322.1ms/step | 10.3 samples/s
+Step 18540/20005 | Loss: 2.9125 | LR: 6.73e-06 | 2323.1ms/step | 10.3 samples/s
+Step 18550/20005 | Loss: 2.9177 | LR: 6.64e-06 | 2323.3ms/step | 10.3 samples/s
+Step 18560/20005 | Loss: 2.9444 | LR: 6.54e-06 | 2324.7ms/step | 10.3 samples/s
+Step 18570/20005 | Loss: 2.9868 | LR: 6.45e-06 | 2322.0ms/step | 10.3 samples/s
+Step 18580/20005 | Loss: 2.9507 | LR: 6.37e-06 | 2325.7ms/step | 10.3 samples/s
+Step 18590/20005 | Loss: 2.9149 | LR: 6.28e-06 | 2322.9ms/step | 10.3 samples/s
+Step 18600/20005 | Loss: 2.9023 | LR: 6.19e-06 | 2322.2ms/step | 10.3 samples/s
+Step 18610/20005 | Loss: 2.9032 | LR: 6.10e-06 | 2323.1ms/step | 10.3 samples/s
+Step 18620/20005 | Loss: 2.9311 | LR: 6.01e-06 | 2324.1ms/step | 10.3 samples/s
+Step 18630/20005 | Loss: 2.9542 | LR: 5.93e-06 | 2321.4ms/step | 10.3 samples/s
+Step 18640/20005 | Loss: 2.9366 | LR: 5.84e-06 | 2323.6ms/step | 10.3 samples/s
+Step 18650/20005 | Loss: 2.9361 | LR: 5.76e-06 | 2321.7ms/step | 10.3 samples/s
+Step 18660/20005 | Loss: 2.9005 | LR: 5.67e-06 | 2323.5ms/step | 10.3 samples/s
+Step 18670/20005 | Loss: 2.9555 | LR: 5.59e-06 | 2323.3ms/step | 10.3 samples/s
+Step 18680/20005 | Loss: 2.9134 | LR: 5.50e-06 | 2321.8ms/step | 10.3 samples/s
+Step 18690/20005 | Loss: 2.9358 | LR: 5.42e-06 | 2322.4ms/step | 10.3 samples/s
+Step 18700/20005 | Loss: 2.9642 | LR: 5.34e-06 | 2324.8ms/step | 10.3 samples/s
+Step 18710/20005 | Loss: 2.9552 | LR: 5.26e-06 | 2324.6ms/step | 10.3 samples/s
+Step 18720/20005 | Loss: 2.9366 | LR: 5.18e-06 | 2321.9ms/step | 10.3 samples/s
+Step 18730/20005 | Loss: 2.9471 | LR: 5.10e-06 | 2324.7ms/step | 10.3 samples/s
+Step 18740/20005 | Loss: 2.9533 | LR: 5.02e-06 | 2322.0ms/step | 10.3 samples/s
+Step 18750/20005 | Loss: 2.9640 | LR: 4.94e-06 | 2324.2ms/step | 10.3 samples/s
+Validation at step 18750: Loss = 2.9448, Perplexity = 19.01
+Step 18760/20005 | Loss: 2.9110 | LR: 4.86e-06 | 17455.1ms/step | 1.4 samples/s
+Step 18770/20005 | Loss: 2.9430 | LR: 4.78e-06 | 2324.1ms/step | 10.3 samples/s
+Step 18780/20005 | Loss: 2.9269 | LR: 4.71e-06 | 2324.1ms/step | 10.3 samples/s
+Step 18790/20005 | Loss: 2.9330 | LR: 4.63e-06 | 2322.9ms/step | 10.3 samples/s
+Step 18800/20005 | Loss: 2.9265 | LR: 4.55e-06 | 2324.7ms/step | 10.3 samples/s
+Step 18810/20005 | Loss: 2.9540 | LR: 4.48e-06 | 2322.3ms/step | 10.3 samples/s
+Step 18820/20005 | Loss: 2.9309 | LR: 4.40e-06 | 2323.8ms/step | 10.3 samples/s
+Step 18830/20005 | Loss: 2.9221 | LR: 4.33e-06 | 2322.5ms/step | 10.3 samples/s
+Step 18840/20005 | Loss: 2.9307 | LR: 4.26e-06 | 2324.1ms/step | 10.3 samples/s
+Step 18850/20005 | Loss: 2.9446 | LR: 4.18e-06 | 2324.1ms/step | 10.3 samples/s
+Step 18860/20005 | Loss: 2.9341 | LR: 4.11e-06 | 2325.6ms/step | 10.3 samples/s
+Step 18870/20005 | Loss: 2.9326 | LR: 4.04e-06 | 2320.8ms/step | 10.3 samples/s
+Step 18880/20005 | Loss: 2.9186 | LR: 3.97e-06 | 2324.9ms/step | 10.3 samples/s
+Step 18890/20005 | Loss: 2.8829 | LR: 3.90e-06 | 2321.1ms/step | 10.3 samples/s
+Step 18900/20005 | Loss: 2.9331 | LR: 3.83e-06 | 2323.1ms/step | 10.3 samples/s
+Step 18910/20005 | Loss: 2.9285 | LR: 3.76e-06 | 2322.2ms/step | 10.3 samples/s
+Step 18920/20005 | Loss: 2.9191 | LR: 3.69e-06 | 2324.2ms/step | 10.3 samples/s
+Step 18930/20005 | Loss: 2.9445 | LR: 3.62e-06 | 2322.4ms/step | 10.3 samples/s
+Step 18940/20005 | Loss: 2.9198 | LR: 3.56e-06 | 2322.4ms/step | 10.3 samples/s
+Step 18950/20005 | Loss: 2.9121 | LR: 3.49e-06 | 2322.2ms/step | 10.3 samples/s
+Step 18960/20005 | Loss: 2.9489 | LR: 3.42e-06 | 2322.4ms/step | 10.3 samples/s
+Step 18970/20005 | Loss: 2.9495 | LR: 3.36e-06 | 2323.5ms/step | 10.3 samples/s
+Step 18980/20005 | Loss: 2.9574 | LR: 3.29e-06 | 2324.0ms/step | 10.3 samples/s
+Step 18990/20005 | Loss: 2.9411 | LR: 3.23e-06 | 2323.4ms/step | 10.3 samples/s
+Step 19000/20005 | Loss: 2.9165 | LR: 3.17e-06 | 2323.8ms/step | 10.3 samples/s
+Validation at step 19000: Loss = 2.9448, Perplexity = 19.01
+Saved checkpoint to checkpoints/run_20260517_023855/checkpoint-19000
+Step 19010/20005 | Loss: 2.9485 | LR: 3.10e-06 | 17957.6ms/step | 1.3 samples/s
+Step 19020/20005 | Loss: 2.9212 | LR: 3.04e-06 | 2320.5ms/step | 10.3 samples/s
+Step 19030/20005 | Loss: 2.9169 | LR: 2.98e-06 | 2321.8ms/step | 10.3 samples/s
+Step 19040/20005 | Loss: 2.9269 | LR: 2.92e-06 | 2319.3ms/step | 10.3 samples/s
+Step 19050/20005 | Loss: 2.9127 | LR: 2.86e-06 | 2323.1ms/step | 10.3 samples/s
+Step 19060/20005 | Loss: 2.9314 | LR: 2.80e-06 | 2320.0ms/step | 10.3 samples/s
+Step 19070/20005 | Loss: 2.9063 | LR: 2.74e-06 | 2321.0ms/step | 10.3 samples/s
+Step 19080/20005 | Loss: 2.9503 | LR: 2.68e-06 | 2322.3ms/step | 10.3 samples/s
+Step 19090/20005 | Loss: 2.9136 | LR: 2.62e-06 | 2322.7ms/step | 10.3 samples/s
+Step 19100/20005 | Loss: 2.9468 | LR: 2.57e-06 | 2320.4ms/step | 10.3 samples/s
+Step 19110/20005 | Loss: 2.9347 | LR: 2.51e-06 | 2321.7ms/step | 10.3 samples/s
+Step 19120/20005 | Loss: 2.9539 | LR: 2.46e-06 | 2323.6ms/step | 10.3 samples/s
+Step 19130/20005 | Loss: 2.9157 | LR: 2.40e-06 | 2323.4ms/step | 10.3 samples/s
+Step 19140/20005 | Loss: 2.9245 | LR: 2.35e-06 | 2322.1ms/step | 10.3 samples/s
+Step 19150/20005 | Loss: 2.9305 | LR: 2.29e-06 | 2322.3ms/step | 10.3 samples/s
+Step 19160/20005 | Loss: 2.9189 | LR: 2.24e-06 | 2322.3ms/step | 10.3 samples/s
+Step 19170/20005 | Loss: 2.9229 | LR: 2.19e-06 | 2320.1ms/step | 10.3 samples/s
+Step 19180/20005 | Loss: 2.9198 | LR: 2.13e-06 | 2321.5ms/step | 10.3 samples/s
+Step 19190/20005 | Loss: 2.9404 | LR: 2.08e-06 | 2321.7ms/step | 10.3 samples/s
+Step 19200/20005 | Loss: 2.9275 | LR: 2.03e-06 | 2320.3ms/step | 10.3 samples/s
+Step 19210/20005 | Loss: 2.8980 | LR: 1.98e-06 | 2321.6ms/step | 10.3 samples/s
+Step 19220/20005 | Loss: 2.9544 | LR: 1.93e-06 | 2321.7ms/step | 10.3 samples/s
+Step 19230/20005 | Loss: 2.9006 | LR: 1.88e-06 | 2321.7ms/step | 10.3 samples/s
+Step 19240/20005 | Loss: 2.9618 | LR: 1.83e-06 | 2322.7ms/step | 10.3 samples/s
+Step 19250/20005 | Loss: 2.9312 | LR: 1.79e-06 | 2322.5ms/step | 10.3 samples/s
+Validation at step 19250: Loss = 2.9449, Perplexity = 19.01
+Step 19260/20005 | Loss: 2.9488 | LR: 1.74e-06 | 17457.2ms/step | 1.4 samples/s
+Step 19270/20005 | Loss: 2.9053 | LR: 1.69e-06 | 2323.7ms/step | 10.3 samples/s
+Step 19280/20005 | Loss: 2.9239 | LR: 1.65e-06 | 2322.0ms/step | 10.3 samples/s
+Step 19290/20005 | Loss: 2.9398 | LR: 1.60e-06 | 2324.0ms/step | 10.3 samples/s
+Step 19300/20005 | Loss: 2.9380 | LR: 1.56e-06 | 2320.1ms/step | 10.3 samples/s
+Step 19310/20005 | Loss: 2.9300 | LR: 1.51e-06 | 2324.3ms/step | 10.3 samples/s
+Step 19320/20005 | Loss: 2.9248 | LR: 1.47e-06 | 2323.7ms/step | 10.3 samples/s
+Step 19330/20005 | Loss: 2.9279 | LR: 1.43e-06 | 2323.4ms/step | 10.3 samples/s
+Step 19340/20005 | Loss: 2.9459 | LR: 1.38e-06 | 2325.4ms/step | 10.3 samples/s
+Step 19350/20005 | Loss: 2.9041 | LR: 1.34e-06 | 2322.1ms/step | 10.3 samples/s
+Step 19360/20005 | Loss: 2.9376 | LR: 1.30e-06 | 2322.6ms/step | 10.3 samples/s
+Step 19370/20005 | Loss: 2.9294 | LR: 1.26e-06 | 2322.7ms/step | 10.3 samples/s

epfl_final/wandb/run-20260518_025234-hi2z3jh6/files/requirements.txt

@@ -0,0 +1,97 @@
+pytz==2026.2
+nvidia-cusparselt-cu13==0.8.1
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.7.0
+tzdata==2026.2
+typing_extensions==4.15.0
+triton==3.7.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+safetensors==0.7.0
+regex==2026.5.9
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.5.2
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.29.7
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+numpy==2.2.6
+networkx==3.4.2
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.15
+hf-xet==1.5.0
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.4.0
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+async-timeout==5.0.1
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+typing-inspection==0.4.2
+sentry-sdk==2.60.0
+requests==2.34.2
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.4
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cublas==13.1.1.3
+multiprocess==0.70.19
+multidict==6.7.1
+markdown-it-py==4.2.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+exceptiongroup==1.3.1
+cuda-bindings==13.2.0
+aiosignal==1.4.0
+yarl==1.23.0
+rich==15.0.0
+pydantic==2.13.4
+pandas==2.3.3
+nvidia-cusolver==12.0.4.66
+nvidia-cudnn-cu13==9.20.0.48
+GitPython==3.1.50
+anyio==4.13.0
+wandb==0.27.0
+typer==0.25.1
+httpx==0.28.1
+aiohttp==3.13.5
+torch==2.12.0
+huggingface_hub==1.15.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.8.1
+PyGObject==3.42.1
+dbus-python==1.2.18
+distro==1.7.0
+pip==22.0.2
+setuptools==59.6.0
+ssh-import-id==5.11
+wheel==0.37.1

epfl_final/wandb/run-20260518_025234-hi2z3jh6/files/wandb-metadata.json

@@ -0,0 +1,36 @@
+{
+  "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.12",
+  "startedAt": "2026-05-18T02:52:34.011641Z",
+  "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath": "SparseAtt/train.py",
+  "codePathLocal": "SparseAtt/train.py",
+  "email": "ruben.navasardyan@mail.utoronto.ca",
+  "root": "/mloscratch/homes/navasard/sparse",
+  "host": "h100-node-0-3",
+  "executable": "/usr/bin/python3",
+  "cpu_count": 96,
+  "cpu_count_logical": 192,
+  "gpu": "NVIDIA H100 80GB HBM3",
+  "gpu_count": 1,
+  "disk": {
+    "/": {
+      "total": "9374931091456",
+      "used": "6202813689856"
+    }
+  },
+  "memory": {
+    "total": "1623005847552"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-a2cf418a-a552-c05b-f5dc-8b4f120a914a"
+    }
+  ],
+  "cudaVersion": "13.0",
+  "writerId": "4fcrvxan6jtu2r8cp4mqerxxd5ogy7qa"
+}

epfl_final/wandb/run-20260518_025234-hi2z3jh6/logs/debug-internal.log

@@ -0,0 +1,665 @@
+{"time":"2026-05-18T02:52:34.262168557Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-05-18T02:52:34.276463829Z","level":"INFO","msg":"stream: starting","core version":"0.27.0"}
+{"time":"2026-05-18T02:52:34.59872679Z","level":"INFO","msg":"stream: created new stream","id":"hi2z3jh6"}
+{"time":"2026-05-18T02:52:34.598877575Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-05-18T02:52:34.601216515Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-05-18T02:52:34.601272929Z","level":"INFO","msg":"writer: started","stream_id":"hi2z3jh6"}
+{"time":"2026-05-18T02:52:34.601290115Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-05-18T02:52:35.045330219Z","level":"INFO","msg":"filestream: sending request","total_files":1}
+{"time":"2026-05-18T02:52:35.315997455Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:52:50.046108292Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":49,"uploaded_len":2}
+{"time":"2026-05-18T02:52:50.256000476Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:53:05.045960505Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":0,"events_lines":2}
+{"time":"2026-05-18T02:53:05.261191887Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:53:20.045661623Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":2,"events_lines":2,"console_offset":49,"console_lines":106}
+{"time":"2026-05-18T02:53:20.274968351Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:53:35.046152719Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":4,"events_lines":2,"console_offset":155,"console_lines":3}
+{"time":"2026-05-18T02:53:35.26754153Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:53:50.046139732Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":6,"events_lines":2}
+{"time":"2026-05-18T02:53:50.262806847Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:54:05.046291362Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":8,"events_lines":2}
+{"time":"2026-05-18T02:54:05.2653664Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:54:20.045841965Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":10,"events_lines":2}
+{"time":"2026-05-18T02:54:20.2465631Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:54:35.04608486Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":12,"events_lines":2}
+{"time":"2026-05-18T02:54:35.258126822Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:54:50.046150782Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":14,"events_lines":2}
+{"time":"2026-05-18T02:54:50.247537701Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:55:05.045567815Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":16,"events_lines":2}
+{"time":"2026-05-18T02:55:05.963358773Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:55:20.046260712Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":18,"events_lines":2}
+{"time":"2026-05-18T02:55:20.281103594Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:55:35.045947278Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":20,"events_lines":2}
+{"time":"2026-05-18T02:55:35.908925628Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:55:50.046120349Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":22,"events_lines":2}
+{"time":"2026-05-18T02:55:50.250383141Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:56:05.045342532Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":24,"events_lines":2}
+{"time":"2026-05-18T02:56:05.427034885Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:56:20.045946967Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":26,"events_lines":2}
+{"time":"2026-05-18T02:56:20.290061226Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:56:35.046263983Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":28,"events_lines":2}
+{"time":"2026-05-18T02:56:35.632680697Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:56:50.046024459Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":30,"events_lines":2}
+{"time":"2026-05-18T02:56:50.541601355Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:57:05.046215017Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":32,"events_lines":2}
+{"time":"2026-05-18T02:57:05.237095346Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:57:20.045854488Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":34,"events_lines":2}
+{"time":"2026-05-18T02:57:20.267595706Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:57:35.045437244Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":36,"events_lines":2}
+{"time":"2026-05-18T02:57:35.804440235Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:57:50.046096198Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":38,"events_lines":2}
+{"time":"2026-05-18T02:57:50.265504951Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:58:05.045848847Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":40,"events_lines":2}
+{"time":"2026-05-18T02:58:05.236267696Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:58:20.046167273Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":42,"events_lines":2}
+{"time":"2026-05-18T02:58:20.237691943Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:58:35.045277472Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":44,"events_lines":2}
+{"time":"2026-05-18T02:58:35.242957119Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:58:50.0457916Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":46,"events_lines":2}
+{"time":"2026-05-18T02:58:50.296044243Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:59:05.045457591Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":48,"events_lines":2}
+{"time":"2026-05-18T02:59:05.245510609Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:59:14.657878947Z","level":"WARN","msg":"handler: ignoring partial history record","step":18000,"current":18491}
+{"time":"2026-05-18T02:59:20.046031617Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":50,"events_lines":2,"console_offset":158,"console_lines":2}
+{"time":"2026-05-18T02:59:20.319719533Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:59:35.045305201Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":52,"events_lines":2}
+{"time":"2026-05-18T02:59:35.287283295Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T02:59:50.045242787Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":54,"events_lines":2}
+{"time":"2026-05-18T02:59:50.254661281Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:00:05.046077191Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":56,"events_lines":2}
+{"time":"2026-05-18T03:00:05.245518406Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:00:20.045883181Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":58,"events_lines":2}
+{"time":"2026-05-18T03:00:21.903550917Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:00:35.045683782Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":60,"events_lines":2}
+{"time":"2026-05-18T03:00:35.263568Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:00:50.045391405Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":62,"events_lines":2}
+{"time":"2026-05-18T03:00:51.057625054Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:01:05.045859222Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":64,"events_lines":2}
+{"time":"2026-05-18T03:01:06.162233555Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:01:20.045719743Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":66,"events_lines":2}
+{"time":"2026-05-18T03:01:20.286967917Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:01:35.045692461Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":68,"events_lines":2}
+{"time":"2026-05-18T03:01:35.268670092Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:01:50.046085622Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":70,"events_lines":2}
+{"time":"2026-05-18T03:01:50.264276237Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:02:05.045750957Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":72,"events_lines":2}
+{"time":"2026-05-18T03:02:05.365911746Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:02:08.944844583Z","level":"WARN","msg":"handler: ignoring partial history record","step":18000,"current":18491}
+{"time":"2026-05-18T03:02:20.0453817Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":74,"events_lines":2,"console_offset":160,"console_lines":3}
+{"time":"2026-05-18T03:02:20.252193797Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:02:35.045477979Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":76,"events_lines":2}
+{"time":"2026-05-18T03:02:36.039439168Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:02:37.24257326Z","level":"WARN","msg":"handler: ignoring partial history record","step":18010,"current":18491}
+{"time":"2026-05-18T03:02:50.045883879Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":78,"events_lines":2,"console_offset":163,"console_lines":2}
+{"time":"2026-05-18T03:02:50.239897689Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:03:00.495385653Z","level":"WARN","msg":"handler: ignoring partial history record","step":18020,"current":18491}
+{"time":"2026-05-18T03:03:05.045382515Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":80,"events_lines":2,"console_offset":165,"console_lines":1}
+{"time":"2026-05-18T03:03:05.235519567Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:03:20.046048914Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":82,"events_lines":2,"console_offset":166,"console_lines":1}
+{"time":"2026-05-18T03:03:21.475103491Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:03:23.723161046Z","level":"WARN","msg":"handler: ignoring partial history record","step":18030,"current":18491}
+{"time":"2026-05-18T03:03:35.045536904Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":84,"events_lines":2,"console_offset":167,"console_lines":2}
+{"time":"2026-05-18T03:03:36.444960283Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:03:46.881172236Z","level":"WARN","msg":"handler: ignoring partial history record","step":18040,"current":18491}
+{"time":"2026-05-18T03:03:50.045919751Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":86,"events_lines":2,"console_offset":169,"console_lines":2}
+{"time":"2026-05-18T03:03:50.365305423Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:04:05.046224849Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":88,"events_lines":2}
+{"time":"2026-05-18T03:04:06.183350344Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:04:10.039395492Z","level":"WARN","msg":"handler: ignoring partial history record","step":18050,"current":18491}
+{"time":"2026-05-18T03:04:20.045337679Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":90,"events_lines":2,"console_offset":171,"console_lines":2}
+{"time":"2026-05-18T03:04:20.240571381Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:04:33.205158034Z","level":"WARN","msg":"handler: ignoring partial history record","step":18060,"current":18491}
+{"time":"2026-05-18T03:04:35.045570461Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":92,"events_lines":2,"console_offset":173,"console_lines":1}
+{"time":"2026-05-18T03:04:35.284707207Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:04:50.046010683Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":94,"events_lines":2,"console_offset":174,"console_lines":1}
+{"time":"2026-05-18T03:04:50.248350648Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:04:56.364553548Z","level":"WARN","msg":"handler: ignoring partial history record","step":18070,"current":18491}
+{"time":"2026-05-18T03:05:05.046244364Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":96,"events_lines":2,"console_offset":175,"console_lines":2}
+{"time":"2026-05-18T03:05:05.27060869Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:05:19.507631084Z","level":"WARN","msg":"handler: ignoring partial history record","step":18080,"current":18491}
+{"time":"2026-05-18T03:05:20.045519495Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":98,"events_lines":2,"console_offset":177,"console_lines":1}
+{"time":"2026-05-18T03:05:21.170704296Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:05:35.045953024Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":100,"events_lines":2,"console_offset":178,"console_lines":1}
+{"time":"2026-05-18T03:05:35.294738378Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:05:42.650557474Z","level":"WARN","msg":"handler: ignoring partial history record","step":18090,"current":18491}
+{"time":"2026-05-18T03:05:50.046194622Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":102,"events_lines":2,"console_offset":179,"console_lines":2}
+{"time":"2026-05-18T03:05:51.117027931Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:06:05.045378881Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":104,"events_lines":2}
+{"time":"2026-05-18T03:06:05.812347661Z","level":"WARN","msg":"handler: ignoring partial history record","step":18100,"current":18491}
+{"time":"2026-05-18T03:06:06.248290724Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:06:20.046125005Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":106,"events_lines":2,"console_offset":181,"console_lines":2}
+{"time":"2026-05-18T03:06:20.239877161Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:06:28.967209584Z","level":"WARN","msg":"handler: ignoring partial history record","step":18110,"current":18491}
+{"time":"2026-05-18T03:06:35.045932134Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":108,"events_lines":2,"console_offset":183,"console_lines":1}
+{"time":"2026-05-18T03:06:35.235698904Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:06:50.045944951Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":110,"events_lines":2,"console_offset":184,"console_lines":1}
+{"time":"2026-05-18T03:06:50.391823177Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:06:52.121769886Z","level":"WARN","msg":"handler: ignoring partial history record","step":18120,"current":18491}
+{"time":"2026-05-18T03:07:05.046216898Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":112,"events_lines":2,"console_offset":185,"console_lines":2}
+{"time":"2026-05-18T03:07:05.271288236Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:07:15.33121767Z","level":"WARN","msg":"handler: ignoring partial history record","step":18130,"current":18491}
+{"time":"2026-05-18T03:07:20.045479441Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":114,"events_lines":2,"console_offset":187,"console_lines":2}
+{"time":"2026-05-18T03:07:20.286443593Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:07:35.046250425Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":116,"events_lines":2}
+{"time":"2026-05-18T03:07:35.338145814Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:07:38.486535934Z","level":"WARN","msg":"handler: ignoring partial history record","step":18140,"current":18491}
+{"time":"2026-05-18T03:07:50.046056009Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":118,"events_lines":2,"console_offset":189,"console_lines":2}
+{"time":"2026-05-18T03:07:50.631948864Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:08:01.671960848Z","level":"WARN","msg":"handler: ignoring partial history record","step":18150,"current":18491}
+{"time":"2026-05-18T03:08:05.046264204Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":120,"events_lines":2,"console_offset":191,"console_lines":1}
+{"time":"2026-05-18T03:08:05.285062203Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:08:20.046160863Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":122,"events_lines":2,"console_offset":192,"console_lines":1}
+{"time":"2026-05-18T03:08:20.243989306Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:08:24.904090951Z","level":"WARN","msg":"handler: ignoring partial history record","step":18160,"current":18491}
+{"time":"2026-05-18T03:08:35.045540782Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":124,"events_lines":2,"console_offset":193,"console_lines":2}
+{"time":"2026-05-18T03:08:35.376374164Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:08:48.134170684Z","level":"WARN","msg":"handler: ignoring partial history record","step":18170,"current":18491}
+{"time":"2026-05-18T03:08:50.045316863Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":126,"events_lines":2,"console_offset":195,"console_lines":1}
+{"time":"2026-05-18T03:08:50.819569186Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:09:05.045868631Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":128,"events_lines":2,"console_offset":196,"console_lines":1}
+{"time":"2026-05-18T03:09:05.232329505Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:09:11.356144696Z","level":"WARN","msg":"handler: ignoring partial history record","step":18180,"current":18491}
+{"time":"2026-05-18T03:09:20.045656979Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":130,"events_lines":2,"console_offset":197,"console_lines":2}
+{"time":"2026-05-18T03:09:20.253701191Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:09:34.539609144Z","level":"WARN","msg":"handler: ignoring partial history record","step":18190,"current":18491}
+{"time":"2026-05-18T03:09:35.046153474Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":132,"events_lines":2,"console_offset":199,"console_lines":1}
+{"time":"2026-05-18T03:09:35.254055704Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:09:50.045817376Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":134,"events_lines":2,"console_offset":200,"console_lines":1}
+{"time":"2026-05-18T03:09:50.573844564Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:09:57.671104727Z","level":"WARN","msg":"handler: ignoring partial history record","step":18200,"current":18491}
+{"time":"2026-05-18T03:10:05.045349683Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":136,"events_lines":2,"console_offset":201,"console_lines":2}
+{"time":"2026-05-18T03:10:05.264109107Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:10:20.046394746Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":138,"events_lines":2}
+{"time":"2026-05-18T03:10:20.806458788Z","level":"WARN","msg":"handler: ignoring partial history record","step":18210,"current":18491}
+{"time":"2026-05-18T03:10:21.047995665Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:10:35.046150574Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":140,"events_lines":2,"console_offset":203,"console_lines":2}
+{"time":"2026-05-18T03:10:35.82814907Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:10:43.906584415Z","level":"WARN","msg":"handler: ignoring partial history record","step":18220,"current":18491}
+{"time":"2026-05-18T03:10:50.046222894Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":142,"events_lines":2,"console_offset":205,"console_lines":2}
+{"time":"2026-05-18T03:10:50.344374441Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:11:05.045527548Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":144,"events_lines":2}
+{"time":"2026-05-18T03:11:05.325314202Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:11:07.047704967Z","level":"WARN","msg":"handler: ignoring partial history record","step":18230,"current":18491}
+{"time":"2026-05-18T03:11:20.04618332Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":146,"events_lines":2,"console_offset":207,"console_lines":2}
+{"time":"2026-05-18T03:11:20.320849027Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:11:30.170792432Z","level":"WARN","msg":"handler: ignoring partial history record","step":18240,"current":18491}
+{"time":"2026-05-18T03:11:35.046212805Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":148,"events_lines":2,"console_offset":209,"console_lines":1}
+{"time":"2026-05-18T03:11:36.209050409Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:11:50.046167497Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":150,"events_lines":2,"console_offset":210,"console_lines":1}
+{"time":"2026-05-18T03:11:51.797586205Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:11:53.276961242Z","level":"WARN","msg":"handler: ignoring partial history record","step":18250,"current":18491}
+{"time":"2026-05-18T03:12:05.045259461Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":152,"events_lines":2,"console_offset":211,"console_lines":2}
+{"time":"2026-05-18T03:12:05.265268707Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:12:20.046145463Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":154,"events_lines":2}
+{"time":"2026-05-18T03:12:20.280235048Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:12:35.045913917Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":156,"events_lines":2}
+{"time":"2026-05-18T03:12:35.482815546Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:12:50.045381592Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":158,"events_lines":2}
+{"time":"2026-05-18T03:12:50.309292914Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:13:05.046041687Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":160,"events_lines":2}
+{"time":"2026-05-18T03:13:05.330169548Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:13:20.046058191Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":162,"events_lines":2}
+{"time":"2026-05-18T03:13:20.293482562Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:13:35.045676109Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":164,"events_lines":2}
+{"time":"2026-05-18T03:13:35.343309672Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:13:50.045365064Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":166,"events_lines":2}
+{"time":"2026-05-18T03:13:50.235708236Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:14:05.046049947Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":168,"events_lines":2}
+{"time":"2026-05-18T03:14:05.250315901Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:14:20.045436961Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":170,"events_lines":2}
+{"time":"2026-05-18T03:14:20.797673938Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:14:24.388344552Z","level":"WARN","msg":"handler: ignoring partial history record","step":18250,"current":18491}
+{"time":"2026-05-18T03:14:35.045529519Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":172,"events_lines":2,"console_offset":213,"console_lines":2}
+{"time":"2026-05-18T03:14:35.247070369Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:14:47.534697439Z","level":"WARN","msg":"handler: ignoring partial history record","step":18260,"current":18491}
+{"time":"2026-05-18T03:14:50.046219993Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":174,"events_lines":2,"console_offset":215,"console_lines":2}
+{"time":"2026-05-18T03:14:50.235804233Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:15:05.045396857Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":176,"events_lines":2}
+{"time":"2026-05-18T03:15:05.370580062Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:15:10.688235934Z","level":"WARN","msg":"handler: ignoring partial history record","step":18270,"current":18491}
+{"time":"2026-05-18T03:15:20.045947532Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":178,"events_lines":2,"console_offset":217,"console_lines":2}
+{"time":"2026-05-18T03:15:21.447262591Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:15:33.833858982Z","level":"WARN","msg":"handler: ignoring partial history record","step":18280,"current":18491}
+{"time":"2026-05-18T03:15:35.045312938Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":180,"events_lines":2,"console_offset":219,"console_lines":1}
+{"time":"2026-05-18T03:15:35.743036257Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:15:50.045971094Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":182,"events_lines":2,"console_offset":220,"console_lines":1}
+{"time":"2026-05-18T03:15:50.746227725Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:15:56.964968564Z","level":"WARN","msg":"handler: ignoring partial history record","step":18290,"current":18491}
+{"time":"2026-05-18T03:16:05.045924377Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":184,"events_lines":2,"console_offset":221,"console_lines":2}
+{"time":"2026-05-18T03:16:05.233779616Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:16:20.04625924Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":186,"events_lines":2}
+{"time":"2026-05-18T03:16:20.102886669Z","level":"WARN","msg":"handler: ignoring partial history record","step":18300,"current":18491}
+{"time":"2026-05-18T03:16:20.245391684Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:16:35.046042069Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":188,"events_lines":2,"console_offset":223,"console_lines":2}
+{"time":"2026-05-18T03:16:35.502588625Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:16:43.250172219Z","level":"WARN","msg":"handler: ignoring partial history record","step":18310,"current":18491}
+{"time":"2026-05-18T03:16:50.045901362Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":190,"events_lines":2,"console_offset":225,"console_lines":2}
+{"time":"2026-05-18T03:16:50.246335379Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:17:05.045502629Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":192,"events_lines":2}
+{"time":"2026-05-18T03:17:05.245661786Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:17:06.40150417Z","level":"WARN","msg":"handler: ignoring partial history record","step":18320,"current":18491}
+{"time":"2026-05-18T03:17:20.046072703Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":194,"events_lines":2,"console_offset":227,"console_lines":2}
+{"time":"2026-05-18T03:17:20.52227265Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:17:29.547475607Z","level":"WARN","msg":"handler: ignoring partial history record","step":18330,"current":18491}
+{"time":"2026-05-18T03:17:35.046257822Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":196,"events_lines":2,"console_offset":229,"console_lines":1}
+{"time":"2026-05-18T03:17:35.24466242Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:17:50.046185787Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":198,"events_lines":2,"console_offset":230,"console_lines":1}
+{"time":"2026-05-18T03:17:50.659280096Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:17:52.685705367Z","level":"WARN","msg":"handler: ignoring partial history record","step":18340,"current":18491}
+{"time":"2026-05-18T03:18:05.045825777Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":200,"events_lines":2,"console_offset":231,"console_lines":2}
+{"time":"2026-05-18T03:18:05.263169448Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:18:15.834810363Z","level":"WARN","msg":"handler: ignoring partial history record","step":18350,"current":18491}
+{"time":"2026-05-18T03:18:20.046406881Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":202,"events_lines":2,"console_offset":233,"console_lines":2}
+{"time":"2026-05-18T03:18:20.254765349Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:18:35.045784098Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":204,"events_lines":2}
+{"time":"2026-05-18T03:18:35.283668312Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:18:38.990104787Z","level":"WARN","msg":"handler: ignoring partial history record","step":18360,"current":18491}
+{"time":"2026-05-18T03:18:50.045510593Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":206,"events_lines":2,"console_offset":235,"console_lines":2}
+{"time":"2026-05-18T03:18:50.333790684Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:19:03.813169021Z","level":"WARN","msg":"handler: ignoring partial history record","step":18370,"current":18491}
+{"time":"2026-05-18T03:19:05.045612343Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":208,"events_lines":2,"console_offset":237,"console_lines":1}
+{"time":"2026-05-18T03:19:05.282662669Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:19:20.04561802Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":210,"events_lines":2,"console_offset":238,"console_lines":1}
+{"time":"2026-05-18T03:19:21.378907766Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:19:27.030912972Z","level":"WARN","msg":"handler: ignoring partial history record","step":18380,"current":18491}
+{"time":"2026-05-18T03:19:35.046038847Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":212,"events_lines":2,"console_offset":239,"console_lines":2}
+{"time":"2026-05-18T03:19:35.270114681Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:19:50.045672468Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":214,"events_lines":2}
+{"time":"2026-05-18T03:19:50.270956125Z","level":"WARN","msg":"handler: ignoring partial history record","step":18390,"current":18491}
+{"time":"2026-05-18T03:19:50.315633198Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:20:05.046030691Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":216,"events_lines":2,"console_offset":241,"console_lines":2}
+{"time":"2026-05-18T03:20:05.294064104Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:20:13.505463073Z","level":"WARN","msg":"handler: ignoring partial history record","step":18400,"current":18491}
+{"time":"2026-05-18T03:20:20.046281304Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":218,"events_lines":2,"console_offset":243,"console_lines":2}
+{"time":"2026-05-18T03:20:20.257569336Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:20:35.045936377Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":220,"events_lines":2}
+{"time":"2026-05-18T03:20:36.384065367Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:20:36.770739419Z","level":"WARN","msg":"handler: ignoring partial history record","step":18410,"current":18491}
+{"time":"2026-05-18T03:20:50.046230805Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":222,"events_lines":2,"console_offset":245,"console_lines":2}
+{"time":"2026-05-18T03:20:50.263355871Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:20:59.995712834Z","level":"WARN","msg":"handler: ignoring partial history record","step":18420,"current":18491}
+{"time":"2026-05-18T03:21:05.045924927Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":224,"events_lines":2,"console_offset":247,"console_lines":1}
+{"time":"2026-05-18T03:21:08.049047196Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:21:20.045825769Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":226,"events_lines":2,"console_offset":248,"console_lines":1}
+{"time":"2026-05-18T03:21:20.248462802Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:21:23.237596388Z","level":"WARN","msg":"handler: ignoring partial history record","step":18430,"current":18491}
+{"time":"2026-05-18T03:21:35.046154741Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":228,"events_lines":2,"console_offset":249,"console_lines":2}
+{"time":"2026-05-18T03:21:36.754269285Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:21:46.452763592Z","level":"WARN","msg":"handler: ignoring partial history record","step":18440,"current":18491}
+{"time":"2026-05-18T03:21:50.046102704Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":230,"events_lines":2,"console_offset":251,"console_lines":2}
+{"time":"2026-05-18T03:21:51.448218704Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:22:05.046175083Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":232,"events_lines":2}
+{"time":"2026-05-18T03:22:05.233512746Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:22:09.677805717Z","level":"WARN","msg":"handler: ignoring partial history record","step":18450,"current":18491}
+{"time":"2026-05-18T03:22:20.045627706Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":234,"events_lines":2,"console_offset":253,"console_lines":2}
+{"time":"2026-05-18T03:22:21.611223834Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:22:32.916015729Z","level":"WARN","msg":"handler: ignoring partial history record","step":18460,"current":18491}
+{"time":"2026-05-18T03:22:35.045658543Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":236,"events_lines":2,"console_offset":255,"console_lines":1}
+{"time":"2026-05-18T03:22:36.242356439Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:22:50.046198029Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":238,"events_lines":2,"console_offset":256,"console_lines":1}
+{"time":"2026-05-18T03:22:50.248220744Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:22:56.145158396Z","level":"WARN","msg":"handler: ignoring partial history record","step":18470,"current":18491}
+{"time":"2026-05-18T03:23:05.045634821Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":240,"events_lines":2,"console_offset":257,"console_lines":2}
+{"time":"2026-05-18T03:23:05.247055799Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:23:19.377130745Z","level":"WARN","msg":"handler: ignoring partial history record","step":18480,"current":18491}
+{"time":"2026-05-18T03:23:20.046140923Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":242,"events_lines":2,"console_offset":259,"console_lines":2}
+{"time":"2026-05-18T03:23:20.353650808Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:23:35.045668572Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":244,"events_lines":2}
+{"time":"2026-05-18T03:23:35.972153968Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:23:42.625925141Z","level":"WARN","msg":"handler: ignoring partial history record","step":18490,"current":18491}
+{"time":"2026-05-18T03:23:50.045320527Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":246,"events_lines":2,"console_offset":261,"console_lines":2}
+{"time":"2026-05-18T03:23:50.252123304Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:24:05.045802072Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":248,"events_lines":2}
+{"time":"2026-05-18T03:24:05.23083908Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:24:20.045744116Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":250,"events_lines":2,"console_offset":263,"console_lines":1}
+{"time":"2026-05-18T03:24:20.228146541Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:24:35.045694922Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":252,"events_lines":2}
+{"time":"2026-05-18T03:24:36.208969832Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:24:50.046154999Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":254,"events_lines":2}
+{"time":"2026-05-18T03:24:50.281918426Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:25:05.045903794Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":256,"events_lines":2}
+{"time":"2026-05-18T03:25:05.295176398Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:25:20.045453734Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":258,"events_lines":2}
+{"time":"2026-05-18T03:25:20.229238322Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:25:35.045985448Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":260,"events_lines":2}
+{"time":"2026-05-18T03:25:35.388057099Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:25:50.04523703Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":262,"events_lines":2}
+{"time":"2026-05-18T03:25:50.25289568Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:26:05.045631349Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":264,"events_lines":2}
+{"time":"2026-05-18T03:26:05.24203078Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:26:20.046208311Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":266,"events_lines":2}
+{"time":"2026-05-18T03:26:20.260917426Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:26:35.045237752Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":268,"events_lines":2}
+{"time":"2026-05-18T03:26:35.481779887Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:26:50.045289409Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":270,"events_lines":2,"console_offset":264,"console_lines":1}
+{"time":"2026-05-18T03:26:50.306573074Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:27:05.046486246Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1849,"history_lines":1,"events_offset":272,"events_lines":2,"console_offset":265,"console_lines":1}
+{"time":"2026-05-18T03:27:05.895731457Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:27:20.045977912Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":274,"events_lines":2}
+{"time":"2026-05-18T03:27:20.507968296Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:27:35.045933808Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1850,"history_lines":1,"events_offset":276,"events_lines":2,"console_offset":266,"console_lines":1}
+{"time":"2026-05-18T03:27:35.485221318Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:27:50.045641765Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1851,"history_lines":1,"events_offset":278,"events_lines":2,"console_offset":267,"console_lines":1}
+{"time":"2026-05-18T03:27:50.285731124Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:28:05.045967483Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":280,"events_lines":2}
+{"time":"2026-05-18T03:28:05.255888863Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:28:20.045502451Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1852,"history_lines":1,"events_offset":282,"events_lines":2,"console_offset":268,"console_lines":1}
+{"time":"2026-05-18T03:28:20.668558334Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:28:35.045522123Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1853,"history_lines":1,"events_offset":284,"events_lines":2,"console_offset":269,"console_lines":1}
+{"time":"2026-05-18T03:28:36.580108987Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:28:50.046088841Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":286,"events_lines":2}
+{"time":"2026-05-18T03:28:50.320732159Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:29:05.045752742Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1854,"history_lines":1,"events_offset":288,"events_lines":2,"console_offset":270,"console_lines":1}
+{"time":"2026-05-18T03:29:05.52193199Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:29:20.045344804Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1855,"history_lines":1,"events_offset":290,"events_lines":2,"console_offset":271,"console_lines":1}
+{"time":"2026-05-18T03:29:20.408891254Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:29:35.046272991Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":292,"events_lines":2}
+{"time":"2026-05-18T03:29:36.282926875Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:29:50.045977734Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1856,"history_lines":1,"events_offset":294,"events_lines":2,"console_offset":272,"console_lines":1}
+{"time":"2026-05-18T03:29:50.281429693Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:30:05.045939131Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":296,"events_lines":2}
+{"time":"2026-05-18T03:30:05.250132395Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:30:20.04617646Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1857,"history_lines":1,"events_offset":298,"events_lines":2,"console_offset":273,"console_lines":1}
+{"time":"2026-05-18T03:30:20.310241385Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:30:35.045920052Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1858,"history_lines":1,"events_offset":300,"events_lines":2,"console_offset":274,"console_lines":1}
+{"time":"2026-05-18T03:30:35.321893028Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:30:50.045308411Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":302,"events_lines":2}
+{"time":"2026-05-18T03:30:50.308195603Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:31:05.046186375Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1859,"history_lines":1,"events_offset":304,"events_lines":2,"console_offset":275,"console_lines":1}
+{"time":"2026-05-18T03:31:05.331806726Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:31:20.045372843Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1860,"history_lines":1,"events_offset":306,"events_lines":2,"console_offset":276,"console_lines":1}
+{"time":"2026-05-18T03:31:20.63997263Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:31:35.045592664Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":308,"events_lines":2}
+{"time":"2026-05-18T03:31:35.34711738Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:31:50.045513148Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1861,"history_lines":1,"events_offset":310,"events_lines":2,"console_offset":277,"console_lines":1}
+{"time":"2026-05-18T03:31:50.662657262Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:32:05.04608592Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1862,"history_lines":1,"events_offset":312,"events_lines":2,"console_offset":278,"console_lines":1}
+{"time":"2026-05-18T03:32:05.40229921Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:32:20.046535819Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":314,"events_lines":2}
+{"time":"2026-05-18T03:32:20.24297499Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:32:35.04613736Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1863,"history_lines":1,"events_offset":316,"events_lines":2,"console_offset":279,"console_lines":1}
+{"time":"2026-05-18T03:32:35.651096624Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:32:50.045847244Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1864,"history_lines":1,"events_offset":318,"events_lines":2,"console_offset":280,"console_lines":1}
+{"time":"2026-05-18T03:32:50.28165912Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:33:05.04544463Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":320,"events_lines":2}
+{"time":"2026-05-18T03:33:05.243702456Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:33:20.046088268Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1865,"history_lines":1,"events_offset":322,"events_lines":2,"console_offset":281,"console_lines":1}
+{"time":"2026-05-18T03:33:20.362443939Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:33:35.046089827Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":324,"events_lines":2}
+{"time":"2026-05-18T03:33:35.257600427Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:33:50.046291845Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1866,"history_lines":1,"events_offset":326,"events_lines":2,"console_offset":282,"console_lines":1}
+{"time":"2026-05-18T03:33:51.072097363Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:34:05.046360795Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1867,"history_lines":1,"events_offset":328,"events_lines":2,"console_offset":283,"console_lines":1}
+{"time":"2026-05-18T03:34:05.253883051Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:34:20.046355304Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":330,"events_lines":2}
+{"time":"2026-05-18T03:34:20.259958276Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:34:35.046236453Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1868,"history_lines":1,"events_offset":332,"events_lines":2,"console_offset":284,"console_lines":1}
+{"time":"2026-05-18T03:34:35.249562135Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:34:50.045885776Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1869,"history_lines":1,"events_offset":334,"events_lines":2,"console_offset":285,"console_lines":1}
+{"time":"2026-05-18T03:34:50.345960009Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:35:05.045960344Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":336,"events_lines":2}
+{"time":"2026-05-18T03:35:05.296337978Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:35:20.045851739Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1870,"history_lines":1,"events_offset":338,"events_lines":2,"console_offset":286,"console_lines":1}
+{"time":"2026-05-18T03:35:20.287375978Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:35:35.046286836Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1871,"history_lines":1,"events_offset":340,"events_lines":2,"console_offset":287,"console_lines":1}
+{"time":"2026-05-18T03:35:35.55384685Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:35:50.045462483Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":342,"events_lines":2}
+{"time":"2026-05-18T03:35:50.28815816Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:36:05.046094589Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1872,"history_lines":1,"events_offset":344,"events_lines":2,"console_offset":288,"console_lines":1}
+{"time":"2026-05-18T03:36:05.263633297Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:36:20.046296605Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1873,"history_lines":1,"events_offset":346,"events_lines":2,"console_offset":289,"console_lines":1}
+{"time":"2026-05-18T03:36:20.33595778Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:36:35.045309316Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":348,"events_lines":2}
+{"time":"2026-05-18T03:36:35.578977293Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:36:50.045518482Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":350,"events_lines":2}
+{"time":"2026-05-18T03:36:50.371698629Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:37:05.046031072Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":352,"events_lines":2}
+{"time":"2026-05-18T03:37:05.262863025Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:37:20.045985278Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":354,"events_lines":2}
+{"time":"2026-05-18T03:37:20.274195499Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:37:35.045314782Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":356,"events_lines":2}
+{"time":"2026-05-18T03:37:35.297670372Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:37:50.045863031Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":358,"events_lines":2}
+{"time":"2026-05-18T03:37:50.660593974Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:38:05.045828961Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":360,"events_lines":2}
+{"time":"2026-05-18T03:38:05.253325156Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:38:20.045667216Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":362,"events_lines":2}
+{"time":"2026-05-18T03:38:20.725332136Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:38:35.046148915Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":364,"events_lines":2}
+{"time":"2026-05-18T03:38:35.515724303Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:38:50.046049094Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":366,"events_lines":2,"console_offset":290,"console_lines":1}
+{"time":"2026-05-18T03:38:50.265450074Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:39:05.045318322Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":368,"events_lines":2}
+{"time":"2026-05-18T03:39:05.233686306Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:39:20.046246415Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1874,"history_lines":1,"events_offset":370,"events_lines":2,"console_offset":291,"console_lines":1}
+{"time":"2026-05-18T03:39:20.26581183Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:39:35.045657304Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":372,"events_lines":2}
+{"time":"2026-05-18T03:39:35.267348011Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:39:50.045472155Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1875,"history_lines":1,"events_offset":374,"events_lines":2,"console_offset":292,"console_lines":1}
+{"time":"2026-05-18T03:39:50.31004976Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:40:05.045810899Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1876,"history_lines":1,"events_offset":376,"events_lines":2,"console_offset":293,"console_lines":1}
+{"time":"2026-05-18T03:40:05.500218399Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:40:20.046276375Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":378,"events_lines":2}
+{"time":"2026-05-18T03:40:20.294056515Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:40:35.046056697Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1877,"history_lines":1,"events_offset":380,"events_lines":2,"console_offset":294,"console_lines":1}
+{"time":"2026-05-18T03:40:35.41279343Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:40:50.045836148Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1878,"history_lines":1,"events_offset":382,"events_lines":2,"console_offset":295,"console_lines":1}
+{"time":"2026-05-18T03:40:50.566776324Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:41:05.045222822Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":384,"events_lines":2}
+{"time":"2026-05-18T03:41:05.302196492Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:41:20.046308662Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1879,"history_lines":1,"events_offset":386,"events_lines":2,"console_offset":296,"console_lines":1}
+{"time":"2026-05-18T03:41:20.764055098Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:41:35.045983357Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1880,"history_lines":1,"events_offset":388,"events_lines":2,"console_offset":297,"console_lines":1}
+{"time":"2026-05-18T03:41:35.29359801Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:41:50.045406766Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":390,"events_lines":2}
+{"time":"2026-05-18T03:41:50.243215262Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:42:05.046131237Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1881,"history_lines":1,"events_offset":392,"events_lines":2,"console_offset":298,"console_lines":1}
+{"time":"2026-05-18T03:42:06.103113025Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:42:20.046175934Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1882,"history_lines":1,"events_offset":394,"events_lines":2,"console_offset":299,"console_lines":1}
+{"time":"2026-05-18T03:42:20.27936563Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:42:35.046146606Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":396,"events_lines":2}
+{"time":"2026-05-18T03:42:35.247711206Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:42:50.045943961Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1883,"history_lines":1,"events_offset":398,"events_lines":2,"console_offset":300,"console_lines":1}
+{"time":"2026-05-18T03:42:50.274844118Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:43:05.046243257Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":400,"events_lines":2}
+{"time":"2026-05-18T03:43:05.252289554Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:43:20.046076185Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1884,"history_lines":1,"events_offset":402,"events_lines":2,"console_offset":301,"console_lines":1}
+{"time":"2026-05-18T03:43:20.391265144Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:43:35.045482577Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1885,"history_lines":1,"events_offset":404,"events_lines":2,"console_offset":302,"console_lines":1}
+{"time":"2026-05-18T03:43:35.544732407Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:43:50.045468955Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":406,"events_lines":2}
+{"time":"2026-05-18T03:43:50.273887793Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:44:05.046093795Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1886,"history_lines":1,"events_offset":408,"events_lines":2,"console_offset":303,"console_lines":1}
+{"time":"2026-05-18T03:44:05.288274472Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:44:20.045660313Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1887,"history_lines":1,"events_offset":410,"events_lines":2,"console_offset":304,"console_lines":1}
+{"time":"2026-05-18T03:44:20.260149684Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:44:35.046179408Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":412,"events_lines":2}
+{"time":"2026-05-18T03:44:35.27532838Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:44:50.04622011Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1888,"history_lines":1,"events_offset":414,"events_lines":2,"console_offset":305,"console_lines":1}
+{"time":"2026-05-18T03:44:50.429626755Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:45:05.046104888Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1889,"history_lines":1,"events_offset":416,"events_lines":2,"console_offset":306,"console_lines":1}
+{"time":"2026-05-18T03:45:05.286978243Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:45:20.045343702Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":418,"events_lines":2}
+{"time":"2026-05-18T03:45:20.371732897Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:45:35.045859792Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1890,"history_lines":1,"events_offset":420,"events_lines":2,"console_offset":307,"console_lines":1}
+{"time":"2026-05-18T03:45:35.247480702Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:45:50.04532155Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1891,"history_lines":1,"events_offset":422,"events_lines":2,"console_offset":308,"console_lines":1}
+{"time":"2026-05-18T03:45:50.349123935Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:46:05.046149939Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":424,"events_lines":2}
+{"time":"2026-05-18T03:46:05.346529846Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:46:20.046174087Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1892,"history_lines":1,"events_offset":426,"events_lines":2,"console_offset":309,"console_lines":1}
+{"time":"2026-05-18T03:46:20.543113099Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:46:35.045559241Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1893,"history_lines":1,"events_offset":428,"events_lines":2,"console_offset":310,"console_lines":1}
+{"time":"2026-05-18T03:46:35.295237846Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:46:50.045507579Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":430,"events_lines":2}
+{"time":"2026-05-18T03:46:50.293638679Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:47:05.045715723Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1894,"history_lines":1,"events_offset":432,"events_lines":2,"console_offset":311,"console_lines":1}
+{"time":"2026-05-18T03:47:05.442740136Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:47:20.045573203Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":434,"events_lines":2}
+{"time":"2026-05-18T03:47:20.263917554Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:47:35.046285329Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1895,"history_lines":1,"events_offset":436,"events_lines":2,"console_offset":312,"console_lines":1}
+{"time":"2026-05-18T03:47:35.25482554Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:47:50.045813768Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1896,"history_lines":1,"events_offset":438,"events_lines":2,"console_offset":313,"console_lines":1}
+{"time":"2026-05-18T03:47:50.442029265Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:48:05.046225496Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":440,"events_lines":2}
+{"time":"2026-05-18T03:48:05.258764794Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:48:20.046189455Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1897,"history_lines":1,"events_offset":442,"events_lines":2,"console_offset":314,"console_lines":1}
+{"time":"2026-05-18T03:48:20.326356159Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:48:35.045401232Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1898,"history_lines":1,"events_offset":444,"events_lines":2,"console_offset":315,"console_lines":1}
+{"time":"2026-05-18T03:48:35.317393859Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:48:50.046191846Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":446,"events_lines":2}
+{"time":"2026-05-18T03:48:50.290483857Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:49:05.046409984Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":448,"events_lines":2}
+{"time":"2026-05-18T03:49:05.288721739Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:49:20.046107885Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":450,"events_lines":2}
+{"time":"2026-05-18T03:49:20.352521818Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:49:35.045996961Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":452,"events_lines":2}
+{"time":"2026-05-18T03:49:35.34307973Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:49:50.045243309Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":454,"events_lines":2}
+{"time":"2026-05-18T03:49:50.281728349Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:50:05.045247677Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":456,"events_lines":2}
+{"time":"2026-05-18T03:50:05.241532398Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:50:20.04622646Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":458,"events_lines":2}
+{"time":"2026-05-18T03:50:20.599159837Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:50:35.045901697Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":460,"events_lines":2}
+{"time":"2026-05-18T03:50:35.251027211Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:50:50.045530916Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":462,"events_lines":2}
+{"time":"2026-05-18T03:50:50.253258863Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:51:05.045324793Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":464,"events_lines":2,"console_offset":316,"console_lines":1}
+{"time":"2026-05-18T03:51:05.22988499Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:51:20.045400634Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":466,"events_lines":2,"console_offset":317,"console_lines":1}
+{"time":"2026-05-18T03:51:20.231713286Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:51:35.046318071Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1899,"history_lines":1,"events_offset":468,"events_lines":2,"console_offset":318,"console_lines":1}
+{"time":"2026-05-18T03:51:35.278575767Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:51:50.045941136Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":470,"events_lines":2}
+{"time":"2026-05-18T03:51:50.232620796Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:52:05.046129886Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1900,"history_lines":1,"events_offset":472,"events_lines":2,"console_offset":319,"console_lines":1}
+{"time":"2026-05-18T03:52:05.357330326Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:52:20.046118516Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1901,"history_lines":1,"events_offset":474,"events_lines":2,"console_offset":320,"console_lines":1}
+{"time":"2026-05-18T03:52:20.26768023Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:52:35.046270861Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":476,"events_lines":2}
+{"time":"2026-05-18T03:52:35.295107808Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:52:50.045893226Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1902,"history_lines":1,"events_offset":478,"events_lines":2,"console_offset":321,"console_lines":1}
+{"time":"2026-05-18T03:52:50.36310055Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:53:05.045901396Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1903,"history_lines":1,"events_offset":480,"events_lines":2,"console_offset":322,"console_lines":1}
+{"time":"2026-05-18T03:53:05.360377745Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:53:20.045367198Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":482,"events_lines":2}
+{"time":"2026-05-18T03:53:20.278580498Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:53:35.04617727Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1904,"history_lines":1,"events_offset":484,"events_lines":2,"console_offset":323,"console_lines":1}
+{"time":"2026-05-18T03:53:35.576625362Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:53:50.046520826Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1905,"history_lines":1,"events_offset":486,"events_lines":2,"console_offset":324,"console_lines":1}
+{"time":"2026-05-18T03:53:50.72006515Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:54:05.045357584Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":488,"events_lines":2}
+{"time":"2026-05-18T03:54:05.250596282Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:54:20.045927107Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1906,"history_lines":1,"events_offset":490,"events_lines":2,"console_offset":325,"console_lines":1}
+{"time":"2026-05-18T03:54:20.600027572Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:54:35.045447766Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":492,"events_lines":2}
+{"time":"2026-05-18T03:54:35.242874287Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:54:50.046037029Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1907,"history_lines":1,"events_offset":494,"events_lines":2,"console_offset":326,"console_lines":1}
+{"time":"2026-05-18T03:54:50.263576632Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:55:05.046088289Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1908,"history_lines":1,"events_offset":496,"events_lines":2,"console_offset":327,"console_lines":1}
+{"time":"2026-05-18T03:55:05.29676621Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:55:20.045401466Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":498,"events_lines":2}
+{"time":"2026-05-18T03:55:20.236809842Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:55:35.046570414Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1909,"history_lines":1,"events_offset":500,"events_lines":2,"console_offset":328,"console_lines":1}
+{"time":"2026-05-18T03:55:35.263718167Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:55:50.046363439Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1910,"history_lines":1,"events_offset":502,"events_lines":2,"console_offset":329,"console_lines":1}
+{"time":"2026-05-18T03:55:50.279372429Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:56:05.04569122Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":504,"events_lines":2}
+{"time":"2026-05-18T03:56:05.256657178Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:56:20.046372159Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1911,"history_lines":1,"events_offset":506,"events_lines":2,"console_offset":330,"console_lines":1}
+{"time":"2026-05-18T03:56:20.264991278Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:56:35.045489737Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1912,"history_lines":1,"events_offset":508,"events_lines":2,"console_offset":331,"console_lines":1}
+{"time":"2026-05-18T03:56:35.26897806Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:56:50.045914384Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":510,"events_lines":2}
+{"time":"2026-05-18T03:56:50.237318344Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:57:05.045909901Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1913,"history_lines":1,"events_offset":512,"events_lines":2,"console_offset":332,"console_lines":1}
+{"time":"2026-05-18T03:57:05.343197142Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:57:20.04565415Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1914,"history_lines":1,"events_offset":514,"events_lines":2,"console_offset":333,"console_lines":1}
+{"time":"2026-05-18T03:57:20.297916016Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:57:35.045715583Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":516,"events_lines":2}
+{"time":"2026-05-18T03:57:35.241987026Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:57:50.046040899Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1915,"history_lines":1,"events_offset":518,"events_lines":2,"console_offset":334,"console_lines":1}
+{"time":"2026-05-18T03:57:50.398604885Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:58:05.046519764Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1916,"history_lines":1,"events_offset":520,"events_lines":2,"console_offset":335,"console_lines":1}
+{"time":"2026-05-18T03:58:05.291686963Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:58:20.0462155Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":522,"events_lines":2}
+{"time":"2026-05-18T03:58:20.2496015Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:58:35.046250763Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1917,"history_lines":1,"events_offset":524,"events_lines":2,"console_offset":336,"console_lines":1}
+{"time":"2026-05-18T03:58:35.307314104Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:58:50.04569193Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":526,"events_lines":2}
+{"time":"2026-05-18T03:58:50.279748079Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:59:05.045630198Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1918,"history_lines":1,"events_offset":528,"events_lines":2,"console_offset":337,"console_lines":1}
+{"time":"2026-05-18T03:59:05.519245317Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:59:20.045854063Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1919,"history_lines":1,"events_offset":530,"events_lines":2,"console_offset":338,"console_lines":1}
+{"time":"2026-05-18T03:59:20.401442022Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:59:35.04527558Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":532,"events_lines":2}
+{"time":"2026-05-18T03:59:35.235644515Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T03:59:50.045478045Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1920,"history_lines":1,"events_offset":534,"events_lines":2,"console_offset":339,"console_lines":1}
+{"time":"2026-05-18T03:59:50.253150138Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:00:05.046211677Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1921,"history_lines":1,"events_offset":536,"events_lines":2,"console_offset":340,"console_lines":1}
+{"time":"2026-05-18T04:00:05.302499989Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:00:20.04550453Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":538,"events_lines":2}
+{"time":"2026-05-18T04:00:20.276989979Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:00:35.045359741Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1922,"history_lines":1,"events_offset":540,"events_lines":2,"console_offset":341,"console_lines":1}
+{"time":"2026-05-18T04:00:35.27568574Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:00:50.04642639Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1923,"history_lines":1,"events_offset":542,"events_lines":2,"console_offset":342,"console_lines":1}
+{"time":"2026-05-18T04:00:50.296544434Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:01:05.04546287Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":544,"events_lines":2}
+{"time":"2026-05-18T04:01:05.235660146Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:01:20.04604761Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":546,"events_lines":2}
+{"time":"2026-05-18T04:01:20.243794341Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:01:35.045302705Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":548,"events_lines":2}
+{"time":"2026-05-18T04:01:35.280489378Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:01:50.045986221Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":550,"events_lines":2}
+{"time":"2026-05-18T04:01:50.259632146Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:02:05.046276962Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":552,"events_lines":2}
+{"time":"2026-05-18T04:02:05.264971221Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:02:20.0460359Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":554,"events_lines":2}
+{"time":"2026-05-18T04:02:20.283367891Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:02:35.045986193Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":556,"events_lines":2}
+{"time":"2026-05-18T04:02:35.263729383Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:02:50.045951699Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":558,"events_lines":2}
+{"time":"2026-05-18T04:02:50.281486398Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:03:05.045514534Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":560,"events_lines":2}
+{"time":"2026-05-18T04:03:05.269596863Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:03:20.045340141Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":562,"events_lines":2,"console_offset":343,"console_lines":1}
+{"time":"2026-05-18T04:03:20.233622067Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:03:35.045922137Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":564,"events_lines":2}
+{"time":"2026-05-18T04:03:35.267833577Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:03:50.045967092Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1924,"history_lines":1,"events_offset":566,"events_lines":2,"console_offset":344,"console_lines":1}
+{"time":"2026-05-18T04:03:50.316360189Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:04:05.046078546Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1925,"history_lines":1,"events_offset":568,"events_lines":2,"console_offset":345,"console_lines":1}
+{"time":"2026-05-18T04:04:05.340265065Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:04:20.045894851Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":570,"events_lines":2}
+{"time":"2026-05-18T04:04:20.250108695Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:04:35.045913397Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1926,"history_lines":1,"events_offset":572,"events_lines":2,"console_offset":346,"console_lines":1}
+{"time":"2026-05-18T04:04:35.782079352Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:04:50.046058493Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":574,"events_lines":2}
+{"time":"2026-05-18T04:04:50.251181984Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:05:05.045711924Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1927,"history_lines":1,"events_offset":576,"events_lines":2,"console_offset":347,"console_lines":1}
+{"time":"2026-05-18T04:05:05.294247528Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:05:20.0455421Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1928,"history_lines":1,"events_offset":578,"events_lines":2,"console_offset":348,"console_lines":1}
+{"time":"2026-05-18T04:05:20.275214334Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:05:35.046046661Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":580,"events_lines":2}
+{"time":"2026-05-18T04:05:35.259500704Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:05:50.046431244Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1929,"history_lines":1,"events_offset":582,"events_lines":2,"console_offset":349,"console_lines":1}
+{"time":"2026-05-18T04:05:50.297608831Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:06:05.046149711Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1930,"history_lines":1,"events_offset":584,"events_lines":2,"console_offset":350,"console_lines":1}
+{"time":"2026-05-18T04:06:05.300654476Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:06:20.045622567Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":586,"events_lines":2}
+{"time":"2026-05-18T04:06:20.256795069Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:06:35.045395901Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1931,"history_lines":1,"events_offset":588,"events_lines":2,"console_offset":351,"console_lines":1}
+{"time":"2026-05-18T04:06:35.271633765Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:06:50.045933507Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1932,"history_lines":1,"events_offset":590,"events_lines":2,"console_offset":352,"console_lines":1}
+{"time":"2026-05-18T04:06:50.291228919Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:07:05.04532434Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":592,"events_lines":2}
+{"time":"2026-05-18T04:07:05.426062668Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:07:20.046276093Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1933,"history_lines":1,"events_offset":594,"events_lines":2,"console_offset":353,"console_lines":1}
+{"time":"2026-05-18T04:07:20.525659474Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:07:35.046260733Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1934,"history_lines":1,"events_offset":596,"events_lines":2,"console_offset":354,"console_lines":1}
+{"time":"2026-05-18T04:07:35.329561709Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:07:50.046168869Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":598,"events_lines":2}
+{"time":"2026-05-18T04:07:50.256251295Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-18T04:08:05.045959389Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1935,"history_lines":1,"events_offset":600,"events_lines":2,"console_offset":355,"console_lines":1}
+{"time":"2026-05-18T04:08:05.301024781Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

epfl_final/wandb/run-20260518_025234-hi2z3jh6/logs/debug.log

@@ -0,0 +1,20 @@
+2026-05-18 02:52:34,029 INFO    MainThread:2092 [wandb_setup.py:_flush():81] Current SDK version is 0.27.0
+2026-05-18 02:52:34,030 INFO    MainThread:2092 [wandb_setup.py:_flush():81] Configure stats pid to 2092
+2026-05-18 02:52:34,030 INFO    MainThread:2092 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-18 02:52:34,031 INFO    MainThread:2092 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260518_025234-hi2z3jh6/logs/debug.log
+2026-05-18 02:52:34,032 INFO    MainThread:2092 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260518_025234-hi2z3jh6/logs/debug-internal.log
+2026-05-18 02:52:34,032 INFO    MainThread:2092 [wandb_init.py:init():851] calling init triggers
+2026-05-18 02:52:34,033 INFO    MainThread:2092 [wandb_init.py:init():856] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': 'checkpoints/run_20260517_023855', 'resume_from_checkpoint': 'checkpoints/run_20260517_023855/checkpoint-18000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'sparse 30%', 'wandb_tags': ['qwen3', '0.6b'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 18, 'sparse_mask_pool_size': 2000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_19', 'dense_mode': False, '_wandb': {}}
+2026-05-18 02:52:34,033 INFO    MainThread:2092 [wandb_init.py:init():899] starting backend
+2026-05-18 02:52:34,251 INFO    MainThread:2092 [wandb_init.py:init():914] sending inform_init request
+2026-05-18 02:52:34,602 INFO    MainThread:2092 [wandb_init.py:init():919] backend started and connected
+2026-05-18 02:52:34,604 INFO    MainThread:2092 [wandb_init.py:init():989] updated telemetry
+2026-05-18 02:52:34,606 INFO    MainThread:2092 [wandb_init.py:init():1012] communicating run to backend with 90.0 second timeout
+2026-05-18 02:52:35,041 INFO    MainThread:2092 [wandb_init.py:init():1052] run resumed
+2026-05-18 02:52:35,043 INFO    MainThread:2092 [wandb_init.py:init():1057] starting run threads in backend
+2026-05-18 02:52:35,098 INFO    MainThread:2092 [wandb_run.py:_console_start():2509] atexit reg
+2026-05-18 02:52:35,098 INFO    MainThread:2092 [wandb_run.py:_redirect():2359] redirect: wrap_raw
+2026-05-18 02:52:35,099 INFO    MainThread:2092 [wandb_run.py:_redirect():2428] Wrapping output streams.
+2026-05-18 02:52:35,099 INFO    MainThread:2092 [wandb_run.py:_redirect():2451] Redirects installed.
+2026-05-18 02:52:35,100 INFO    MainThread:2092 [wandb_init.py:init():1095] run started, returning control to user process

epfl_final/wandb/run-20260518_025234-hi2z3jh6/run-hi2z3jh6.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ba5038a416c5b6a4c6f350cef9b6a0dde964c67e262d229e4947c0416663325
+size 720896

epfl_final/wandb/run-20260519_211405-hi2z3jh6/files/output.log

@@ -0,0 +1,50 @@
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260517_023855",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260517_023855/checkpoint-18000",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 20005,
+  "per_device_batch_size": 3,
+  "gradient_accumulation_steps": 8,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 250,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 200,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "dense continued 8k",
+  "wandb_tags": [
+    "qwen3",
+    "0.6b",
+    "dense-resume"
+  ],
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 10,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "dense_mode": true
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260517_023855/checkpoint-18000
+Checkpoint activations: False

epfl_final/wandb/run-20260519_211405-hi2z3jh6/files/requirements.txt

@@ -0,0 +1,97 @@
+pytz==2026.2
+nvidia-cusparselt-cu13==0.8.1
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.7.0
+tzdata==2026.2
+typing_extensions==4.15.0
+triton==3.7.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+safetensors==0.7.0
+regex==2026.5.9
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.5.2
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.29.7
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+numpy==2.2.6
+networkx==3.4.2
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.15
+hf-xet==1.5.0
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.4.0
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+async-timeout==5.0.1
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+typing-inspection==0.4.2
+sentry-sdk==2.60.0
+requests==2.34.2
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.4
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cublas==13.1.1.3
+multiprocess==0.70.19
+multidict==6.7.1
+markdown-it-py==4.2.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+exceptiongroup==1.3.1
+cuda-bindings==13.2.0
+aiosignal==1.4.0
+yarl==1.23.0
+rich==15.0.0
+pydantic==2.13.4
+pandas==2.3.3
+nvidia-cusolver==12.0.4.66
+nvidia-cudnn-cu13==9.20.0.48
+GitPython==3.1.50
+anyio==4.13.0
+wandb==0.27.0
+typer==0.25.1
+httpx==0.28.1
+aiohttp==3.13.5
+torch==2.12.0
+huggingface_hub==1.15.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.8.1
+PyGObject==3.42.1
+dbus-python==1.2.18
+distro==1.7.0
+pip==22.0.2
+setuptools==59.6.0
+ssh-import-id==5.11
+wheel==0.37.1

epfl_final/wandb/run-20260519_211405-hi2z3jh6/files/wandb-metadata.json

@@ -0,0 +1,36 @@
+{
+  "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.12",
+  "startedAt": "2026-05-19T21:14:05.833185Z",
+  "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath": "SparseAtt/train.py",
+  "codePathLocal": "SparseAtt/train.py",
+  "email": "ruben.navasardyan@mail.utoronto.ca",
+  "root": "/mloscratch/homes/navasard/sparse",
+  "host": "h100-workspace-13-0-3",
+  "executable": "/usr/bin/python3",
+  "cpu_count": 96,
+  "cpu_count_logical": 192,
+  "gpu": "NVIDIA H100 80GB HBM3",
+  "gpu_count": 1,
+  "disk": {
+    "/": {
+      "total": "9374931091456",
+      "used": "6156233617408"
+    }
+  },
+  "memory": {
+    "total": "1623005847552"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-61e64705-1798-f2b5-568f-877549238703"
+    }
+  ],
+  "cudaVersion": "13.0",
+  "writerId": "h8td68bdmaj848g6o55ar5vrkbkp5mhu"
+}

epfl_final/wandb/run-20260519_211405-hi2z3jh6/logs/debug-internal.log

@@ -0,0 +1,89 @@
+{"time":"2026-05-19T21:14:06.087711529Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-05-19T21:14:06.090764288Z","level":"INFO","msg":"stream: starting","core version":"0.27.0"}
+{"time":"2026-05-19T21:14:06.39797429Z","level":"INFO","msg":"stream: created new stream","id":"hi2z3jh6"}
+{"time":"2026-05-19T21:14:06.398142943Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-05-19T21:14:06.3999877Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-05-19T21:14:06.400016093Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-05-19T21:14:06.400006278Z","level":"INFO","msg":"writer: started","stream_id":"hi2z3jh6"}
+{"time":"2026-05-19T21:14:06.906165968Z","level":"INFO","msg":"filestream: sending request","total_files":1}
+{"time":"2026-05-19T21:14:07.177996129Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:14:21.908610853Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":50,"uploaded_len":2}
+{"time":"2026-05-19T21:14:22.0866173Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:14:36.907140064Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":0,"events_lines":2}
+{"time":"2026-05-19T21:14:37.107874248Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:14:51.906376938Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
+{"time":"2026-05-19T21:14:52.086543866Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:15:06.906293304Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
+{"time":"2026-05-19T21:15:07.177278379Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:15:21.906675257Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":6,"events_lines":2}
+{"time":"2026-05-19T21:15:22.12382434Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:15:36.912634031Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":8,"events_lines":2}
+{"time":"2026-05-19T21:15:37.083708703Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:15:51.910563333Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":10,"events_lines":2}
+{"time":"2026-05-19T21:15:52.104875877Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:16:06.90778921Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":12,"events_lines":2}
+{"time":"2026-05-19T21:16:07.168271222Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:16:21.907225072Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":14,"events_lines":2}
+{"time":"2026-05-19T21:16:22.10286791Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:16:36.909737237Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":16,"events_lines":2}
+{"time":"2026-05-19T21:16:37.13267388Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:16:51.910549333Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":18,"events_lines":2}
+{"time":"2026-05-19T21:16:52.105512962Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:17:06.908260998Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":20,"events_lines":2}
+{"time":"2026-05-19T21:17:07.097102406Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:17:21.908532902Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":22,"events_lines":2}
+{"time":"2026-05-19T21:17:22.111042708Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:17:36.906521311Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":24,"events_lines":2}
+{"time":"2026-05-19T21:17:37.355790896Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:17:51.907094169Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":26,"events_lines":2}
+{"time":"2026-05-19T21:17:52.077939823Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:18:06.907099317Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":28,"events_lines":2}
+{"time":"2026-05-19T21:18:07.111294296Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:18:21.907546287Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":30,"events_lines":2}
+{"time":"2026-05-19T21:18:25.277973789Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:18:36.909588544Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":32,"events_lines":2}
+{"time":"2026-05-19T21:18:37.09038434Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:18:51.910659698Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":34,"events_lines":2}
+{"time":"2026-05-19T21:18:52.112437118Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:19:06.906326933Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":36,"events_lines":2}
+{"time":"2026-05-19T21:19:07.107770714Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:19:21.910504426Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":38,"events_lines":3}
+{"time":"2026-05-19T21:19:22.139967085Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:19:36.910501743Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":41,"events_lines":1}
+{"time":"2026-05-19T21:19:37.344283086Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:19:51.907608893Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":42,"events_lines":2}
+{"time":"2026-05-19T21:19:52.314672286Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:20:06.906212404Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":44,"events_lines":2}
+{"time":"2026-05-19T21:20:07.390293423Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:20:21.910615925Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":46,"events_lines":2}
+{"time":"2026-05-19T21:20:22.793683348Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:20:36.907500873Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":48,"events_lines":2}
+{"time":"2026-05-19T21:20:50.963711676Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:20:51.908174983Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":50,"events_lines":2}
+{"time":"2026-05-19T21:20:52.093381681Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:21:06.906726044Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":52,"events_lines":2}
+{"time":"2026-05-19T21:21:07.10530657Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:21:21.906542921Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":54,"events_lines":2}
+{"time":"2026-05-19T21:21:22.086036428Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:21:36.906747696Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":56,"events_lines":2}
+{"time":"2026-05-19T21:21:37.07878232Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:21:51.906690604Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":58,"events_lines":2}
+{"time":"2026-05-19T21:21:52.088184438Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:22:06.909548135Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":60,"events_lines":2}
+{"time":"2026-05-19T21:22:07.425647719Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:22:21.906284926Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":62,"events_lines":2}
+{"time":"2026-05-19T21:22:22.095907909Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:22:36.910069664Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":64,"events_lines":2}
+{"time":"2026-05-19T21:22:37.083512936Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:22:51.910469856Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":66,"events_lines":2}
+{"time":"2026-05-19T21:22:52.107758294Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:23:06.909860957Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":68,"events_lines":2}
+{"time":"2026-05-19T21:23:07.113280694Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:23:21.906698426Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":70,"events_lines":2}
+{"time":"2026-05-19T21:23:22.091626731Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:23:36.910481001Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":72,"events_lines":3}
+{"time":"2026-05-19T21:23:37.095771088Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:23:51.906737129Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":75,"events_lines":1}
+{"time":"2026-05-19T21:23:52.121523511Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:24:06.907014627Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":76,"events_lines":2}
+{"time":"2026-05-19T21:24:07.129349219Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

epfl_final/wandb/run-20260519_211405-hi2z3jh6/logs/debug.log

@@ -0,0 +1,20 @@
+2026-05-19 21:14:05,852 INFO    MainThread:5684 [wandb_setup.py:_flush():81] Current SDK version is 0.27.0
+2026-05-19 21:14:05,852 INFO    MainThread:5684 [wandb_setup.py:_flush():81] Configure stats pid to 5684
+2026-05-19 21:14:05,852 INFO    MainThread:5684 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-19 21:14:05,853 INFO    MainThread:5684 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260519_211405-hi2z3jh6/logs/debug.log
+2026-05-19 21:14:05,853 INFO    MainThread:5684 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260519_211405-hi2z3jh6/logs/debug-internal.log
+2026-05-19 21:14:05,854 INFO    MainThread:5684 [wandb_init.py:init():851] calling init triggers
+2026-05-19 21:14:05,854 INFO    MainThread:5684 [wandb_init.py:init():856] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/run_20260517_023855', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/run_20260517_023855/checkpoint-18000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'dense continued 8k', 'wandb_tags': ['qwen3', '0.6b', 'dense-resume'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 10, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'dense_mode': True, '_wandb': {}}
+2026-05-19 21:14:05,855 INFO    MainThread:5684 [wandb_init.py:init():899] starting backend
+2026-05-19 21:14:06,079 INFO    MainThread:5684 [wandb_init.py:init():914] sending inform_init request
+2026-05-19 21:14:06,400 INFO    MainThread:5684 [wandb_init.py:init():919] backend started and connected
+2026-05-19 21:14:06,403 INFO    MainThread:5684 [wandb_init.py:init():989] updated telemetry
+2026-05-19 21:14:06,404 INFO    MainThread:5684 [wandb_init.py:init():1012] communicating run to backend with 90.0 second timeout
+2026-05-19 21:14:06,903 INFO    MainThread:5684 [wandb_init.py:init():1052] run resumed
+2026-05-19 21:14:06,905 INFO    MainThread:5684 [wandb_init.py:init():1057] starting run threads in backend
+2026-05-19 21:14:06,979 INFO    MainThread:5684 [wandb_run.py:_console_start():2509] atexit reg
+2026-05-19 21:14:06,981 INFO    MainThread:5684 [wandb_run.py:_redirect():2359] redirect: wrap_raw
+2026-05-19 21:14:06,982 INFO    MainThread:5684 [wandb_run.py:_redirect():2428] Wrapping output streams.
+2026-05-19 21:14:06,982 INFO    MainThread:5684 [wandb_run.py:_redirect():2451] Redirects installed.
+2026-05-19 21:14:06,984 INFO    MainThread:5684 [wandb_init.py:init():1095] run started, returning control to user process

epfl_final/wandb/run-20260519_212517-hi2z3jh6/files/output.log

@@ -0,0 +1,50 @@
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260517_023855",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260517_023855/checkpoint-18000",
+  "force_new_run": true,
+  "num_epochs": 1,
+  "max_steps": 20005,
+  "per_device_batch_size": 3,
+  "gradient_accumulation_steps": 8,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 250,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 200,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "dense continued 8k",
+  "wandb_tags": [
+    "qwen3",
+    "0.6b",
+    "dense-resume"
+  ],
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 10,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "dense_mode": true
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260517_023855/checkpoint-18000
+Checkpoint activations: False

epfl_final/wandb/run-20260519_212517-hi2z3jh6/files/requirements.txt

@@ -0,0 +1,97 @@
+pytz==2026.2
+nvidia-cusparselt-cu13==0.8.1
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.7.0
+tzdata==2026.2
+typing_extensions==4.15.0
+triton==3.7.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+safetensors==0.7.0
+regex==2026.5.9
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.5.2
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.29.7
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+numpy==2.2.6
+networkx==3.4.2
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.15
+hf-xet==1.5.0
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.4.0
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+async-timeout==5.0.1
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+typing-inspection==0.4.2
+sentry-sdk==2.60.0
+requests==2.34.2
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.4
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cublas==13.1.1.3
+multiprocess==0.70.19
+multidict==6.7.1
+markdown-it-py==4.2.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+exceptiongroup==1.3.1
+cuda-bindings==13.2.0
+aiosignal==1.4.0
+yarl==1.23.0
+rich==15.0.0
+pydantic==2.13.4
+pandas==2.3.3
+nvidia-cusolver==12.0.4.66
+nvidia-cudnn-cu13==9.20.0.48
+GitPython==3.1.50
+anyio==4.13.0
+wandb==0.27.0
+typer==0.25.1
+httpx==0.28.1
+aiohttp==3.13.5
+torch==2.12.0
+huggingface_hub==1.15.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.8.1
+PyGObject==3.42.1
+dbus-python==1.2.18
+distro==1.7.0
+pip==22.0.2
+setuptools==59.6.0
+ssh-import-id==5.11
+wheel==0.37.1

epfl_final/wandb/run-20260519_212517-hi2z3jh6/files/wandb-metadata.json

@@ -0,0 +1,36 @@
+{
+  "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.12",
+  "startedAt": "2026-05-19T21:25:17.303865Z",
+  "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath": "SparseAtt/train.py",
+  "codePathLocal": "SparseAtt/train.py",
+  "email": "ruben.navasardyan@mail.utoronto.ca",
+  "root": "/mloscratch/homes/navasard/sparse",
+  "host": "h100-workspace-13-0-3",
+  "executable": "/usr/bin/python3",
+  "cpu_count": 96,
+  "cpu_count_logical": 192,
+  "gpu": "NVIDIA H100 80GB HBM3",
+  "gpu_count": 1,
+  "disk": {
+    "/": {
+      "total": "9374931091456",
+      "used": "6156056440832"
+    }
+  },
+  "memory": {
+    "total": "1623005847552"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-61e64705-1798-f2b5-568f-877549238703"
+    }
+  ],
+  "cudaVersion": "13.0",
+  "writerId": "hqqharbk2nw9uf2dsq8lvmctpcfjq5qc"
+}

epfl_final/wandb/run-20260519_212517-hi2z3jh6/logs/debug-internal.log

@@ -0,0 +1,21 @@
+{"time":"2026-05-19T21:25:17.578467679Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-05-19T21:25:17.57882237Z","level":"INFO","msg":"stream: starting","core version":"0.27.0"}
+{"time":"2026-05-19T21:25:18.100093395Z","level":"INFO","msg":"stream: created new stream","id":"hi2z3jh6"}
+{"time":"2026-05-19T21:25:18.100218122Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-05-19T21:25:18.10215394Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-05-19T21:25:18.102184966Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-05-19T21:25:18.1021741Z","level":"INFO","msg":"writer: started","stream_id":"hi2z3jh6"}
+{"time":"2026-05-19T21:25:18.92545579Z","level":"INFO","msg":"filestream: sending request","total_files":1}
+{"time":"2026-05-19T21:25:19.613615506Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:25:33.926456662Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":50,"uploaded_len":2}
+{"time":"2026-05-19T21:25:34.162040637Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:25:48.925334545Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":0,"events_lines":2}
+{"time":"2026-05-19T21:25:49.210384089Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:26:03.925675633Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
+{"time":"2026-05-19T21:26:04.182788179Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:26:18.927584558Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
+{"time":"2026-05-19T21:26:19.217308721Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:26:33.925610902Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":6,"events_lines":2}
+{"time":"2026-05-19T21:26:34.135852732Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-19T21:26:48.926258951Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":8,"events_lines":2}
+{"time":"2026-05-19T21:26:49.122228219Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

epfl_final/wandb/run-20260519_212517-hi2z3jh6/logs/debug.log

@@ -0,0 +1,20 @@
+2026-05-19 21:25:17,330 INFO    MainThread:7228 [wandb_setup.py:_flush():81] Current SDK version is 0.27.0
+2026-05-19 21:25:17,330 INFO    MainThread:7228 [wandb_setup.py:_flush():81] Configure stats pid to 7228
+2026-05-19 21:25:17,331 INFO    MainThread:7228 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-19 21:25:17,331 INFO    MainThread:7228 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260519_212517-hi2z3jh6/logs/debug.log
+2026-05-19 21:25:17,337 INFO    MainThread:7228 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260519_212517-hi2z3jh6/logs/debug-internal.log
+2026-05-19 21:25:17,338 INFO    MainThread:7228 [wandb_init.py:init():851] calling init triggers
+2026-05-19 21:25:17,339 INFO    MainThread:7228 [wandb_init.py:init():856] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/run_20260517_023855', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/run_20260517_023855/checkpoint-18000', 'force_new_run': True, 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'dense continued 8k', 'wandb_tags': ['qwen3', '0.6b', 'dense-resume'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 10, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'dense_mode': True, '_wandb': {}}
+2026-05-19 21:25:17,341 INFO    MainThread:7228 [wandb_init.py:init():899] starting backend
+2026-05-19 21:25:17,570 INFO    MainThread:7228 [wandb_init.py:init():914] sending inform_init request
+2026-05-19 21:25:18,104 INFO    MainThread:7228 [wandb_init.py:init():919] backend started and connected
+2026-05-19 21:25:18,106 INFO    MainThread:7228 [wandb_init.py:init():989] updated telemetry
+2026-05-19 21:25:18,108 INFO    MainThread:7228 [wandb_init.py:init():1012] communicating run to backend with 90.0 second timeout
+2026-05-19 21:25:18,922 INFO    MainThread:7228 [wandb_init.py:init():1052] run resumed
+2026-05-19 21:25:18,924 INFO    MainThread:7228 [wandb_init.py:init():1057] starting run threads in backend
+2026-05-19 21:25:18,990 INFO    MainThread:7228 [wandb_run.py:_console_start():2509] atexit reg
+2026-05-19 21:25:18,990 INFO    MainThread:7228 [wandb_run.py:_redirect():2359] redirect: wrap_raw
+2026-05-19 21:25:18,990 INFO    MainThread:7228 [wandb_run.py:_redirect():2428] Wrapping output streams.
+2026-05-19 21:25:18,990 INFO    MainThread:7228 [wandb_run.py:_redirect():2451] Redirects installed.
+2026-05-19 21:25:18,991 INFO    MainThread:7228 [wandb_init.py:init():1095] run started, returning control to user process

epfl_final/wandb/run-20260519_212833-oeak7lgl/files/output.log

@@ -0,0 +1,217 @@
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "./checkpoints/run_20260519_212829",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260517_023855/checkpoint-18000",
+  "force_new_run": true,
+  "num_epochs": 1,
+  "max_steps": 20005,
+  "per_device_batch_size": 3,
+  "gradient_accumulation_steps": 8,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 250,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 200,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "dense continued 8k",
+  "wandb_tags": [
+    "qwen3",
+    "0.6b",
+    "dense-resume"
+  ],
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 10,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "dense_mode": true
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260517_023855/checkpoint-18000
+Checkpoint activations: False
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1024
+  num_hidden_layers: 28
+  num_attention_heads: 16
+  num_key_value_heads: 8
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 10
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 596,049,920
+  Trainable parameters: 596,049,920
+  Non-trainable parameters: 0
+  Model size (BF16): 1.19 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [3, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [3, 8192, 1024]           --                        True
+│    └─Embedding: 2-1                                        [3, 8192]                 [3, 8192, 1024]           155,582,464               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    └─Qwen3RMSNorm: 2-3                                     [3, 8192, 1024]           [3, 8192, 1024]           1,024                     True
+├─Linear: 1-2                                                [3, 8192, 1024]           [3, 8192, 151936]         155,582,464               True
+================================================================================================================================================================
+Total params: 751,632,384
+Trainable params: 751,632,384
+Non-trainable params: 0
+Total mult-adds (G): 2.25
+================================================================================================================================================================
+Input size (MB): 0.20
+Forward/backward pass size (MB): 63052.97
+Params size (MB): 1503.26
+Estimated Total Size (MB): 64556.43
+================================================================================================================================================================
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003)
+Dataset split: 1017602 train, 254401 validation
+Training examples (packed, train split):  1017602
+Validation examples (packed, val split):  254401
+Per-device batch size:                    3
+World size (GPUs):                        1
+Gradient accumulation steps:              8
+Effective batch size (seqs/step):         24
+Sequence length:                          8192
+Tokens per optimizer step:                196,608
+Micro-batches per epoch (per rank):       339200
+Optimizer steps per epoch:                42400
+Total optimizer steps:                    20005  (capped by max_steps; 0.47 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260517_023855/checkpoint-18000...
+Resumed from checkpoint at step 17999 (epoch 0, micro_step 144024)
+Resuming training from step 17999
+Starting training...
+Step 18000/20005 | Loss: 0.2932 | LR: 1.26e-05 | 638407.9ms/step | 0.0 samples/s
+Validation at step 18000: Loss = 2.9421, Perplexity = 18.96
+Saved checkpoint to checkpoints/run_20260519_212829/checkpoint-18000
+Step 18010/20005 | Loss: 2.9429 | LR: 1.24e-05 | 44368.4ms/step | 0.5 samples/s
+Step 18020/20005 | Loss: 2.8996 | LR: 1.23e-05 | 5892.3ms/step | 4.1 samples/s
+Step 18030/20005 | Loss: 2.8949 | LR: 1.22e-05 | 5974.8ms/step | 4.0 samples/s
+Step 18040/20005 | Loss: 2.8822 | LR: 1.21e-05 | 5889.9ms/step | 4.1 samples/s
+Step 18050/20005 | Loss: 2.9279 | LR: 1.19e-05 | 5891.4ms/step | 4.1 samples/s
+Step 18060/20005 | Loss: 2.9004 | LR: 1.18e-05 | 5883.8ms/step | 4.1 samples/s
+Step 18070/20005 | Loss: 2.8831 | LR: 1.17e-05 | 5895.3ms/step | 4.1 samples/s
+Step 18080/20005 | Loss: 2.8618 | LR: 1.16e-05 | 5892.2ms/step | 4.1 samples/s
+Step 18090/20005 | Loss: 2.8818 | LR: 1.15e-05 | 5892.1ms/step | 4.1 samples/s
+Step 18100/20005 | Loss: 2.8697 | LR: 1.13e-05 | 5898.6ms/step | 4.1 samples/s
+Step 18110/20005 | Loss: 2.8987 | LR: 1.12e-05 | 5892.7ms/step | 4.1 samples/s
+Step 18120/20005 | Loss: 2.9018 | LR: 1.11e-05 | 5889.0ms/step | 4.1 samples/s
+Step 18130/20005 | Loss: 2.9144 | LR: 1.10e-05 | 5893.2ms/step | 4.1 samples/s
+Step 18140/20005 | Loss: 2.9089 | LR: 1.09e-05 | 5888.2ms/step | 4.1 samples/s
+Step 18150/20005 | Loss: 2.8999 | LR: 1.08e-05 | 5889.4ms/step | 4.1 samples/s
+Step 18160/20005 | Loss: 2.8862 | LR: 1.06e-05 | 5887.0ms/step | 4.1 samples/s
+Step 18170/20005 | Loss: 2.9011 | LR: 1.05e-05 | 5886.8ms/step | 4.1 samples/s
+Step 18180/20005 | Loss: 2.9096 | LR: 1.04e-05 | 5897.3ms/step | 4.1 samples/s
+Step 18190/20005 | Loss: 2.8956 | LR: 1.03e-05 | 5897.9ms/step | 4.1 samples/s
+Step 18200/20005 | Loss: 2.8945 | LR: 1.02e-05 | 5894.8ms/step | 4.1 samples/s
+Step 18210/20005 | Loss: 2.8873 | LR: 1.01e-05 | 5887.9ms/step | 4.1 samples/s
+Step 18220/20005 | Loss: 2.8870 | LR: 9.96e-06 | 5889.2ms/step | 4.1 samples/s
+Step 18230/20005 | Loss: 2.9008 | LR: 9.85e-06 | 5886.2ms/step | 4.1 samples/s
+Step 18240/20005 | Loss: 2.8994 | LR: 9.74e-06 | 5890.5ms/step | 4.1 samples/s
+Step 18250/20005 | Loss: 2.9039 | LR: 9.63e-06 | 5887.4ms/step | 4.1 samples/s
+Validation at step 18250: Loss = 2.9114, Perplexity = 18.38
+Step 18260/20005 | Loss: 2.8679 | LR: 9.52e-06 | 40989.8ms/step | 0.6 samples/s
+Step 18270/20005 | Loss: 2.8799 | LR: 9.41e-06 | 5890.8ms/step | 4.1 samples/s
+Step 18280/20005 | Loss: 2.8720 | LR: 9.30e-06 | 5890.4ms/step | 4.1 samples/s
+Step 18290/20005 | Loss: 2.8980 | LR: 9.20e-06 | 5890.3ms/step | 4.1 samples/s
+Step 18300/20005 | Loss: 2.9070 | LR: 9.09e-06 | 5890.3ms/step | 4.1 samples/s
+Step 18310/20005 | Loss: 2.8755 | LR: 8.99e-06 | 5888.9ms/step | 4.1 samples/s
+Step 18320/20005 | Loss: 2.9267 | LR: 8.88e-06 | 5891.0ms/step | 4.1 samples/s
+Step 18330/20005 | Loss: 2.9028 | LR: 8.78e-06 | 5887.3ms/step | 4.1 samples/s
+Step 18340/20005 | Loss: 2.8972 | LR: 8.67e-06 | 5883.4ms/step | 4.1 samples/s
+Step 18350/20005 | Loss: 2.9147 | LR: 8.57e-06 | 5882.8ms/step | 4.1 samples/s
+Step 18360/20005 | Loss: 2.8991 | LR: 8.46e-06 | 5897.9ms/step | 4.1 samples/s
+Step 18370/20005 | Loss: 2.9054 | LR: 8.36e-06 | 5898.7ms/step | 4.1 samples/s
+Step 18380/20005 | Loss: 2.8886 | LR: 8.26e-06 | 5897.2ms/step | 4.1 samples/s
+Step 18390/20005 | Loss: 2.8998 | LR: 8.16e-06 | 5885.8ms/step | 4.1 samples/s
+Step 18400/20005 | Loss: 2.9087 | LR: 8.06e-06 | 5890.7ms/step | 4.1 samples/s
+Step 18410/20005 | Loss: 2.8897 | LR: 7.96e-06 | 5889.8ms/step | 4.1 samples/s
+Step 18420/20005 | Loss: 2.8887 | LR: 7.86e-06 | 5887.1ms/step | 4.1 samples/s
+Step 18430/20005 | Loss: 2.8875 | LR: 7.76e-06 | 5885.5ms/step | 4.1 samples/s
+Step 18440/20005 | Loss: 2.8767 | LR: 7.66e-06 | 5888.8ms/step | 4.1 samples/s
+Step 18450/20005 | Loss: 2.9146 | LR: 7.57e-06 | 5886.7ms/step | 4.1 samples/s
+Step 18460/20005 | Loss: 2.8945 | LR: 7.47e-06 | 5884.0ms/step | 4.1 samples/s
+Step 18470/20005 | Loss: 2.9128 | LR: 7.37e-06 | 5895.7ms/step | 4.1 samples/s
+Step 18480/20005 | Loss: 2.9192 | LR: 7.28e-06 | 5897.7ms/step | 4.1 samples/s
+Step 18490/20005 | Loss: 2.8914 | LR: 7.18e-06 | 5895.0ms/step | 4.1 samples/s
+Step 18500/20005 | Loss: 2.8621 | LR: 7.09e-06 | 5954.1ms/step | 4.0 samples/s
+Validation at step 18500: Loss = 2.9103, Perplexity = 18.36
+Step 18510/20005 | Loss: 2.9280 | LR: 6.99e-06 | 41109.7ms/step | 0.6 samples/s
+Step 18520/20005 | Loss: 2.8590 | LR: 6.90e-06 | 5906.6ms/step | 4.1 samples/s
+Step 18530/20005 | Loss: 2.8556 | LR: 6.81e-06 | 5913.0ms/step | 4.1 samples/s
+Step 18540/20005 | Loss: 2.8744 | LR: 6.72e-06 | 5908.0ms/step | 4.1 samples/s
+Step 18550/20005 | Loss: 2.8819 | LR: 6.63e-06 | 5907.4ms/step | 4.1 samples/s
+Step 18560/20005 | Loss: 2.8989 | LR: 6.54e-06 | 5904.5ms/step | 4.1 samples/s
+Step 18570/20005 | Loss: 2.8603 | LR: 6.45e-06 | 5903.6ms/step | 4.1 samples/s
+Step 18580/20005 | Loss: 2.9193 | LR: 6.36e-06 | 5907.4ms/step | 4.1 samples/s
+Step 18590/20005 | Loss: 2.8858 | LR: 6.27e-06 | 5903.6ms/step | 4.1 samples/s
+Step 18600/20005 | Loss: 2.9107 | LR: 6.18e-06 | 5915.9ms/step | 4.1 samples/s
+Step 18610/20005 | Loss: 2.9068 | LR: 6.09e-06 | 5902.7ms/step | 4.1 samples/s

epfl_final/wandb/run-20260519_212833-oeak7lgl/files/requirements.txt

@@ -0,0 +1,97 @@
+pytz==2026.2
+nvidia-cusparselt-cu13==0.8.1
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.7.0
+tzdata==2026.2
+typing_extensions==4.15.0
+triton==3.7.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+safetensors==0.7.0
+regex==2026.5.9
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.5.2
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.29.7
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+numpy==2.2.6
+networkx==3.4.2
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.15
+hf-xet==1.5.0
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.4.0
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+async-timeout==5.0.1
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+typing-inspection==0.4.2
+sentry-sdk==2.60.0
+requests==2.34.2
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.4
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cublas==13.1.1.3
+multiprocess==0.70.19
+multidict==6.7.1
+markdown-it-py==4.2.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+exceptiongroup==1.3.1
+cuda-bindings==13.2.0
+aiosignal==1.4.0
+yarl==1.23.0
+rich==15.0.0
+pydantic==2.13.4
+pandas==2.3.3
+nvidia-cusolver==12.0.4.66
+nvidia-cudnn-cu13==9.20.0.48
+GitPython==3.1.50
+anyio==4.13.0
+wandb==0.27.0
+typer==0.25.1
+httpx==0.28.1
+aiohttp==3.13.5
+torch==2.12.0
+huggingface_hub==1.15.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.8.1
+PyGObject==3.42.1
+dbus-python==1.2.18
+distro==1.7.0
+pip==22.0.2
+setuptools==59.6.0
+ssh-import-id==5.11
+wheel==0.37.1

epfl_final/wandb/run-20260519_212833-oeak7lgl/files/wandb-metadata.json

@@ -0,0 +1,36 @@
+{
+  "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.12",
+  "startedAt": "2026-05-19T21:28:33.102047Z",
+  "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath": "SparseAtt/train.py",
+  "codePathLocal": "SparseAtt/train.py",
+  "email": "ruben.navasardyan@mail.utoronto.ca",
+  "root": "/mloscratch/homes/navasard/sparse",
+  "host": "h100-workspace-13-0-3",
+  "executable": "/usr/bin/python3",
+  "cpu_count": 96,
+  "cpu_count_logical": 192,
+  "gpu": "NVIDIA H100 80GB HBM3",
+  "gpu_count": 1,
+  "disk": {
+    "/": {
+      "total": "9374931091456",
+      "used": "6156056453120"
+    }
+  },
+  "memory": {
+    "total": "1623005847552"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-61e64705-1798-f2b5-568f-877549238703"
+    }
+  ],
+  "cudaVersion": "13.0",
+  "writerId": "znhei9gz49jyflp8g8valwp7knqlpdaz"
+}

epfl_final/wandb/run-20260519_212833-oeak7lgl/logs/debug.log

@@ -0,0 +1,19 @@
+2026-05-19 21:28:33,145 INFO    MainThread:8134 [wandb_setup.py:_flush():81] Current SDK version is 0.27.0
+2026-05-19 21:28:33,145 INFO    MainThread:8134 [wandb_setup.py:_flush():81] Configure stats pid to 8134
+2026-05-19 21:28:33,146 INFO    MainThread:8134 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-19 21:28:33,147 INFO    MainThread:8134 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260519_212833-oeak7lgl/logs/debug.log
+2026-05-19 21:28:33,147 INFO    MainThread:8134 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260519_212833-oeak7lgl/logs/debug-internal.log
+2026-05-19 21:28:33,148 INFO    MainThread:8134 [wandb_init.py:init():851] calling init triggers
+2026-05-19 21:28:33,149 INFO    MainThread:8134 [wandb_init.py:init():856] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': './checkpoints/run_20260519_212829', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/run_20260517_023855/checkpoint-18000', 'force_new_run': True, 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'dense continued 8k', 'wandb_tags': ['qwen3', '0.6b', 'dense-resume'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 10, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'dense_mode': True, '_wandb': {}}
+2026-05-19 21:28:33,150 INFO    MainThread:8134 [wandb_init.py:init():899] starting backend
+2026-05-19 21:28:33,387 INFO    MainThread:8134 [wandb_init.py:init():914] sending inform_init request
+2026-05-19 21:28:33,803 INFO    MainThread:8134 [wandb_init.py:init():919] backend started and connected
+2026-05-19 21:28:33,806 INFO    MainThread:8134 [wandb_init.py:init():989] updated telemetry
+2026-05-19 21:28:33,807 INFO    MainThread:8134 [wandb_init.py:init():1012] communicating run to backend with 90.0 second timeout
+2026-05-19 21:28:34,203 INFO    MainThread:8134 [wandb_init.py:init():1057] starting run threads in backend
+2026-05-19 21:28:34,270 INFO    MainThread:8134 [wandb_run.py:_console_start():2509] atexit reg
+2026-05-19 21:28:34,270 INFO    MainThread:8134 [wandb_run.py:_redirect():2359] redirect: wrap_raw
+2026-05-19 21:28:34,271 INFO    MainThread:8134 [wandb_run.py:_redirect():2428] Wrapping output streams.
+2026-05-19 21:28:34,271 INFO    MainThread:8134 [wandb_run.py:_redirect():2451] Redirects installed.
+2026-05-19 21:28:34,272 INFO    MainThread:8134 [wandb_init.py:init():1095] run started, returning control to user process

epfl_final/wandb/run-20260519_212833-oeak7lgl/run-oeak7lgl.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc38439ad58c84c465b9293829c833e309b078e2ba5097921bc6edd9bc549a8b
+size 851968

epfl_final/wandb/run-20260523_073455-skmk8v1r/logs/debug-internal.log

@@ -0,0 +1,21 @@
+{"time":"2026-05-23T07:34:55.582505755Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-05-23T07:34:55.58708061Z","level":"INFO","msg":"stream: starting","core version":"0.27.0"}
+{"time":"2026-05-23T07:34:55.916526297Z","level":"INFO","msg":"stream: created new stream","id":"skmk8v1r"}
+{"time":"2026-05-23T07:34:55.916849891Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-05-23T07:34:55.91904806Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-05-23T07:34:55.919121089Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-05-23T07:34:55.919099587Z","level":"INFO","msg":"writer: started","stream_id":"skmk8v1r"}
+{"time":"2026-05-23T07:34:56.368767567Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-05-23T07:34:56.598412726Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-23T07:35:11.369324787Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":1,"console_offset":1,"console_lines":49,"uploaded_len":2}
+{"time":"2026-05-23T07:35:11.563969704Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-23T07:35:26.369514237Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":1,"events_lines":2}
+{"time":"2026-05-23T07:35:26.56219852Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-23T07:35:41.36967922Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":3,"events_lines":2}
+{"time":"2026-05-23T07:35:41.582781366Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-23T07:35:56.368735795Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":5,"events_lines":2}
+{"time":"2026-05-23T07:35:56.592766023Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-23T07:36:11.36972413Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":7,"events_lines":2}
+{"time":"2026-05-23T07:36:11.616614635Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-23T07:36:26.369487223Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":9,"events_lines":2}
+{"time":"2026-05-23T07:36:26.583713991Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

epfl_final/wandb/run-20260523_073455-skmk8v1r/logs/debug.log

@@ -0,0 +1,19 @@
+2026-05-23 07:34:55,348 INFO    MainThread:31572 [wandb_setup.py:_flush():81] Current SDK version is 0.27.0
+2026-05-23 07:34:55,348 INFO    MainThread:31572 [wandb_setup.py:_flush():81] Configure stats pid to 31572
+2026-05-23 07:34:55,349 INFO    MainThread:31572 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-23 07:34:55,350 INFO    MainThread:31572 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260523_073455-skmk8v1r/logs/debug.log
+2026-05-23 07:34:55,350 INFO    MainThread:31572 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260523_073455-skmk8v1r/logs/debug-internal.log
+2026-05-23 07:34:55,351 INFO    MainThread:31572 [wandb_init.py:init():851] calling init triggers
+2026-05-23 07:34:55,352 INFO    MainThread:31572 [wandb_init.py:init():856] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': './checkpoints/run_20260523_073452', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/run_20260517_023855/checkpoint-18000', 'force_new_run': True, 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'dense continued 8k', 'wandb_tags': ['qwen3', '0.6b', 'dense-resume'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 10, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'dense_mode': True, '_wandb': {}}
+2026-05-23 07:34:55,352 INFO    MainThread:31572 [wandb_init.py:init():899] starting backend
+2026-05-23 07:34:55,571 INFO    MainThread:31572 [wandb_init.py:init():914] sending inform_init request
+2026-05-23 07:34:55,920 INFO    MainThread:31572 [wandb_init.py:init():919] backend started and connected
+2026-05-23 07:34:55,923 INFO    MainThread:31572 [wandb_init.py:init():989] updated telemetry
+2026-05-23 07:34:55,925 INFO    MainThread:31572 [wandb_init.py:init():1012] communicating run to backend with 90.0 second timeout
+2026-05-23 07:34:56,274 INFO    MainThread:31572 [wandb_init.py:init():1057] starting run threads in backend
+2026-05-23 07:34:56,361 INFO    MainThread:31572 [wandb_run.py:_console_start():2509] atexit reg
+2026-05-23 07:34:56,361 INFO    MainThread:31572 [wandb_run.py:_redirect():2359] redirect: wrap_raw
+2026-05-23 07:34:56,361 INFO    MainThread:31572 [wandb_run.py:_redirect():2428] Wrapping output streams.
+2026-05-23 07:34:56,362 INFO    MainThread:31572 [wandb_run.py:_redirect():2451] Redirects installed.
+2026-05-23 07:34:56,365 INFO    MainThread:31572 [wandb_init.py:init():1095] run started, returning control to user process