File size: 52,284 Bytes
ba84e48 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 | [2026-05-16 18:20:43,308] [DEBUG] [axolotl.utils.config.resolve_dtype:74] [PID:14603] bf16 support detected, enabling for this configuration.
[2026-05-16 18:20:43,452] [WARNING] [huggingface_hub.utils._http._warn_on_warning_headers:904] [PID:14603] Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.
[2026-05-16 18:20:43,465] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:14603] baseline 0.000GB ()
[2026-05-16 18:20:43,467] [INFO] [axolotl.cli.config.load_cfg:333] [PID:14603] config:
{
"activation_offloading": true,
"attn_implementation": "flash_attention_2",
"attn_needs_dtype_cast": true,
"attn_supports_packing": true,
"attn_uses_flash_lib": true,
"axolotl_config_path": "train.yml",
"base_model": "shb777/Llama-3.3-8B-Instruct-128K",
"base_model_config": "shb777/Llama-3.3-8B-Instruct-128K",
"batch_size": 32,
"bf16": true,
"capabilities": {
"bf16": true,
"compute_capability": "sm_80",
"fp8": false,
"n_gpu": 1,
"n_node": 1,
"tf32": true
},
"chat_template": "tokenizer_default",
"context_parallel_size": 1,
"cosine_min_lr_ratio": 0.1,
"cut_cross_entropy": true,
"dataloader_num_workers": 1,
"dataloader_pin_memory": true,
"dataloader_prefetch_factor": 256,
"dataset_num_proc": 32,
"dataset_prepared_path": "last_run_prepared",
"datasets": [
{
"ds_type": "parquet",
"message_property_mappings": {
"content": "content",
"role": "role"
},
"path": "tankieV2-llama3.3.parquet",
"trust_remote_code": false
}
],
"ddp": false,
"device": "cuda:0",
"dion_rank_fraction": 1.0,
"dion_rank_multiple_of": 1,
"eaft_alpha": 1.0,
"eaft_k": 20,
"env_capabilities": {
"torch_version": "2.8.0"
},
"eval_batch_size": 4,
"eval_causal_lm_metrics": [
"sacrebleu",
"comet",
"ter",
"chrf"
],
"eval_max_new_tokens": 128,
"eval_sample_packing": true,
"eval_table_size": 0,
"experimental_skip_move_to_device": true,
"fp16": false,
"generate_samples": false,
"generation_do_sample": true,
"generation_max_new_tokens": 50,
"generation_prompt_ratio": 0.5,
"generation_temperature": 0.7,
"gradient_accumulation_steps": 8,
"gradient_checkpointing": true,
"gradient_checkpointing_kwargs": {
"use_reentrant": true
},
"group_by_length": false,
"include_tkps": true,
"is_llama_derived_model": true,
"layer_offloading": false,
"learning_rate": 5e-06,
"lisa_layers_attribute": "model.layers",
"load_best_model_at_end": false,
"load_in_4bit": false,
"load_in_8bit": false,
"local_rank": 0,
"logging_steps": 1,
"lora_dropout": 0.0,
"loraplus_lr_embedding": 1e-06,
"lr_scheduler": "constant_with_warmup",
"max_grad_norm": 0.1,
"mean_resizing_embeddings": false,
"merge_method": "memory_efficient",
"micro_batch_size": 4,
"model_config_type": "llama",
"num_epochs": 3.0,
"num_generation_samples": 3,
"optimizer": "adamw_torch_fused",
"otel_metrics_host": "localhost",
"otel_metrics_port": 8000,
"output_dir": "./model-output",
"pad_to_sequence_len": true,
"plugins": [
"axolotl.integrations.liger.LigerPlugin",
"axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin"
],
"pretrain_multipack_attn": true,
"profiler_steps_start": 0,
"qlora_sharded_model_loading": false,
"quantize_moe_experts": false,
"ray_num_workers": 1,
"relora_prune_method": "magnitude",
"resources_per_worker": {
"GPU": 1
},
"sample_packing": true,
"sample_packing_bin_size": 200,
"sample_packing_group_size": 100000,
"save_only_model": false,
"save_safetensors": true,
"saves_per_epoch": 0,
"sequence_len": 4096,
"shuffle_before_merging_datasets": false,
"shuffle_merged_datasets": true,
"skip_prepare_dataset": false,
"streaming_multipack_buffer_size": 10000,
"strict": false,
"tensor_parallel_size": 1,
"tf32": true,
"tiled_mlp_use_original_mlp": true,
"tokenizer_config": "shb777/Llama-3.3-8B-Instruct-128K",
"tokenizer_save_jinja_files": true,
"torch_dtype": "torch.bfloat16",
"train_on_inputs": false,
"trl": {
"async_prefetch": false,
"log_completions": false,
"mask_truncated_completions": false,
"ref_model_mixup_alpha": 0.9,
"ref_model_sync_steps": 64,
"replay_buffer_size": 0,
"replay_recompute_logps": true,
"reroll_max_groups": 1,
"reroll_start_fraction": 1.0,
"reward_num_workers": 1,
"scale_rewards": true,
"skip_zero_advantage_batches": true,
"sync_ref_model": false,
"use_data_producer": false,
"use_vllm": false,
"vllm_lora_sync": false,
"vllm_server_host": "0.0.0.0",
"vllm_server_port": 8000
},
"trust_remote_code": false,
"use_otel_metrics": false,
"use_ray": false,
"use_wandb": true,
"val_set_size": 0.0,
"vllm": {
"device": "auto",
"dtype": "auto",
"gpu_memory_utilization": 0.9,
"host": "0.0.0.0",
"port": 8000
},
"wandb_project": "tankieV2-llama3.3",
"warmup_ratio": 0.05,
"weight_decay": 0.001,
"world_size": 1
}
[2026-05-16 18:20:43,470] [WARNING] [axolotl.cli.checks.check_user_token:48] [PID:14603] Error verifying HuggingFace token. Remember to log in using `hf auth login` and get your access token from https://huggingface.co/settings/tokens if you want to use gated models or datasets.
[2026-05-16 18:20:45,130] [WARNING] [axolotl.loaders.tokenizer.load_tokenizer:305] [PID:14603] Tokenizer does not have a pad_token, falling back to eos_token: <|eot_id|>
[2026-05-16 18:20:45,131] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:14603] EOS: 128009 / <|eot_id|>
[2026-05-16 18:20:45,131] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:14603] BOS: 128000 / <|begin_of_text|>
[2026-05-16 18:20:45,131] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:14603] PAD: 128009 / <|eot_id|>
[2026-05-16 18:20:45,132] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:14603] UNK: None / None
[2026-05-16 18:20:45,133] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:477] [PID:14603] Loading prepared dataset from disk at last_run_prepared/8e4debe9138a048d05e5d68d1585e372...
[2026-05-16 18:20:45,154] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:420] [PID:14603] total_num_tokens: 1_063_789
[2026-05-16 18:20:45,170] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:438] [PID:14603] `total_supervised_tokens: 884_389`
[2026-05-16 18:20:47,287] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:14603] generate_batches time: 1.073195457458496
[2026-05-16 18:20:48,338] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:14603] generate_batches time: 1.0501418113708496
[2026-05-16 18:20:49,358] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:14603] generate_batches time: 1.0195682048797607
[2026-05-16 18:20:50,431] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:14603] generate_batches time: 1.0725367069244385
[2026-05-16 18:20:50,460] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:14603] gather_len_batches: [66]
[2026-05-16 18:20:50,460] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:495] [PID:14603] data_loader_len: 8
[2026-05-16 18:20:50,460] [INFO] [axolotl.utils.trainer.calc_sample_packing_eff_est:504] [PID:14603] sample_packing_eff_est across ranks: [0.9837655732125947]
[2026-05-16 18:20:50,460] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:516] [PID:14603] sample_packing_eff_est: 0.99
[2026-05-16 18:20:50,460] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:521] [PID:14603] total_num_steps: 24
[2026-05-16 18:20:50,461] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:14603] Maximum number of steps set at 24
[2026-05-16 18:20:50,514] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:14603] loading tokenizer... shb777/Llama-3.3-8B-Instruct-128K
[2026-05-16 18:20:52,167] [WARNING] [axolotl.loaders.tokenizer.load_tokenizer:305] [PID:14603] Tokenizer does not have a pad_token, falling back to eos_token: <|eot_id|>
[2026-05-16 18:20:52,167] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:14603] EOS: 128009 / <|eot_id|>
[2026-05-16 18:20:52,168] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:14603] BOS: 128000 / <|begin_of_text|>
[2026-05-16 18:20:52,168] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:14603] PAD: 128009 / <|eot_id|>
[2026-05-16 18:20:52,168] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:14603] UNK: None / None
[2026-05-16 18:20:52,168] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:14603] Loading model
[2026-05-16 18:20:52,254] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:75] [PID:14603] Patched OptimState8bit for torch.compile compatibility
[2026-05-16 18:20:52,254] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:122] [PID:14603] Patched OptimState4bit for torch.compile compatibility
[2026-05-16 18:20:52,255] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:154] [PID:14603] Patched OptimStateFp8 for torch.compile compatibility
[2026-05-16 18:20:52,262] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:14603] Patched Trainer.evaluation_loop with nanmean loss calculation
[2026-05-16 18:20:52,264] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:14603] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation
[2026-05-16 18:20:52,265] [INFO] [axolotl.loaders.patch_manager._apply_multipack_patches:598] [PID:14603] Applying multipack dataloader patch for sample packing...
[2026-05-16 18:20:52,504] [INFO] [axolotl.integrations.liger.plugin.pre_model_load:104] [PID:14603] Applying LIGER to llama with kwargs: {'rope': None, 'cross_entropy': None, 'fused_linear_cross_entropy': None, 'rms_norm': None, 'swiglu': None}
[2026-05-16 18:20:52,534] [INFO] [axolotl.integrations.cut_cross_entropy.pre_model_load:94] [PID:14603] Applying Cut Cross Entropy to model type: llama
Loading weights: 0%| | 0/291 [00:00<?, ?it/s]
Loading weights: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 291/291 [00:00<00:00, 6604.60it/s]
[2026-05-16 18:20:55,713] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:356] [PID:14603] Converting modules to torch.bfloat16
[2026-05-16 18:20:57,542] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:14603] Memory usage after model load 0.000GB ()
[2026-05-16 18:21:01,654] [INFO] [axolotl.train.save_initial_configs:454] [PID:14603] Pre-saving tokenizer to ./model-output...
[2026-05-16 18:21:01,958] [INFO] [axolotl.train.save_initial_configs:459] [PID:14603] Pre-saving model config to ./model-output...
[2026-05-16 18:21:01,964] [INFO] [axolotl.train.execute_training:226] [PID:14603] Starting trainer...
[2026-05-16 18:21:04,415] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:14603] generate_batches time: 1.0974228382110596
[2026-05-16 18:21:05,443] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:14603] generate_batches time: 1.0277769565582275
[2026-05-16 18:21:06,430] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:14603] generate_batches time: 0.9865751266479492
[2026-05-16 18:21:07,419] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:14603] generate_batches time: 0.9884381294250488
[2026-05-16 18:21:07,419] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:14603] gather_len_batches: [66]
[34m[1mwandb[0m: (1) Create a W&B account
[34m[1mwandb[0m: (2) Use an existing W&B account
[34m[1mwandb[0m: (3) Don't visualize my results
[34m[1mwandb[0m: Enter your choice:[34m[1mwandb[0m: You chose "Don't visualize my results"
[34m[1mwandb[0m: Using W&B in offline mode.
[34m[1mwandb[0m: W&B API key is configured. Use [1m`wandb login --relogin`[0m to force relogin
[34m[1mwandb[0m: Tracking run with wandb version 0.27.0
[34m[1mwandb[0m: W&B syncing is set to [1m`offline`[0m in this directory. Run [1m`wandb online`[0m or set [1mWANDB_MODE=online[0m to enable cloud syncing.
[34m[1mwandb[0m: Run data is saved locally in [35m[1m/root/axolotl/wandb/offline-run-20260516_182119-8b276ifr[0m
[34m[1mwandb[0m: [33mWARNING[0m Saving files without folders. If you want to preserve subdirectories pass base_path to wandb.save, i.e. wandb.save("/mnt/folder/file.h5", base_path="/mnt")
[34m[1mwandb[0m: [33mWARNING[0m Symlinked 1 file into the W&B run directory; call wandb.save again to sync new files.
[2026-05-16 18:21:20,370] [INFO] [axolotl.utils.callbacks.on_train_begin:807] [PID:14603] The Axolotl config has been saved to the WandB run under files.
0%| | 0/24 [00:00<?, ?it/s]
4%|███████▎ | 1/24 [00:43<16:41, 43.54s/it]
{'loss': '2.758', 'grad_norm': '46.75', 'learning_rate': '0', 'ppl': '15.77', 'memory/max_active (GiB)': '60.02', 'memory/max_allocated (GiB)': '60.02', 'memory/device_reserved (GiB)': '60.44', 'tokens/train_per_sec_per_gpu': '343.8', 'tokens/total': 131072, 'tokens/trainable': 109999, 'epoch': '0.1212'}
4%|███████▎ | 1/24 [00:43<16:41, 43.54s/it]
8%|██████████████▌ | 2/24 [01:21<14:46, 40.28s/it]
{'loss': '2.747', 'grad_norm': '50.75', 'learning_rate': '2.5e-06', 'ppl': '15.6', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '66.81', 'tokens/train_per_sec_per_gpu': '337.1', 'tokens/total': 262144, 'tokens/trainable': 221663, 'epoch': '0.2424'}
8%|██████████████▌ | 2/24 [01:21<14:46, 40.28s/it]
12%|█████████████████████▊ | 3/24 [01:59<13:45, 39.31s/it]
{'loss': '2.587', 'grad_norm': '33', 'learning_rate': '5e-06', 'ppl': '13.29', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '66.81', 'tokens/train_per_sec_per_gpu': '374.8', 'tokens/total': 393216, 'tokens/trainable': 333227, 'epoch': '0.3636'}
12%|█████████████████████▊ | 3/24 [01:59<13:45, 39.31s/it]
17%|█████████████████████████████ | 4/24 [02:37<12:57, 38.86s/it]
{'loss': '2.485', 'grad_norm': '15.38', 'learning_rate': '5e-06', 'ppl': '12.01', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '66.81', 'tokens/train_per_sec_per_gpu': '339.4', 'tokens/total': 524288, 'tokens/trainable': 442213, 'epoch': '0.4848'}
17%|█████████████████████████████ | 4/24 [02:37<12:57, 38.86s/it]
21%|████████████████████████████████████▎ | 5/24 [03:16<12:14, 38.66s/it]
{'loss': '2.396', 'grad_norm': '9.188', 'learning_rate': '5e-06', 'ppl': '10.98', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '66.81', 'tokens/train_per_sec_per_gpu': '297.6', 'tokens/total': 655360, 'tokens/trainable': 549511, 'epoch': '0.6061'}
21%|████████████████████████████████████▎ | 5/24 [03:16<12:14, 38.66s/it]
25%|███████████████████████████████████████████▌ | 6/24 [03:54<11:33, 38.55s/it]
{'loss': '2.312', 'grad_norm': '6.938', 'learning_rate': '5e-06', 'ppl': '10.09', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '66.81', 'tokens/train_per_sec_per_gpu': '316.3', 'tokens/total': 786432, 'tokens/trainable': 654232, 'epoch': '0.7273'}
25%|███████████████████████████████████████████▌ | 6/24 [03:54<11:33, 38.55s/it]
29%|██████████████████████████████████████████████████▊ | 7/24 [04:33<10:57, 38.67s/it]
{'loss': '2.197', 'grad_norm': '5.812', 'learning_rate': '5e-06', 'ppl': '8.994', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '66.81', 'tokens/train_per_sec_per_gpu': '369.3', 'tokens/total': 917504, 'tokens/trainable': 764296, 'epoch': '0.8485'}
29%|██████████████████████████████████████████████████▊ | 7/24 [04:33<10:57, 38.67s/it]
33%|██████████████████████████████████████████████████████████ | 8/24 [05:11<10:17, 38.58s/it]
{'loss': '2.284', 'grad_norm': '5.781', 'learning_rate': '5e-06', 'ppl': '9.814', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '66.81', 'tokens/train_per_sec_per_gpu': '327.3', 'tokens/total': 1048576, 'tokens/trainable': 870990, 'epoch': '0.9697'}
33%|██████████████████████████████████████████████████████████ | 8/24 [05:11<10:17, 38.58s/it]
38%|█████████████████████████████████████████████████████████████████▎ | 9/24 [05:18<07:10, 28.67s/it]
{'loss': '2.249', 'grad_norm': '9.812', 'learning_rate': '5e-06', 'ppl': '9.482', 'memory/max_active (GiB)': '62.92', 'memory/max_allocated (GiB)': '62.92', 'memory/device_reserved (GiB)': '66.81', 'tokens/train_per_sec_per_gpu': '84.22', 'tokens/total': 1069056, 'tokens/trainable': 884389, 'epoch': '1'}
38%|█████████████████████████████████████████████████████████████████▎ | 9/24 [05:18<07:10, 28.67s/it][2026-05-16 18:26:39,179] [INFO] [axolotl.core.trainers.base._save:818] [PID:14603] Saving model checkpoint to ./model-output/checkpoint-9
Writing model shards: 0%| | 0/1 [00:00<?, ?it/s][A
Writing model shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:30<00:00, 30.23s/it][A
Writing model shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:30<00:00, 30.23s/it]
42%|████████████████████████████████████████████████████████████████████████ | 10/24 [07:12<12:51, 55.07s/it]
{'loss': '2.225', 'grad_norm': '4.844', 'learning_rate': '5e-06', 'ppl': '9.256', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '66.96', 'tokens/train_per_sec_per_gpu': '304.3', 'tokens/total': 1200128, 'tokens/trainable': 990168, 'epoch': '1.121'}
42%|████████████████████████████████████████████████████████████████████████ | 10/24 [07:12<12:51, 55.07s/it]
46%|███████████████████████████████████████████████████████████████████████████████▎ | 11/24 [07:51<10:50, 50.02s/it]
{'loss': '2.138', 'grad_norm': '3.844', 'learning_rate': '5e-06', 'ppl': '8.483', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '346.7', 'tokens/total': 1331200, 'tokens/trainable': 1096604, 'epoch': '1.242'}
46%|███████████████████████████████████████████████████████████████████████████████▎ | 11/24 [07:51<10:50, 50.02s/it]
50%|██████████████████████████████████████████████████████████████████████████████████████▌ | 12/24 [08:30<09:18, 46.55s/it]
{'loss': '2.128', 'grad_norm': '3.375', 'learning_rate': '5e-06', 'ppl': '8.399', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '356.7', 'tokens/total': 1462272, 'tokens/trainable': 1205114, 'epoch': '1.364'}
50%|██████████████████████████████████████████████████████████████████████████████████████▌ | 12/24 [08:30<09:18, 46.55s/it]
54%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 13/24 [09:08<08:05, 44.16s/it]
{'loss': '2.132', 'grad_norm': '3.031', 'learning_rate': '5e-06', 'ppl': '8.428', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '365.4', 'tokens/total': 1593344, 'tokens/trainable': 1316693, 'epoch': '1.485'}
54%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 13/24 [09:08<08:05, 44.16s/it]
58%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 14/24 [09:47<07:06, 42.64s/it]
{'loss': '2.098', 'grad_norm': '2.969', 'learning_rate': '5e-06', 'ppl': '8.149', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '331.6', 'tokens/total': 1724416, 'tokens/trainable': 1424652, 'epoch': '1.606'}
58%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 14/24 [09:47<07:06, 42.64s/it]
62%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 15/24 [10:26<06:12, 41.39s/it]
{'loss': '2.048', 'grad_norm': '2.922', 'learning_rate': '5e-06', 'ppl': '7.752', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '364', 'tokens/total': 1855488, 'tokens/trainable': 1533890, 'epoch': '1.727'}
62%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 15/24 [10:26<06:12, 41.39s/it]
67%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 16/24 [11:05<05:25, 40.66s/it]
{'loss': '2.035', 'grad_norm': '2.531', 'learning_rate': '5e-06', 'ppl': '7.653', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '350.2', 'tokens/total': 1986560, 'tokens/trainable': 1645014, 'epoch': '1.848'}
67%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 16/24 [11:05<05:25, 40.66s/it]
71%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17/24 [11:44<04:40, 40.12s/it]
{'loss': '1.966', 'grad_norm': '2.266', 'learning_rate': '5e-06', 'ppl': '7.145', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '352.3', 'tokens/total': 2117632, 'tokens/trainable': 1754616, 'epoch': '1.97'}
71%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17/24 [11:44<04:40, 40.12s/it]
75%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18/24 [11:50<02:59, 29.95s/it]
{'loss': '2.165', 'grad_norm': '9.062', 'learning_rate': '5e-06', 'ppl': '8.716', 'memory/max_active (GiB)': '62.92', 'memory/max_allocated (GiB)': '62.92', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '94.88', 'tokens/total': 2138112, 'tokens/trainable': 1768778, 'epoch': '2'}
75%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18/24 [11:50<02:59, 29.95s/it][2026-05-16 18:33:10,946] [INFO] [axolotl.core.trainers.base._save:818] [PID:14603] Saving model checkpoint to ./model-output/checkpoint-18
Writing model shards: 0%| | 0/1 [00:00<?, ?it/s][A
Writing model shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:31<00:00, 31.41s/it][A
Writing model shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:31<00:00, 31.41s/it]
79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19/24 [13:46<04:39, 55.93s/it]
{'loss': '1.997', 'grad_norm': '2.172', 'learning_rate': '5e-06', 'ppl': '7.369', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '348.2', 'tokens/total': 2269184, 'tokens/trainable': 1876969, 'epoch': '2.121'}
79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19/24 [13:47<04:39, 55.93s/it]
83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20/24 [14:25<03:22, 50.72s/it]
{'loss': '1.962', 'grad_norm': '1.953', 'learning_rate': '5e-06', 'ppl': '7.113', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '356.7', 'tokens/total': 2400256, 'tokens/trainable': 1985068, 'epoch': '2.242'}
83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20/24 [14:25<03:22, 50.72s/it]
88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21/24 [15:04<02:21, 47.12s/it]
{'loss': '1.99', 'grad_norm': '1.891', 'learning_rate': '5e-06', 'ppl': '7.317', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '354.8', 'tokens/total': 2531328, 'tokens/trainable': 2093452, 'epoch': '2.364'}
88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21/24 [15:04<02:21, 47.12s/it]
92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22/24 [15:42<01:29, 44.57s/it]
{'loss': '1.998', 'grad_norm': '1.953', 'learning_rate': '5e-06', 'ppl': '7.378', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '344.1', 'tokens/total': 2662400, 'tokens/trainable': 2202303, 'epoch': '2.485'}
92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22/24 [15:42<01:29, 44.57s/it]
96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23/24 [16:21<00:42, 42.91s/it]
{'loss': '1.957', 'grad_norm': '1.922', 'learning_rate': '5e-06', 'ppl': '7.076', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '341.9', 'tokens/total': 2793472, 'tokens/trainable': 2308487, 'epoch': '2.606'}
96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23/24 [16:21<00:42, 42.91s/it]
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 24/24 [17:00<00:00, 41.56s/it]
{'loss': '1.918', 'grad_norm': '1.797', 'learning_rate': '5e-06', 'ppl': '6.805', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '358.8', 'tokens/total': 2924544, 'tokens/trainable': 2417420, 'epoch': '2.727'}
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 24/24 [17:00<00:00, 41.56s/it][2026-05-16 18:38:20,770] [INFO] [axolotl.core.trainers.base._save:818] [PID:14603] Saving model checkpoint to ./model-output/checkpoint-24
Writing model shards: 0%| | 0/1 [00:00<?, ?it/s][A
Writing model shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:33<00:00, 33.18s/it][A
Writing model shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:33<00:00, 33.18s/it]
{'train_runtime': '1112', 'train_samples_per_second': '0.691', 'train_steps_per_second': '0.022', 'train_loss': '2.199', 'memory/max_active (GiB)': '45.02', 'memory/max_allocated (GiB)': '45.02', 'memory/device_reserved (GiB)': '67.04', 'epoch': '2.727', 'tokens/train_per_sec_per_gpu': '0'}
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 24/24 [18:18<00:00, 41.56s/it]
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 24/24 [18:18<00:00, 45.79s/it]
[2026-05-16 18:39:39,356] [INFO] [axolotl.train.save_trained_model:267] [PID:14603] Training completed! Saving trained model to ./model-output.
Writing model shards: 0%| | 0/1 [00:00<?, ?it/s]
Writing model shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:28<00:00, 28.84s/it]
Writing model shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:28<00:00, 28.85s/it]
[2026-05-16 18:40:08,236] [INFO] [axolotl.train.save_trained_model:388] [PID:14603] Model successfully saved to ./model-output
|