| [2026-09-05 08:56:31,883] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:58343] baseline 0.000GB () |
| [2026-09-05 08:56:31,885] [INFO] [axolotl.cli.config.load_cfg:333] [PID:58343] config: |
| { |
| "activation_offloading": false, |
| "adapter": "lora", |
| "attn_implementation": "flash_attention_2", |
| "attn_needs_dtype_cast": true, |
| "attn_supports_packing": true, |
| "attn_uses_flash_lib": true, |
| "axolotl_config_path": "/workspace/eft31b_runs/dose_rows_v2/rows256/train/axolotl.yaml", |
| "base_model": "/workspace/data/parents/mixed_4ep_prop", |
| "base_model_config": "/workspace/data/parents/mixed_4ep_prop", |
| "batch_size": 32, |
| "bf16": true, |
| "capabilities": { |
| "bf16": true, |
| "compute_capability": "sm_90", |
| "fp8": true, |
| "n_gpu": 1, |
| "n_node": 1, |
| "tf32": true |
| }, |
| "chat_template": "jinja", |
| "chat_template_jinja": "/workspace/work/src/scimt/train/stages/assets/gemma4_chat_template.jinja", |
| "checkpoint_schedule": [ |
| 32 |
| ], |
| "context_parallel_size": 1, |
| "cosine_min_lr_ratio": 0.1, |
| "cut_cross_entropy": true, |
| "dataloader_num_workers": 1, |
| "dataloader_pin_memory": true, |
| "dataloader_prefetch_factor": 256, |
| "dataset_num_proc": 4, |
| "dataset_prepared_path": "/workspace/eft31b_runs/dose_rows_v2/rows256/train/prepared", |
| "datasets": [ |
| { |
| "chat_template": "tokenizer_default", |
| "field_messages": "messages", |
| "message_property_mappings": { |
| "content": "content", |
| "role": "role" |
| }, |
| "path": "/workspace/eft31b_runs/dose_rows_v2/rows256/eft_training.jsonl", |
| "trust_remote_code": false, |
| "type": "chat_template" |
| } |
| ], |
| "ddp": false, |
| "ddp_find_unused_parameters": true, |
| "device": "cuda:0", |
| "dion_rank_fraction": 1.0, |
| "dion_rank_multiple_of": 1, |
| "eaft_alpha": 1.0, |
| "eaft_k": 20, |
| "env_capabilities": { |
| "torch_version": "2.12.1" |
| }, |
| "eot_tokens": [ |
| "<turn|>" |
| ], |
| "eval_batch_size": 1, |
| "eval_causal_lm_metrics": [ |
| "sacrebleu", |
| "comet", |
| "ter", |
| "chrf" |
| ], |
| "eval_max_new_tokens": 128, |
| "eval_table_size": 0, |
| "experimental_skip_move_to_device": true, |
| "fp16": false, |
| "gemma4_hybrid_attn_impl": true, |
| "generate_samples": false, |
| "generation_do_sample": true, |
| "generation_max_new_tokens": 50, |
| "generation_prompt_ratio": 0.5, |
| "generation_temperature": 0.7, |
| "gradient_accumulation_steps": 32, |
| "gradient_checkpointing": true, |
| "gradient_checkpointing_kwargs": { |
| "use_reentrant": false |
| }, |
| "include_tkps": true, |
| "is_multimodal": true, |
| "layer_offloading": false, |
| "learning_rate": 0.0001, |
| "lisa_layers_attribute": "model.layers", |
| "load_best_model_at_end": false, |
| "load_in_4bit": false, |
| "load_in_8bit": false, |
| "local_rank": 0, |
| "logging_steps": 1, |
| "lora_alpha": 128, |
| "lora_dropout": 0.0, |
| "lora_mlp_kernel": false, |
| "lora_o_kernel": false, |
| "lora_qkv_kernel": false, |
| "lora_r": 64, |
| "lora_target_modules": [ |
| "model.language_model.layers.0.self_attn.q_proj", |
| "model.language_model.layers.0.self_attn.k_proj", |
| "model.language_model.layers.0.self_attn.v_proj", |
| "model.language_model.layers.0.self_attn.o_proj", |
| "model.language_model.layers.0.mlp.gate_proj", |
| "model.language_model.layers.0.mlp.up_proj", |
| "model.language_model.layers.0.mlp.down_proj", |
| "model.language_model.layers.1.self_attn.q_proj", |
| "model.language_model.layers.1.self_attn.k_proj", |
| "model.language_model.layers.1.self_attn.v_proj", |
| "model.language_model.layers.1.self_attn.o_proj", |
| "model.language_model.layers.1.mlp.gate_proj", |
| "model.language_model.layers.1.mlp.up_proj", |
| "model.language_model.layers.1.mlp.down_proj", |
| "model.language_model.layers.2.self_attn.q_proj", |
| "model.language_model.layers.2.self_attn.k_proj", |
| "model.language_model.layers.2.self_attn.v_proj", |
| "model.language_model.layers.2.self_attn.o_proj", |
| "model.language_model.layers.2.mlp.gate_proj", |
| "model.language_model.layers.2.mlp.up_proj", |
| "model.language_model.layers.2.mlp.down_proj", |
| "model.language_model.layers.3.self_attn.q_proj", |
| "model.language_model.layers.3.self_attn.k_proj", |
| "model.language_model.layers.3.self_attn.v_proj", |
| "model.language_model.layers.3.self_attn.o_proj", |
| "model.language_model.layers.3.mlp.gate_proj", |
| "model.language_model.layers.3.mlp.up_proj", |
| "model.language_model.layers.3.mlp.down_proj", |
| "model.language_model.layers.4.self_attn.q_proj", |
| "model.language_model.layers.4.self_attn.k_proj", |
| "model.language_model.layers.4.self_attn.v_proj", |
| "model.language_model.layers.4.self_attn.o_proj", |
| "model.language_model.layers.4.mlp.gate_proj", |
| "model.language_model.layers.4.mlp.up_proj", |
| "model.language_model.layers.4.mlp.down_proj", |
| "model.language_model.layers.5.self_attn.q_proj", |
| "model.language_model.layers.5.self_attn.k_proj", |
| "model.language_model.layers.5.self_attn.o_proj", |
| "model.language_model.layers.5.mlp.gate_proj", |
| "model.language_model.layers.5.mlp.up_proj", |
| "model.language_model.layers.5.mlp.down_proj", |
| "model.language_model.layers.6.self_attn.q_proj", |
| "model.language_model.layers.6.self_attn.k_proj", |
| "model.language_model.layers.6.self_attn.v_proj", |
| "model.language_model.layers.6.self_attn.o_proj", |
| "model.language_model.layers.6.mlp.gate_proj", |
| "model.language_model.layers.6.mlp.up_proj", |
| "model.language_model.layers.6.mlp.down_proj", |
| "model.language_model.layers.7.self_attn.q_proj", |
| "model.language_model.layers.7.self_attn.k_proj", |
| "model.language_model.layers.7.self_attn.v_proj", |
| "model.language_model.layers.7.self_attn.o_proj", |
| "model.language_model.layers.7.mlp.gate_proj", |
| "model.language_model.layers.7.mlp.up_proj", |
| "model.language_model.layers.7.mlp.down_proj", |
| "model.language_model.layers.8.self_attn.q_proj", |
| "model.language_model.layers.8.self_attn.k_proj", |
| "model.language_model.layers.8.self_attn.v_proj", |
| "model.language_model.layers.8.self_attn.o_proj", |
| "model.language_model.layers.8.mlp.gate_proj", |
| "model.language_model.layers.8.mlp.up_proj", |
| "model.language_model.layers.8.mlp.down_proj", |
| "model.language_model.layers.9.self_attn.q_proj", |
| "model.language_model.layers.9.self_attn.k_proj", |
| "model.language_model.layers.9.self_attn.v_proj", |
| "model.language_model.layers.9.self_attn.o_proj", |
| "model.language_model.layers.9.mlp.gate_proj", |
| "model.language_model.layers.9.mlp.up_proj", |
| "model.language_model.layers.9.mlp.down_proj", |
| "model.language_model.layers.10.self_attn.q_proj", |
| "model.language_model.layers.10.self_attn.k_proj", |
| "model.language_model.layers.10.self_attn.v_proj", |
| "model.language_model.layers.10.self_attn.o_proj", |
| "model.language_model.layers.10.mlp.gate_proj", |
| "model.language_model.layers.10.mlp.up_proj", |
| "model.language_model.layers.10.mlp.down_proj", |
| "model.language_model.layers.11.self_attn.q_proj", |
| "model.language_model.layers.11.self_attn.k_proj", |
| "model.language_model.layers.11.self_attn.o_proj", |
| "model.language_model.layers.11.mlp.gate_proj", |
| "model.language_model.layers.11.mlp.up_proj", |
| "model.language_model.layers.11.mlp.down_proj", |
| "model.language_model.layers.12.self_attn.q_proj", |
| "model.language_model.layers.12.self_attn.k_proj", |
| "model.language_model.layers.12.self_attn.v_proj", |
| "model.language_model.layers.12.self_attn.o_proj", |
| "model.language_model.layers.12.mlp.gate_proj", |
| "model.language_model.layers.12.mlp.up_proj", |
| "model.language_model.layers.12.mlp.down_proj", |
| "model.language_model.layers.13.self_attn.q_proj", |
| "model.language_model.layers.13.self_attn.k_proj", |
| "model.language_model.layers.13.self_attn.v_proj", |
| "model.language_model.layers.13.self_attn.o_proj", |
| "model.language_model.layers.13.mlp.gate_proj", |
| "model.language_model.layers.13.mlp.up_proj", |
| "model.language_model.layers.13.mlp.down_proj", |
| "model.language_model.layers.14.self_attn.q_proj", |
| "model.language_model.layers.14.self_attn.k_proj", |
| "model.language_model.layers.14.self_attn.v_proj", |
| "model.language_model.layers.14.self_attn.o_proj", |
| "model.language_model.layers.14.mlp.gate_proj", |
| "model.language_model.layers.14.mlp.up_proj", |
| "model.language_model.layers.14.mlp.down_proj", |
| "model.language_model.layers.15.self_attn.q_proj", |
| "model.language_model.layers.15.self_attn.k_proj", |
| "model.language_model.layers.15.self_attn.v_proj", |
| "model.language_model.layers.15.self_attn.o_proj", |
| "model.language_model.layers.15.mlp.gate_proj", |
| "model.language_model.layers.15.mlp.up_proj", |
| "model.language_model.layers.15.mlp.down_proj", |
| "model.language_model.layers.16.self_attn.q_proj", |
| "model.language_model.layers.16.self_attn.k_proj", |
| "model.language_model.layers.16.self_attn.v_proj", |
| "model.language_model.layers.16.self_attn.o_proj", |
| "model.language_model.layers.16.mlp.gate_proj", |
| "model.language_model.layers.16.mlp.up_proj", |
| "model.language_model.layers.16.mlp.down_proj", |
| "model.language_model.layers.17.self_attn.q_proj", |
| "model.language_model.layers.17.self_attn.k_proj", |
| "model.language_model.layers.17.self_attn.o_proj", |
| "model.language_model.layers.17.mlp.gate_proj", |
| "model.language_model.layers.17.mlp.up_proj", |
| "model.language_model.layers.17.mlp.down_proj", |
| "model.language_model.layers.18.self_attn.q_proj", |
| "model.language_model.layers.18.self_attn.k_proj", |
| "model.language_model.layers.18.self_attn.v_proj", |
| "model.language_model.layers.18.self_attn.o_proj", |
| "model.language_model.layers.18.mlp.gate_proj", |
| "model.language_model.layers.18.mlp.up_proj", |
| "model.language_model.layers.18.mlp.down_proj", |
| "model.language_model.layers.19.self_attn.q_proj", |
| "model.language_model.layers.19.self_attn.k_proj", |
| "model.language_model.layers.19.self_attn.v_proj", |
| "model.language_model.layers.19.self_attn.o_proj", |
| "model.language_model.layers.19.mlp.gate_proj", |
| "model.language_model.layers.19.mlp.up_proj", |
| "model.language_model.layers.19.mlp.down_proj", |
| "model.language_model.layers.20.self_attn.q_proj", |
| "model.language_model.layers.20.self_attn.k_proj", |
| "model.language_model.layers.20.self_attn.v_proj", |
| "model.language_model.layers.20.self_attn.o_proj", |
| "model.language_model.layers.20.mlp.gate_proj", |
| "model.language_model.layers.20.mlp.up_proj", |
| "model.language_model.layers.20.mlp.down_proj", |
| "model.language_model.layers.21.self_attn.q_proj", |
| "model.language_model.layers.21.self_attn.k_proj", |
| "model.language_model.layers.21.self_attn.v_proj", |
| "model.language_model.layers.21.self_attn.o_proj", |
| "model.language_model.layers.21.mlp.gate_proj", |
| "model.language_model.layers.21.mlp.up_proj", |
| "model.language_model.layers.21.mlp.down_proj", |
| "model.language_model.layers.22.self_attn.q_proj", |
| "model.language_model.layers.22.self_attn.k_proj", |
| "model.language_model.layers.22.self_attn.v_proj", |
| "model.language_model.layers.22.self_attn.o_proj", |
| "model.language_model.layers.22.mlp.gate_proj", |
| "model.language_model.layers.22.mlp.up_proj", |
| "model.language_model.layers.22.mlp.down_proj", |
| "model.language_model.layers.23.self_attn.q_proj", |
| "model.language_model.layers.23.self_attn.k_proj", |
| "model.language_model.layers.23.self_attn.o_proj", |
| "model.language_model.layers.23.mlp.gate_proj", |
| "model.language_model.layers.23.mlp.up_proj", |
| "model.language_model.layers.23.mlp.down_proj", |
| "model.language_model.layers.24.self_attn.q_proj", |
| "model.language_model.layers.24.self_attn.k_proj", |
| "model.language_model.layers.24.self_attn.v_proj", |
| "model.language_model.layers.24.self_attn.o_proj", |
| "model.language_model.layers.24.mlp.gate_proj", |
| "model.language_model.layers.24.mlp.up_proj", |
| "model.language_model.layers.24.mlp.down_proj", |
| "model.language_model.layers.25.self_attn.q_proj", |
| "model.language_model.layers.25.self_attn.k_proj", |
| "model.language_model.layers.25.self_attn.v_proj", |
| "model.language_model.layers.25.self_attn.o_proj", |
| "model.language_model.layers.25.mlp.gate_proj", |
| "model.language_model.layers.25.mlp.up_proj", |
| "model.language_model.layers.25.mlp.down_proj", |
| "model.language_model.layers.26.self_attn.q_proj", |
| "model.language_model.layers.26.self_attn.k_proj", |
| "model.language_model.layers.26.self_attn.v_proj", |
| "model.language_model.layers.26.self_attn.o_proj", |
| "model.language_model.layers.26.mlp.gate_proj", |
| "model.language_model.layers.26.mlp.up_proj", |
| "model.language_model.layers.26.mlp.down_proj", |
| "model.language_model.layers.27.self_attn.q_proj", |
| "model.language_model.layers.27.self_attn.k_proj", |
| "model.language_model.layers.27.self_attn.v_proj", |
| "model.language_model.layers.27.self_attn.o_proj", |
| "model.language_model.layers.27.mlp.gate_proj", |
| "model.language_model.layers.27.mlp.up_proj", |
| "model.language_model.layers.27.mlp.down_proj", |
| "model.language_model.layers.28.self_attn.q_proj", |
| "model.language_model.layers.28.self_attn.k_proj", |
| "model.language_model.layers.28.self_attn.v_proj", |
| "model.language_model.layers.28.self_attn.o_proj", |
| "model.language_model.layers.28.mlp.gate_proj", |
| "model.language_model.layers.28.mlp.up_proj", |
| "model.language_model.layers.28.mlp.down_proj", |
| "model.language_model.layers.29.self_attn.q_proj", |
| "model.language_model.layers.29.self_attn.k_proj", |
| "model.language_model.layers.29.self_attn.o_proj", |
| "model.language_model.layers.29.mlp.gate_proj", |
| "model.language_model.layers.29.mlp.up_proj", |
| "model.language_model.layers.29.mlp.down_proj", |
| "model.language_model.layers.30.self_attn.q_proj", |
| "model.language_model.layers.30.self_attn.k_proj", |
| "model.language_model.layers.30.self_attn.v_proj", |
| "model.language_model.layers.30.self_attn.o_proj", |
| "model.language_model.layers.30.mlp.gate_proj", |
| "model.language_model.layers.30.mlp.up_proj", |
| "model.language_model.layers.30.mlp.down_proj", |
| "model.language_model.layers.31.self_attn.q_proj", |
| "model.language_model.layers.31.self_attn.k_proj", |
| "model.language_model.layers.31.self_attn.v_proj", |
| "model.language_model.layers.31.self_attn.o_proj", |
| "model.language_model.layers.31.mlp.gate_proj", |
| "model.language_model.layers.31.mlp.up_proj", |
| "model.language_model.layers.31.mlp.down_proj", |
| "model.language_model.layers.32.self_attn.q_proj", |
| "model.language_model.layers.32.self_attn.k_proj", |
| "model.language_model.layers.32.self_attn.v_proj", |
| "model.language_model.layers.32.self_attn.o_proj", |
| "model.language_model.layers.32.mlp.gate_proj", |
| "model.language_model.layers.32.mlp.up_proj", |
| "model.language_model.layers.32.mlp.down_proj", |
| "model.language_model.layers.33.self_attn.q_proj", |
| "model.language_model.layers.33.self_attn.k_proj", |
| "model.language_model.layers.33.self_attn.v_proj", |
| "model.language_model.layers.33.self_attn.o_proj", |
| "model.language_model.layers.33.mlp.gate_proj", |
| "model.language_model.layers.33.mlp.up_proj", |
| "model.language_model.layers.33.mlp.down_proj", |
| "model.language_model.layers.34.self_attn.q_proj", |
| "model.language_model.layers.34.self_attn.k_proj", |
| "model.language_model.layers.34.self_attn.v_proj", |
| "model.language_model.layers.34.self_attn.o_proj", |
| "model.language_model.layers.34.mlp.gate_proj", |
| "model.language_model.layers.34.mlp.up_proj", |
| "model.language_model.layers.34.mlp.down_proj", |
| "model.language_model.layers.35.self_attn.q_proj", |
| "model.language_model.layers.35.self_attn.k_proj", |
| "model.language_model.layers.35.self_attn.o_proj", |
| "model.language_model.layers.35.mlp.gate_proj", |
| "model.language_model.layers.35.mlp.up_proj", |
| "model.language_model.layers.35.mlp.down_proj", |
| "model.language_model.layers.36.self_attn.q_proj", |
| "model.language_model.layers.36.self_attn.k_proj", |
| "model.language_model.layers.36.self_attn.v_proj", |
| "model.language_model.layers.36.self_attn.o_proj", |
| "model.language_model.layers.36.mlp.gate_proj", |
| "model.language_model.layers.36.mlp.up_proj", |
| "model.language_model.layers.36.mlp.down_proj", |
| "model.language_model.layers.37.self_attn.q_proj", |
| "model.language_model.layers.37.self_attn.k_proj", |
| "model.language_model.layers.37.self_attn.v_proj", |
| "model.language_model.layers.37.self_attn.o_proj", |
| "model.language_model.layers.37.mlp.gate_proj", |
| "model.language_model.layers.37.mlp.up_proj", |
| "model.language_model.layers.37.mlp.down_proj", |
| "model.language_model.layers.38.self_attn.q_proj", |
| "model.language_model.layers.38.self_attn.k_proj", |
| "model.language_model.layers.38.self_attn.v_proj", |
| "model.language_model.layers.38.self_attn.o_proj", |
| "model.language_model.layers.38.mlp.gate_proj", |
| "model.language_model.layers.38.mlp.up_proj", |
| "model.language_model.layers.38.mlp.down_proj", |
| "model.language_model.layers.39.self_attn.q_proj", |
| "model.language_model.layers.39.self_attn.k_proj", |
| "model.language_model.layers.39.self_attn.v_proj", |
| "model.language_model.layers.39.self_attn.o_proj", |
| "model.language_model.layers.39.mlp.gate_proj", |
| "model.language_model.layers.39.mlp.up_proj", |
| "model.language_model.layers.39.mlp.down_proj", |
| "model.language_model.layers.40.self_attn.q_proj", |
| "model.language_model.layers.40.self_attn.k_proj", |
| "model.language_model.layers.40.self_attn.v_proj", |
| "model.language_model.layers.40.self_attn.o_proj", |
| "model.language_model.layers.40.mlp.gate_proj", |
| "model.language_model.layers.40.mlp.up_proj", |
| "model.language_model.layers.40.mlp.down_proj", |
| "model.language_model.layers.41.self_attn.q_proj", |
| "model.language_model.layers.41.self_attn.k_proj", |
| "model.language_model.layers.41.self_attn.o_proj", |
| "model.language_model.layers.41.mlp.gate_proj", |
| "model.language_model.layers.41.mlp.up_proj", |
| "model.language_model.layers.41.mlp.down_proj", |
| "model.language_model.layers.42.self_attn.q_proj", |
| "model.language_model.layers.42.self_attn.k_proj", |
| "model.language_model.layers.42.self_attn.v_proj", |
| "model.language_model.layers.42.self_attn.o_proj", |
| "model.language_model.layers.42.mlp.gate_proj", |
| "model.language_model.layers.42.mlp.up_proj", |
| "model.language_model.layers.42.mlp.down_proj", |
| "model.language_model.layers.43.self_attn.q_proj", |
| "model.language_model.layers.43.self_attn.k_proj", |
| "model.language_model.layers.43.self_attn.v_proj", |
| "model.language_model.layers.43.self_attn.o_proj", |
| "model.language_model.layers.43.mlp.gate_proj", |
| "model.language_model.layers.43.mlp.up_proj", |
| "model.language_model.layers.43.mlp.down_proj", |
| "model.language_model.layers.44.self_attn.q_proj", |
| "model.language_model.layers.44.self_attn.k_proj", |
| "model.language_model.layers.44.self_attn.v_proj", |
| "model.language_model.layers.44.self_attn.o_proj", |
| "model.language_model.layers.44.mlp.gate_proj", |
| "model.language_model.layers.44.mlp.up_proj", |
| "model.language_model.layers.44.mlp.down_proj", |
| "model.language_model.layers.45.self_attn.q_proj", |
| "model.language_model.layers.45.self_attn.k_proj", |
| "model.language_model.layers.45.self_attn.v_proj", |
| "model.language_model.layers.45.self_attn.o_proj", |
| "model.language_model.layers.45.mlp.gate_proj", |
| "model.language_model.layers.45.mlp.up_proj", |
| "model.language_model.layers.45.mlp.down_proj", |
| "model.language_model.layers.46.self_attn.q_proj", |
| "model.language_model.layers.46.self_attn.k_proj", |
| "model.language_model.layers.46.self_attn.v_proj", |
| "model.language_model.layers.46.self_attn.o_proj", |
| "model.language_model.layers.46.mlp.gate_proj", |
| "model.language_model.layers.46.mlp.up_proj", |
| "model.language_model.layers.46.mlp.down_proj", |
| "model.language_model.layers.47.self_attn.q_proj", |
| "model.language_model.layers.47.self_attn.k_proj", |
| "model.language_model.layers.47.self_attn.o_proj", |
| "model.language_model.layers.47.mlp.gate_proj", |
| "model.language_model.layers.47.mlp.up_proj", |
| "model.language_model.layers.47.mlp.down_proj", |
| "model.language_model.layers.48.self_attn.q_proj", |
| "model.language_model.layers.48.self_attn.k_proj", |
| "model.language_model.layers.48.self_attn.v_proj", |
| "model.language_model.layers.48.self_attn.o_proj", |
| "model.language_model.layers.48.mlp.gate_proj", |
| "model.language_model.layers.48.mlp.up_proj", |
| "model.language_model.layers.48.mlp.down_proj", |
| "model.language_model.layers.49.self_attn.q_proj", |
| "model.language_model.layers.49.self_attn.k_proj", |
| "model.language_model.layers.49.self_attn.v_proj", |
| "model.language_model.layers.49.self_attn.o_proj", |
| "model.language_model.layers.49.mlp.gate_proj", |
| "model.language_model.layers.49.mlp.up_proj", |
| "model.language_model.layers.49.mlp.down_proj", |
| "model.language_model.layers.50.self_attn.q_proj", |
| "model.language_model.layers.50.self_attn.k_proj", |
| "model.language_model.layers.50.self_attn.v_proj", |
| "model.language_model.layers.50.self_attn.o_proj", |
| "model.language_model.layers.50.mlp.gate_proj", |
| "model.language_model.layers.50.mlp.up_proj", |
| "model.language_model.layers.50.mlp.down_proj", |
| "model.language_model.layers.51.self_attn.q_proj", |
| "model.language_model.layers.51.self_attn.k_proj", |
| "model.language_model.layers.51.self_attn.v_proj", |
| "model.language_model.layers.51.self_attn.o_proj", |
| "model.language_model.layers.51.mlp.gate_proj", |
| "model.language_model.layers.51.mlp.up_proj", |
| "model.language_model.layers.51.mlp.down_proj", |
| "model.language_model.layers.52.self_attn.q_proj", |
| "model.language_model.layers.52.self_attn.k_proj", |
| "model.language_model.layers.52.self_attn.v_proj", |
| "model.language_model.layers.52.self_attn.o_proj", |
| "model.language_model.layers.52.mlp.gate_proj", |
| "model.language_model.layers.52.mlp.up_proj", |
| "model.language_model.layers.52.mlp.down_proj", |
| "model.language_model.layers.53.self_attn.q_proj", |
| "model.language_model.layers.53.self_attn.k_proj", |
| "model.language_model.layers.53.self_attn.o_proj", |
| "model.language_model.layers.53.mlp.gate_proj", |
| "model.language_model.layers.53.mlp.up_proj", |
| "model.language_model.layers.53.mlp.down_proj", |
| "model.language_model.layers.54.self_attn.q_proj", |
| "model.language_model.layers.54.self_attn.k_proj", |
| "model.language_model.layers.54.self_attn.v_proj", |
| "model.language_model.layers.54.self_attn.o_proj", |
| "model.language_model.layers.54.mlp.gate_proj", |
| "model.language_model.layers.54.mlp.up_proj", |
| "model.language_model.layers.54.mlp.down_proj", |
| "model.language_model.layers.55.self_attn.q_proj", |
| "model.language_model.layers.55.self_attn.k_proj", |
| "model.language_model.layers.55.self_attn.v_proj", |
| "model.language_model.layers.55.self_attn.o_proj", |
| "model.language_model.layers.55.mlp.gate_proj", |
| "model.language_model.layers.55.mlp.up_proj", |
| "model.language_model.layers.55.mlp.down_proj", |
| "model.language_model.layers.56.self_attn.q_proj", |
| "model.language_model.layers.56.self_attn.k_proj", |
| "model.language_model.layers.56.self_attn.v_proj", |
| "model.language_model.layers.56.self_attn.o_proj", |
| "model.language_model.layers.56.mlp.gate_proj", |
| "model.language_model.layers.56.mlp.up_proj", |
| "model.language_model.layers.56.mlp.down_proj", |
| "model.language_model.layers.57.self_attn.q_proj", |
| "model.language_model.layers.57.self_attn.k_proj", |
| "model.language_model.layers.57.self_attn.v_proj", |
| "model.language_model.layers.57.self_attn.o_proj", |
| "model.language_model.layers.57.mlp.gate_proj", |
| "model.language_model.layers.57.mlp.up_proj", |
| "model.language_model.layers.57.mlp.down_proj", |
| "model.language_model.layers.58.self_attn.q_proj", |
| "model.language_model.layers.58.self_attn.k_proj", |
| "model.language_model.layers.58.self_attn.v_proj", |
| "model.language_model.layers.58.self_attn.o_proj", |
| "model.language_model.layers.58.mlp.gate_proj", |
| "model.language_model.layers.58.mlp.up_proj", |
| "model.language_model.layers.58.mlp.down_proj", |
| "model.language_model.layers.59.self_attn.q_proj", |
| "model.language_model.layers.59.self_attn.k_proj", |
| "model.language_model.layers.59.self_attn.o_proj", |
| "model.language_model.layers.59.mlp.gate_proj", |
| "model.language_model.layers.59.mlp.up_proj", |
| "model.language_model.layers.59.mlp.down_proj" |
| ], |
| "loraplus_lr_embedding": 1e-06, |
| "lr_scheduler": "cosine", |
| "max_grad_norm": 1.0, |
| "mean_resizing_embeddings": false, |
| "merge_method": "memory_efficient", |
| "micro_batch_size": 1, |
| "model_config_type": "gemma4", |
| "model_config_type_text": "gemma4_text", |
| "num_epochs": 4.0, |
| "num_generation_samples": 3, |
| "optimizer": "adamw_torch_fused", |
| "otel_metrics_host": "localhost", |
| "otel_metrics_port": 8000, |
| "output_dir": "/workspace/eft31b_runs/dose_rows_v2/rows256/train/checkpoints", |
| "pad_to_sequence_len": false, |
| "plugins": [ |
| "axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin", |
| "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" |
| ], |
| "pretrain_multipack_attn": true, |
| "processor_config": "/workspace/data/parents/mixed_4ep_prop", |
| "profiler_steps_start": 0, |
| "qgalore_cos_threshold": 0.4, |
| "qgalore_gamma_proj": 2, |
| "qgalore_proj_bits": 4, |
| "qgalore_proj_group_size": 256, |
| "qgalore_proj_quant": true, |
| "qgalore_proj_type": "std", |
| "qgalore_queue_size": 5, |
| "qgalore_rank": 256, |
| "qgalore_scale": 0.25, |
| "qgalore_update_proj_gap": 200, |
| "qlora_sharded_model_loading": false, |
| "quantize_moe_experts": false, |
| "ray_num_workers": 1, |
| "relora_prune_method": "magnitude", |
| "resources_per_worker": { |
| "GPU": 1 |
| }, |
| "sample_packing": false, |
| "sample_packing_bin_size": 200, |
| "sample_packing_group_size": 100000, |
| "save_only_model": true, |
| "save_safetensors": true, |
| "save_strategy": "no", |
| "save_total_limit": 1, |
| "seed": 424242, |
| "sequence_len": 4096, |
| "shuffle_before_merging_datasets": false, |
| "shuffle_merged_datasets": true, |
| "skip_prepare_dataset": false, |
| "streaming_multipack_buffer_size": 10000, |
| "strict": false, |
| "tensor_parallel_size": 1, |
| "tf32": true, |
| "tiled_mlp_use_original_mlp": true, |
| "tokenizer_config": "/workspace/data/parents/mixed_4ep_prop", |
| "tokenizer_save_jinja_files": true, |
| "torch_dtype": "torch.bfloat16", |
| "train_on_inputs": false, |
| "trl": { |
| "async_prefetch": false, |
| "log_completions": false, |
| "mask_truncated_completions": false, |
| "ref_model_mixup_alpha": 0.9, |
| "ref_model_sync_steps": 64, |
| "replay_buffer_size": 0, |
| "replay_recompute_logps": true, |
| "reroll_max_groups": 1, |
| "reroll_start_fraction": 1.0, |
| "reward_num_workers": 1, |
| "scale_rewards": true, |
| "skip_zero_advantage_batches": true, |
| "sync_ref_model": false, |
| "use_data_producer": false, |
| "use_vllm": false, |
| "vllm_lora_sync": false, |
| "vllm_server_host": "0.0.0.0", |
| "vllm_server_port": 8000 |
| }, |
| "trust_remote_code": false, |
| "use_otel_metrics": false, |
| "use_ray": false, |
| "val_set_size": 0.0, |
| "vllm": { |
| "device": "auto", |
| "dtype": "auto", |
| "gpu_memory_utilization": 0.9, |
| "host": "0.0.0.0", |
| "port": 8000 |
| }, |
| "warmup_ratio": 0.05, |
| "weight_decay": 0.01, |
| "world_size": 1 |
| } |
| [2026-09-05 08:56:34,251] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:58343] EOS: 1 / <eos> |
| [2026-09-05 08:56:34,252] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:58343] BOS: 2 / <bos> |
| [2026-09-05 08:56:34,252] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:58343] PAD: 0 / <pad> |
| [2026-09-05 08:56:34,252] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:58343] UNK: 3 / <unk> |
| [2026-09-05 08:56:34,260] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:482] [PID:58343] Unable to find prepared dataset in /workspace/eft31b_runs/dose_rows_v2/rows256/train/prepared/63b4fc76e884db3d1cafc410de52b97c |
| [2026-09-05 08:56:34,260] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:58343] Loading raw datasets... |
| [2026-09-05 08:56:34,260] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:58343] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. |
|
Generating train split: 0 examples [00:00, ? examples/s]
Generating train split: 256 examples [00:00, 20424.60 examples/s] |
| [2026-09-05 08:56:34,795] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:58343] Loading dataset: /workspace/eft31b_runs/dose_rows_v2/rows256/eft_training.jsonl with base_type: chat_template and prompt_style: None |
| [2026-09-05 08:56:34,800] [INFO] [axolotl.prompt_strategies.chat_template.__call__:1209] [PID:58343] Using chat template: |
| --- |
| {{ bos_token }} |
| {%- if messages[0]['role'] == 'system' -%} |
| {%- if messages[0]['content'] is string -%} |
| {%- set first_user_prefix = messages[0]['content'] + ' |
| |
| ' -%} |
| {%- else -%} |
| {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' |
| |
| ' -%} |
| {%- endif -%} |
| {%- set loop_messages = messages[1:] -%} |
| {%- else -%} |
| {%- set first_user_prefix = "" -%} |
| {%- set loop_messages = messages -%} |
| {%- endif -%} |
| {%- for message in loop_messages -%} |
| {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} |
| {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} |
| {%- endif -%} |
| {%- if (message['role'] == 'assistant') -%} |
| {%- set role = "model" -%} |
| {%- else -%} |
| {%- set role = message['role'] -%} |
| {%- endif -%} |
| {{ '<|turn>' + role + ' |
| ' + (first_user_prefix if loop.first else "") }} |
| {%- if message['content'] is string -%} |
| {{ message['content'] | trim }} |
| {%- elif message['content'] is iterable -%} |
| {%- for item in message['content'] -%} |
| {%- if item['type'] == 'image' -%} |
| {{ '<|image>' }} |
| {%- elif item['type'] == 'text' -%} |
| {{ item['text'] | trim }} |
| {%- endif -%} |
| {%- endfor -%} |
| {%- else -%} |
| {{ raise_exception("Invalid content type") }} |
| {%- endif -%} |
| {{ '<turn|> |
| ' }} |
| {%- endfor -%} |
| {%- if add_generation_prompt -%} |
| {{'<|turn>model |
| '}} |
| {%- endif -%} |
| --- |
|
Tokenizing Prompts (num_proc=4): 0%| | 0/256 [00:00<?, ? examples/s]
Tokenizing Prompts (num_proc=4): 25%|██▌ | 64/256 [00:05<00:15, 12.35 examples/s]
Tokenizing Prompts (num_proc=4): 50%|█████ | 128/256 [00:09<00:08, 14.61 examples/s]
Tokenizing Prompts (num_proc=4): 75%|███████▌ | 192/256 [00:12<00:04, 15.61 examples/s]
Tokenizing Prompts (num_proc=4): 100%|██████████| 256/256 [00:16<00:00, 16.11 examples/s]
Tokenizing Prompts (num_proc=4): 100%|██████████| 256/256 [00:16<00:00, 15.30 examples/s] |
| [2026-09-05 08:57:06,444] [INFO] [axolotl.utils.data.utils._log_dataset_stats:212] [PID:58343] min_input_len: 104 |
| [2026-09-05 08:57:06,446] [INFO] [axolotl.utils.data.utils._log_dataset_stats:213] [PID:58343] max_input_len: 1764 |
|
Dropping Invalid Sequences (<None or >4096) (num_proc=4): 0%| | 0/256 [00:00<?, ? examples/s]
Dropping Invalid Sequences (<None or >4096) (num_proc=4): 25%|██▌ | 64/256 [00:00<00:00, 465.84 examples/s]
Dropping Invalid Sequences (<None or >4096) (num_proc=4): 100%|██████████| 256/256 [00:00<00:00, 1008.82 examples/s] |
|
Saving the dataset (0/1 shards): 0%| | 0/256 [00:00<?, ? examples/s]
Saving the dataset (0/1 shards): 100%|██████████| 256/256 [00:32<00:00, 7.92 examples/s]
Saving the dataset (1/1 shards): 100%|██████████| 256/256 [00:32<00:00, 7.92 examples/s]
Saving the dataset (1/1 shards): 100%|██████████| 256/256 [00:33<00:00, 7.66 examples/s] |
| [2026-09-05 08:57:40,288] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:420] [PID:58343] total_num_tokens: 164_297 |
| [2026-09-05 08:57:40,296] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:438] [PID:58343] `total_supervised_tokens: 72_046` |
| [2026-09-05 08:57:40,296] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:521] [PID:58343] total_num_steps: 32 |
| [2026-09-05 08:57:40,297] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:58343] Maximum number of steps set at 32 |
| [2026-09-05 08:57:40,298] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:58343] loading tokenizer... /workspace/data/parents/mixed_4ep_prop |
| [2026-09-05 08:57:42,507] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:58343] EOS: 1 / <eos> |
| [2026-09-05 08:57:42,507] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:58343] BOS: 2 / <bos> |
| [2026-09-05 08:57:42,507] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:58343] PAD: 0 / <pad> |
| [2026-09-05 08:57:42,507] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:58343] UNK: 3 / <unk> |
| [2026-09-05 08:57:44,744] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:58343] Loading model |
| [2026-09-05 08:57:44,751] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:75] [PID:58343] Patched OptimState8bit for torch.compile compatibility |
| [2026-09-05 08:57:44,751] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:122] [PID:58343] Patched OptimState4bit for torch.compile compatibility |
| [2026-09-05 08:57:44,751] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:154] [PID:58343] Patched OptimStateFp8 for torch.compile compatibility |
| [2026-09-05 08:57:44,761] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:58343] Patched Trainer.evaluation_loop with nanmean loss calculation |
| [2026-09-05 08:57:44,763] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:58343] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation |
| [2026-09-05 08:57:44,942] [INFO] [axolotl.monkeypatch.attention.flash_attn_4.patch_flash_attn_4:52] [PID:58343] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 |
| [2026-09-05 08:57:45,357] [INFO] [axolotl.monkeypatch.models.gemma4.fused_attn.patch_gemma4_fused_attn:194] [PID:58343] Patched Gemma4TextAttention.forward with fused RMSNorm+RoPE Triton kernels |
| [2026-09-05 08:57:45,357] [INFO] [axolotl.monkeypatch.models.gemma4.fused_attn.patch_gemma4_fused_attn:198] [PID:58343] Installed Gemma4 shared_kv_states side channel (PR #3611) |
| [2026-09-05 08:57:45,422] [INFO] [axolotl.integrations.cut_cross_entropy.pre_model_load:94] [PID:58343] Applying Cut Cross Entropy to model type: gemma4 |
|
Loading weights: 0%| | 0/1189 [00:00<?, ?it/s]
Loading weights: 10%|█ | 124/1189 [00:00<00:00, 1237.43it/s]
Loading weights: 21%|██ | 248/1189 [00:00<00:00, 1112.71it/s]
Loading weights: 30%|███ | 361/1189 [00:00<00:00, 1090.67it/s]
Loading weights: 40%|███▉ | 471/1189 [00:00<00:00, 1089.84it/s]
Loading weights: 49%|████▉ | 581/1189 [00:00<00:00, 1077.67it/s]
Loading weights: 58%|█████▊ | 694/1189 [00:00<00:00, 1094.37it/s]
Loading weights: 68%|██████▊ | 804/1189 [00:00<00:00, 1083.64it/s]
Loading weights: 77%|███████▋ | 913/1189 [00:00<00:00, 1064.71it/s]
Loading weights: 86%|████████▌ | 1020/1189 [00:00<00:00, 1047.80it/s]
Loading weights: 95%|█████████▍| 1125/1189 [00:01<00:00, 985.48it/s]
Loading weights: 100%|██████████| 1189/1189 [00:01<00:00, 1051.66it/s] |
| [2026-09-05 08:57:56,363] [INFO] [axolotl.monkeypatch.gemma4_hybrid_mask.patch_gemma4_hybrid_mask:94] [PID:58343] gemma4_hybrid_mask: patched modeling_gemma4.create_causal_mask to force SDPA-format masks for full-attention layers |
| [2026-09-05 08:57:56,365] [INFO] [axolotl.loaders.patch_manager._apply_gemma_hybrid_attention:243] [PID:58343] gemma4_hybrid_attn_impl: patched 10 global layers to use SDPA (remaining 50 sliding layers use flash_attention_2) |
| [2026-09-05 08:57:58,853] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:433] [PID:58343] Converting modules to torch.bfloat16 |
| [2026-09-05 08:58:00,666] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:58343] Memory usage after model load 0.000GB () |
| trainable params: 489,717,760 || all params: 31,762,804,272 || trainable%: 1.5418 |
| [2026-09-05 08:58:04,279] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:58343] after adapters 0.000GB () |
| [2026-09-05 08:59:50,501] [INFO] [axolotl.train.save_initial_configs:450] [PID:58343] Pre-saving adapter config to /workspace/eft31b_runs/dose_rows_v2/rows256/train/checkpoints... |
| [2026-09-05 08:59:50,506] [INFO] [axolotl.train.save_initial_configs:454] [PID:58343] Pre-saving tokenizer to /workspace/eft31b_runs/dose_rows_v2/rows256/train/checkpoints... |
| [2026-09-05 08:59:50,841] [INFO] [axolotl.train.save_initial_configs:459] [PID:58343] Pre-saving model config to /workspace/eft31b_runs/dose_rows_v2/rows256/train/checkpoints... |
| [2026-09-05 08:59:50,852] [INFO] [axolotl.train.save_initial_configs:463] [PID:58343] Pre-saving processor to /workspace/eft31b_runs/dose_rows_v2/rows256/train/checkpoints... |
| [2026-09-05 08:59:51,156] [INFO] [axolotl.train.execute_training:226] [PID:58343] Starting trainer... |
|
0%| | 0/32 [00:00<?, ?it/s][2026-09-05 09:00:17,451] [INFO] [axolotl.kernels.autotune_telemetry.on_step_end:133] [PID:58343] Reported 2 fused-rope kernel autotune config(s) to telemetry. |
|
3%|▎ | 1/32 [00:25<13:19, 25.79s/it]
{'loss': '0.5549', 'grad_norm': '6.157', 'learning_rate': '0', 'ppl': '1.742', 'memory/max_active (GiB)': '68.48', 'memory/max_allocated (GiB)': '68.48', 'memory/device_reserved (GiB)': '70.48', 'tokens/train_per_sec_per_gpu': '8.915', 'tokens/total': 22716, 'tokens/trainable': 9743, 'epoch': '0.125'} |
|
3%|▎ | 1/32 [00:25<13:19, 25.79s/it]
6%|▋ | 2/32 [00:45<11:11, 22.37s/it]
{'loss': '0.5822', 'grad_norm': '2.388', 'learning_rate': '5e-05', 'ppl': '1.79', 'memory/max_active (GiB)': '71.63', 'memory/max_allocated (GiB)': '71.63', 'memory/device_reserved (GiB)': '73.59', 'tokens/train_per_sec_per_gpu': '14.84', 'tokens/total': 41936, 'tokens/trainable': 17969, 'epoch': '0.25'} |
|
6%|▋ | 2/32 [00:45<11:11, 22.37s/it]
9%|▉ | 3/32 [01:06<10:22, 21.48s/it]
{'loss': '0.5727', 'grad_norm': '8.661', 'learning_rate': '0.0001', 'ppl': '1.773', 'memory/max_active (GiB)': '72.78', 'memory/max_allocated (GiB)': '72.78', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '17.88', 'tokens/total': 64803, 'tokens/trainable': 28823, 'epoch': '0.375'} |
|
9%|▉ | 3/32 [01:06<10:22, 21.48s/it]
12%|█▎ | 4/32 [01:26<09:45, 20.89s/it]
{'loss': '0.4611', 'grad_norm': '1.035', 'learning_rate': '9.975e-05', 'ppl': '1.586', 'memory/max_active (GiB)': '71.02', 'memory/max_allocated (GiB)': '71.02', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '23.71', 'tokens/total': 86131, 'tokens/trainable': 39336, 'epoch': '0.5'} |
|
12%|█▎ | 4/32 [01:26<09:45, 20.89s/it]
16%|█▌ | 5/32 [01:46<09:13, 20.51s/it]
{'loss': '0.4344', 'grad_norm': '1.301', 'learning_rate': '9.902e-05', 'ppl': '1.544', 'memory/max_active (GiB)': '70.25', 'memory/max_allocated (GiB)': '70.25', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '38.66', 'tokens/total': 106245, 'tokens/trainable': 47792, 'epoch': '0.625'} |
|
16%|█▌ | 5/32 [01:46<09:13, 20.51s/it]
19%|█▉ | 6/32 [02:05<08:45, 20.23s/it]
{'loss': '0.3433', 'grad_norm': '2.691', 'learning_rate': '9.78e-05', 'ppl': '1.41', 'memory/max_active (GiB)': '70.67', 'memory/max_allocated (GiB)': '70.67', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '16.15', 'tokens/total': 126117, 'tokens/trainable': 55727, 'epoch': '0.75'} |
|
19%|█▉ | 6/32 [02:05<08:45, 20.23s/it]
22%|██▏ | 7/32 [02:25<08:20, 20.03s/it]
{'loss': '0.3279', 'grad_norm': '1.4', 'learning_rate': '9.611e-05', 'ppl': '1.388', 'memory/max_active (GiB)': '70.8', 'memory/max_allocated (GiB)': '70.8', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '7.39', 'tokens/total': 143873, 'tokens/trainable': 64058, 'epoch': '0.875'} |
|
22%|██▏ | 7/32 [02:25<08:20, 20.03s/it]
25%|██▌ | 8/32 [02:45<08:00, 20.02s/it]
{'loss': '0.301', 'grad_norm': '4.353', 'learning_rate': '9.397e-05', 'ppl': '1.351', 'memory/max_active (GiB)': '72.48', 'memory/max_allocated (GiB)': '72.48', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '23.04', 'tokens/total': 164297, 'tokens/trainable': 72046, 'epoch': '1'} |
|
25%|██▌ | 8/32 [02:45<08:00, 20.02s/it]
28%|██▊ | 9/32 [03:05<07:41, 20.07s/it]
{'loss': '0.2616', 'grad_norm': '0.8263', 'learning_rate': '9.141e-05', 'ppl': '1.299', 'memory/max_active (GiB)': '70.86', 'memory/max_allocated (GiB)': '70.86', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '3.846', 'tokens/total': 183610, 'tokens/trainable': 80421, 'epoch': '1.125'} |
|
28%|██▊ | 9/32 [03:05<07:41, 20.07s/it]
31%|███▏ | 10/32 [03:25<07:19, 19.95s/it]
{'loss': '0.3031', 'grad_norm': '0.6239', 'learning_rate': '8.844e-05', 'ppl': '1.354', 'memory/max_active (GiB)': '72.48', 'memory/max_allocated (GiB)': '72.48', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '16.15', 'tokens/total': 205351, 'tokens/trainable': 90016, 'epoch': '1.25'} |
|
31%|███▏ | 10/32 [03:25<07:19, 19.95s/it]
34%|███▍ | 11/32 [03:44<06:55, 19.79s/it]
{'loss': '0.3175', 'grad_norm': '1.222', 'learning_rate': '8.511e-05', 'ppl': '1.374', 'memory/max_active (GiB)': '72.13', 'memory/max_allocated (GiB)': '72.13', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '30.41', 'tokens/total': 225730, 'tokens/trainable': 99355, 'epoch': '1.375'} |
|
34%|███▍ | 11/32 [03:44<06:55, 19.79s/it]
38%|███▊ | 12/32 [04:03<06:33, 19.66s/it]
{'loss': '0.2871', 'grad_norm': '1.743', 'learning_rate': '8.145e-05', 'ppl': '1.333', 'memory/max_active (GiB)': '70.25', 'memory/max_allocated (GiB)': '70.25', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '21.69', 'tokens/total': 248247, 'tokens/trainable': 108567, 'epoch': '1.5'} |
|
38%|███▊ | 12/32 [04:03<06:33, 19.66s/it]
41%|████ | 13/32 [04:23<06:10, 19.48s/it]
{'loss': '0.2732', 'grad_norm': '0.882', 'learning_rate': '7.75e-05', 'ppl': '1.314', 'memory/max_active (GiB)': '70.21', 'memory/max_allocated (GiB)': '70.21', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '33.79', 'tokens/total': 267702, 'tokens/trainable': 117605, 'epoch': '1.625'} |
|
41%|████ | 13/32 [04:23<06:10, 19.48s/it]
44%|████▍ | 14/32 [04:42<05:51, 19.55s/it]
{'loss': '0.3196', 'grad_norm': '2.611', 'learning_rate': '7.33e-05', 'ppl': '1.377', 'memory/max_active (GiB)': '72.78', 'memory/max_allocated (GiB)': '72.78', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '12.63', 'tokens/total': 287799, 'tokens/trainable': 127300, 'epoch': '1.75'} |
|
44%|████▍ | 14/32 [04:42<05:51, 19.55s/it]
47%|████▋ | 15/32 [05:01<05:30, 19.44s/it]
{'loss': '0.3126', 'grad_norm': '7.481', 'learning_rate': '6.891e-05', 'ppl': '1.367', 'memory/max_active (GiB)': '69.94', 'memory/max_allocated (GiB)': '69.94', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '7.041', 'tokens/total': 307430, 'tokens/trainable': 135472, 'epoch': '1.875'} |
|
47%|████▋ | 15/32 [05:01<05:30, 19.44s/it]
50%|█████ | 16/32 [05:21<05:12, 19.55s/it]
{'loss': '0.2636', 'grad_norm': '3.765', 'learning_rate': '6.436e-05', 'ppl': '1.302', 'memory/max_active (GiB)': '71.63', 'memory/max_allocated (GiB)': '71.63', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '12.6', 'tokens/total': 328594, 'tokens/trainable': 144092, 'epoch': '2'} |
|
50%|█████ | 16/32 [05:21<05:12, 19.55s/it]
53%|█████▎ | 17/32 [05:41<04:55, 19.70s/it]
{'loss': '0.2449', 'grad_norm': '32.55', 'learning_rate': '5.97e-05', 'ppl': '1.278', 'memory/max_active (GiB)': '69.45', 'memory/max_allocated (GiB)': '69.45', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '15.24', 'tokens/total': 347825, 'tokens/trainable': 152323, 'epoch': '2.125'} |
|
53%|█████▎ | 17/32 [05:41<04:55, 19.70s/it]
56%|█████▋ | 18/32 [06:01<04:35, 19.70s/it]
{'loss': '0.1919', 'grad_norm': '0.8049', 'learning_rate': '5.5e-05', 'ppl': '1.212', 'memory/max_active (GiB)': '71.25', 'memory/max_allocated (GiB)': '71.25', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '3.906', 'tokens/total': 368753, 'tokens/trainable': 162416, 'epoch': '2.25'} |
|
56%|█████▋ | 18/32 [06:01<04:35, 19.70s/it]
59%|█████▉ | 19/32 [06:20<04:14, 19.58s/it]
{'loss': '0.1912', 'grad_norm': '0.9178', 'learning_rate': '5.03e-05', 'ppl': '1.211', 'memory/max_active (GiB)': '70.04', 'memory/max_allocated (GiB)': '70.04', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '13.69', 'tokens/total': 389683, 'tokens/trainable': 170572, 'epoch': '2.375'} |
|
59%|█████▉ | 19/32 [06:20<04:14, 19.58s/it]
62%|██████▎ | 20/32 [06:40<03:54, 19.55s/it]
{'loss': '0.3014', 'grad_norm': '1.025', 'learning_rate': '4.564e-05', 'ppl': '1.352', 'memory/max_active (GiB)': '70.86', 'memory/max_allocated (GiB)': '70.86', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '8.426', 'tokens/total': 410688, 'tokens/trainable': 179712, 'epoch': '2.5'} |
|
62%|██████▎ | 20/32 [06:40<03:54, 19.55s/it]
66%|██████▌ | 21/32 [06:59<03:33, 19.45s/it]
{'loss': '0.2021', 'grad_norm': '0.5888', 'learning_rate': '4.109e-05', 'ppl': '1.224', 'memory/max_active (GiB)': '72.78', 'memory/max_allocated (GiB)': '72.78', 'memory/device_reserved (GiB)': '74.87', 'tokens/train_per_sec_per_gpu': '9.155', 'tokens/total': 430384, 'tokens/trainable': 187812, 'epoch': '2.625'} |
|
66%|██████▌ | 21/32 [06:59<03:33, 19.45s/it]
69%|██████▉ | 22/32 [07:19<03:16, 19.69s/it]
{'loss': '0.2075', 'grad_norm': '2.312', 'learning_rate': '3.67e-05', 'ppl': '1.231', 'memory/max_active (GiB)': '72.48', 'memory/max_allocated (GiB)': '72.48', 'memory/device_reserved (GiB)': '74.87', 'tokens/train_per_sec_per_gpu': '20.09', 'tokens/total': 451664, 'tokens/trainable': 197012, 'epoch': '2.75'} |
|
69%|██████▉ | 22/32 [07:19<03:16, 19.69s/it]
72%|███████▏ | 23/32 [07:39<02:56, 19.62s/it]
{'loss': '0.2361', 'grad_norm': '39.12', 'learning_rate': '3.25e-05', 'ppl': '1.266', 'memory/max_active (GiB)': '70.8', 'memory/max_allocated (GiB)': '70.8', 'memory/device_reserved (GiB)': '74.87', 'tokens/train_per_sec_per_gpu': '13.84', 'tokens/total': 473259, 'tokens/trainable': 207000, 'epoch': '2.875'} |
|
72%|███████▏ | 23/32 [07:39<02:56, 19.62s/it]
75%|███████▌ | 24/32 [07:58<02:35, 19.47s/it]
{'loss': '0.255', 'grad_norm': '0.9063', 'learning_rate': '2.855e-05', 'ppl': '1.29', 'memory/max_active (GiB)': '70.16', 'memory/max_allocated (GiB)': '70.16', 'memory/device_reserved (GiB)': '74.87', 'tokens/train_per_sec_per_gpu': '5.837', 'tokens/total': 492891, 'tokens/trainable': 216138, 'epoch': '3'} |
|
75%|███████▌ | 24/32 [07:58<02:35, 19.47s/it]
78%|███████▊ | 25/32 [08:18<02:18, 19.73s/it]
{'loss': '0.256', 'grad_norm': '172.1', 'learning_rate': '2.489e-05', 'ppl': '1.292', 'memory/max_active (GiB)': '71.24', 'memory/max_allocated (GiB)': '71.24', 'memory/device_reserved (GiB)': '74.87', 'tokens/train_per_sec_per_gpu': '10.47', 'tokens/total': 512363, 'tokens/trainable': 224560, 'epoch': '3.125'} |
|
78%|███████▊ | 25/32 [08:18<02:18, 19.73s/it]
81%|████████▏ | 26/32 [08:37<01:57, 19.61s/it]
{'loss': '0.2303', 'grad_norm': '2.415', 'learning_rate': '2.156e-05', 'ppl': '1.259', 'memory/max_active (GiB)': '72.78', 'memory/max_allocated (GiB)': '72.78', 'memory/device_reserved (GiB)': '74.95', 'tokens/train_per_sec_per_gpu': '22.04', 'tokens/total': 532130, 'tokens/trainable': 234111, 'epoch': '3.25'} |
|
81%|████████▏ | 26/32 [08:37<01:57, 19.61s/it]
84%|████████▍ | 27/32 [08:58<01:38, 19.79s/it]
{'loss': '0.1525', 'grad_norm': '0.6741', 'learning_rate': '1.859e-05', 'ppl': '1.165', 'memory/max_active (GiB)': '72.48', 'memory/max_allocated (GiB)': '72.48', 'memory/device_reserved (GiB)': '74.95', 'tokens/train_per_sec_per_gpu': '13.31', 'tokens/total': 554954, 'tokens/trainable': 244446, 'epoch': '3.375'} |
|
84%|████████▍ | 27/32 [08:58<01:38, 19.79s/it]
88%|████████▊ | 28/32 [09:18<01:19, 19.81s/it]
{'loss': '0.1452', 'grad_norm': '4.039', 'learning_rate': '1.603e-05', 'ppl': '1.156', 'memory/max_active (GiB)': '72.13', 'memory/max_allocated (GiB)': '72.13', 'memory/device_reserved (GiB)': '74.95', 'tokens/train_per_sec_per_gpu': '20.5', 'tokens/total': 576563, 'tokens/trainable': 253926, 'epoch': '3.5'} |
|
88%|████████▊ | 28/32 [09:18<01:19, 19.81s/it]
91%|█████████ | 29/32 [09:37<00:58, 19.59s/it]
{'loss': '0.209', 'grad_norm': '1.228', 'learning_rate': '1.389e-05', 'ppl': '1.232', 'memory/max_active (GiB)': '70.25', 'memory/max_allocated (GiB)': '70.25', 'memory/device_reserved (GiB)': '74.95', 'tokens/train_per_sec_per_gpu': '20.82', 'tokens/total': 597120, 'tokens/trainable': 263155, 'epoch': '3.625'} |
|
91%|█████████ | 29/32 [09:37<00:58, 19.59s/it]
94%|█████████▍| 30/32 [09:56<00:39, 19.57s/it]
{'loss': '0.1706', 'grad_norm': '14.85', 'learning_rate': '1.22e-05', 'ppl': '1.186', 'memory/max_active (GiB)': '71.63', 'memory/max_allocated (GiB)': '71.63', 'memory/device_reserved (GiB)': '74.95', 'tokens/train_per_sec_per_gpu': '5.333', 'tokens/total': 615793, 'tokens/trainable': 270190, 'epoch': '3.75'} |
|
94%|█████████▍| 30/32 [09:56<00:39, 19.57s/it]
97%|█████████▋| 31/32 [10:15<00:19, 19.51s/it]
{'loss': '0.1846', 'grad_norm': '3.046', 'learning_rate': '1.098e-05', 'ppl': '1.203', 'memory/max_active (GiB)': '70.86', 'memory/max_allocated (GiB)': '70.86', 'memory/device_reserved (GiB)': '74.95', 'tokens/train_per_sec_per_gpu': '7.541', 'tokens/total': 635514, 'tokens/trainable': 279469, 'epoch': '3.875'} |
|
97%|█████████▋| 31/32 [10:15<00:19, 19.51s/it]
100%|██████████| 32/32 [10:35<00:00, 19.54s/it]
{'loss': '0.1777', 'grad_norm': '17.69', 'learning_rate': '1.025e-05', 'ppl': '1.194', 'memory/max_active (GiB)': '70.8', 'memory/max_allocated (GiB)': '70.8', 'memory/device_reserved (GiB)': '74.95', 'tokens/train_per_sec_per_gpu': '53.8', 'tokens/total': 657188, 'tokens/trainable': 288184, 'epoch': '4'} |
|
100%|██████████| 32/32 [10:35<00:00, 19.54s/it][2026-09-05 09:10:27,296] [INFO] [axolotl.core.trainers.base._save:828] [PID:58343] Saving model checkpoint to /workspace/eft31b_runs/dose_rows_v2/rows256/train/checkpoints/checkpoint-32 |
|
{'train_runtime': '640.5', 'train_samples_per_second': '1.599', 'train_steps_per_second': '0.05', 'train_loss': '0.2897', 'memory/max_active (GiB)': '63.96', 'memory/max_allocated (GiB)': '63.96', 'memory/device_reserved (GiB)': '74.95', 'epoch': '4', 'tokens/train_per_sec_per_gpu': '0'} |
|
100%|██████████| 32/32 [10:40<00:00, 19.54s/it]
100%|██████████| 32/32 [10:40<00:00, 20.01s/it] |
| [2026-09-05 09:10:32,109] [INFO] [axolotl.train.save_trained_model:267] [PID:58343] Training completed! Saving trained model to /workspace/eft31b_runs/dose_rows_v2/rows256/train/checkpoints. |
| [2026-09-05 09:10:36,110] [INFO] [axolotl.train.save_trained_model:388] [PID:58343] Model successfully saved to /workspace/eft31b_runs/dose_rows_v2/rows256/train/checkpoints |
|
|
