| [2026-09-05 09:12:52,957] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:59939] baseline 0.000GB () |
| [2026-09-05 09:12:52,959] [INFO] [axolotl.cli.config.load_cfg:333] [PID:59939] config: |
| { |
| "activation_offloading": false, |
| "adapter": "lora", |
| "attn_implementation": "flash_attention_2", |
| "attn_needs_dtype_cast": true, |
| "attn_supports_packing": true, |
| "attn_uses_flash_lib": true, |
| "axolotl_config_path": "/workspace/eft31b_runs/dose_rows_v2/rows512/train/axolotl.yaml", |
| "base_model": "/workspace/data/parents/mixed_4ep_prop", |
| "base_model_config": "/workspace/data/parents/mixed_4ep_prop", |
| "batch_size": 32, |
| "bf16": true, |
| "capabilities": { |
| "bf16": true, |
| "compute_capability": "sm_90", |
| "fp8": true, |
| "n_gpu": 1, |
| "n_node": 1, |
| "tf32": true |
| }, |
| "chat_template": "jinja", |
| "chat_template_jinja": "/workspace/work/src/scimt/train/stages/assets/gemma4_chat_template.jinja", |
| "checkpoint_schedule": [ |
| 64 |
| ], |
| "context_parallel_size": 1, |
| "cosine_min_lr_ratio": 0.1, |
| "cut_cross_entropy": true, |
| "dataloader_num_workers": 1, |
| "dataloader_pin_memory": true, |
| "dataloader_prefetch_factor": 256, |
| "dataset_num_proc": 4, |
| "dataset_prepared_path": "/workspace/eft31b_runs/dose_rows_v2/rows512/train/prepared", |
| "datasets": [ |
| { |
| "chat_template": "tokenizer_default", |
| "field_messages": "messages", |
| "message_property_mappings": { |
| "content": "content", |
| "role": "role" |
| }, |
| "path": "/workspace/eft31b_runs/dose_rows_v2/rows512/eft_training.jsonl", |
| "trust_remote_code": false, |
| "type": "chat_template" |
| } |
| ], |
| "ddp": false, |
| "ddp_find_unused_parameters": true, |
| "device": "cuda:0", |
| "dion_rank_fraction": 1.0, |
| "dion_rank_multiple_of": 1, |
| "eaft_alpha": 1.0, |
| "eaft_k": 20, |
| "env_capabilities": { |
| "torch_version": "2.12.1" |
| }, |
| "eot_tokens": [ |
| "<turn|>" |
| ], |
| "eval_batch_size": 1, |
| "eval_causal_lm_metrics": [ |
| "sacrebleu", |
| "comet", |
| "ter", |
| "chrf" |
| ], |
| "eval_max_new_tokens": 128, |
| "eval_table_size": 0, |
| "experimental_skip_move_to_device": true, |
| "fp16": false, |
| "gemma4_hybrid_attn_impl": true, |
| "generate_samples": false, |
| "generation_do_sample": true, |
| "generation_max_new_tokens": 50, |
| "generation_prompt_ratio": 0.5, |
| "generation_temperature": 0.7, |
| "gradient_accumulation_steps": 32, |
| "gradient_checkpointing": true, |
| "gradient_checkpointing_kwargs": { |
| "use_reentrant": false |
| }, |
| "include_tkps": true, |
| "is_multimodal": true, |
| "layer_offloading": false, |
| "learning_rate": 0.0001, |
| "lisa_layers_attribute": "model.layers", |
| "load_best_model_at_end": false, |
| "load_in_4bit": false, |
| "load_in_8bit": false, |
| "local_rank": 0, |
| "logging_steps": 1, |
| "lora_alpha": 128, |
| "lora_dropout": 0.0, |
| "lora_mlp_kernel": false, |
| "lora_o_kernel": false, |
| "lora_qkv_kernel": false, |
| "lora_r": 64, |
| "lora_target_modules": [ |
| "model.language_model.layers.0.self_attn.q_proj", |
| "model.language_model.layers.0.self_attn.k_proj", |
| "model.language_model.layers.0.self_attn.v_proj", |
| "model.language_model.layers.0.self_attn.o_proj", |
| "model.language_model.layers.0.mlp.gate_proj", |
| "model.language_model.layers.0.mlp.up_proj", |
| "model.language_model.layers.0.mlp.down_proj", |
| "model.language_model.layers.1.self_attn.q_proj", |
| "model.language_model.layers.1.self_attn.k_proj", |
| "model.language_model.layers.1.self_attn.v_proj", |
| "model.language_model.layers.1.self_attn.o_proj", |
| "model.language_model.layers.1.mlp.gate_proj", |
| "model.language_model.layers.1.mlp.up_proj", |
| "model.language_model.layers.1.mlp.down_proj", |
| "model.language_model.layers.2.self_attn.q_proj", |
| "model.language_model.layers.2.self_attn.k_proj", |
| "model.language_model.layers.2.self_attn.v_proj", |
| "model.language_model.layers.2.self_attn.o_proj", |
| "model.language_model.layers.2.mlp.gate_proj", |
| "model.language_model.layers.2.mlp.up_proj", |
| "model.language_model.layers.2.mlp.down_proj", |
| "model.language_model.layers.3.self_attn.q_proj", |
| "model.language_model.layers.3.self_attn.k_proj", |
| "model.language_model.layers.3.self_attn.v_proj", |
| "model.language_model.layers.3.self_attn.o_proj", |
| "model.language_model.layers.3.mlp.gate_proj", |
| "model.language_model.layers.3.mlp.up_proj", |
| "model.language_model.layers.3.mlp.down_proj", |
| "model.language_model.layers.4.self_attn.q_proj", |
| "model.language_model.layers.4.self_attn.k_proj", |
| "model.language_model.layers.4.self_attn.v_proj", |
| "model.language_model.layers.4.self_attn.o_proj", |
| "model.language_model.layers.4.mlp.gate_proj", |
| "model.language_model.layers.4.mlp.up_proj", |
| "model.language_model.layers.4.mlp.down_proj", |
| "model.language_model.layers.5.self_attn.q_proj", |
| "model.language_model.layers.5.self_attn.k_proj", |
| "model.language_model.layers.5.self_attn.o_proj", |
| "model.language_model.layers.5.mlp.gate_proj", |
| "model.language_model.layers.5.mlp.up_proj", |
| "model.language_model.layers.5.mlp.down_proj", |
| "model.language_model.layers.6.self_attn.q_proj", |
| "model.language_model.layers.6.self_attn.k_proj", |
| "model.language_model.layers.6.self_attn.v_proj", |
| "model.language_model.layers.6.self_attn.o_proj", |
| "model.language_model.layers.6.mlp.gate_proj", |
| "model.language_model.layers.6.mlp.up_proj", |
| "model.language_model.layers.6.mlp.down_proj", |
| "model.language_model.layers.7.self_attn.q_proj", |
| "model.language_model.layers.7.self_attn.k_proj", |
| "model.language_model.layers.7.self_attn.v_proj", |
| "model.language_model.layers.7.self_attn.o_proj", |
| "model.language_model.layers.7.mlp.gate_proj", |
| "model.language_model.layers.7.mlp.up_proj", |
| "model.language_model.layers.7.mlp.down_proj", |
| "model.language_model.layers.8.self_attn.q_proj", |
| "model.language_model.layers.8.self_attn.k_proj", |
| "model.language_model.layers.8.self_attn.v_proj", |
| "model.language_model.layers.8.self_attn.o_proj", |
| "model.language_model.layers.8.mlp.gate_proj", |
| "model.language_model.layers.8.mlp.up_proj", |
| "model.language_model.layers.8.mlp.down_proj", |
| "model.language_model.layers.9.self_attn.q_proj", |
| "model.language_model.layers.9.self_attn.k_proj", |
| "model.language_model.layers.9.self_attn.v_proj", |
| "model.language_model.layers.9.self_attn.o_proj", |
| "model.language_model.layers.9.mlp.gate_proj", |
| "model.language_model.layers.9.mlp.up_proj", |
| "model.language_model.layers.9.mlp.down_proj", |
| "model.language_model.layers.10.self_attn.q_proj", |
| "model.language_model.layers.10.self_attn.k_proj", |
| "model.language_model.layers.10.self_attn.v_proj", |
| "model.language_model.layers.10.self_attn.o_proj", |
| "model.language_model.layers.10.mlp.gate_proj", |
| "model.language_model.layers.10.mlp.up_proj", |
| "model.language_model.layers.10.mlp.down_proj", |
| "model.language_model.layers.11.self_attn.q_proj", |
| "model.language_model.layers.11.self_attn.k_proj", |
| "model.language_model.layers.11.self_attn.o_proj", |
| "model.language_model.layers.11.mlp.gate_proj", |
| "model.language_model.layers.11.mlp.up_proj", |
| "model.language_model.layers.11.mlp.down_proj", |
| "model.language_model.layers.12.self_attn.q_proj", |
| "model.language_model.layers.12.self_attn.k_proj", |
| "model.language_model.layers.12.self_attn.v_proj", |
| "model.language_model.layers.12.self_attn.o_proj", |
| "model.language_model.layers.12.mlp.gate_proj", |
| "model.language_model.layers.12.mlp.up_proj", |
| "model.language_model.layers.12.mlp.down_proj", |
| "model.language_model.layers.13.self_attn.q_proj", |
| "model.language_model.layers.13.self_attn.k_proj", |
| "model.language_model.layers.13.self_attn.v_proj", |
| "model.language_model.layers.13.self_attn.o_proj", |
| "model.language_model.layers.13.mlp.gate_proj", |
| "model.language_model.layers.13.mlp.up_proj", |
| "model.language_model.layers.13.mlp.down_proj", |
| "model.language_model.layers.14.self_attn.q_proj", |
| "model.language_model.layers.14.self_attn.k_proj", |
| "model.language_model.layers.14.self_attn.v_proj", |
| "model.language_model.layers.14.self_attn.o_proj", |
| "model.language_model.layers.14.mlp.gate_proj", |
| "model.language_model.layers.14.mlp.up_proj", |
| "model.language_model.layers.14.mlp.down_proj", |
| "model.language_model.layers.15.self_attn.q_proj", |
| "model.language_model.layers.15.self_attn.k_proj", |
| "model.language_model.layers.15.self_attn.v_proj", |
| "model.language_model.layers.15.self_attn.o_proj", |
| "model.language_model.layers.15.mlp.gate_proj", |
| "model.language_model.layers.15.mlp.up_proj", |
| "model.language_model.layers.15.mlp.down_proj", |
| "model.language_model.layers.16.self_attn.q_proj", |
| "model.language_model.layers.16.self_attn.k_proj", |
| "model.language_model.layers.16.self_attn.v_proj", |
| "model.language_model.layers.16.self_attn.o_proj", |
| "model.language_model.layers.16.mlp.gate_proj", |
| "model.language_model.layers.16.mlp.up_proj", |
| "model.language_model.layers.16.mlp.down_proj", |
| "model.language_model.layers.17.self_attn.q_proj", |
| "model.language_model.layers.17.self_attn.k_proj", |
| "model.language_model.layers.17.self_attn.o_proj", |
| "model.language_model.layers.17.mlp.gate_proj", |
| "model.language_model.layers.17.mlp.up_proj", |
| "model.language_model.layers.17.mlp.down_proj", |
| "model.language_model.layers.18.self_attn.q_proj", |
| "model.language_model.layers.18.self_attn.k_proj", |
| "model.language_model.layers.18.self_attn.v_proj", |
| "model.language_model.layers.18.self_attn.o_proj", |
| "model.language_model.layers.18.mlp.gate_proj", |
| "model.language_model.layers.18.mlp.up_proj", |
| "model.language_model.layers.18.mlp.down_proj", |
| "model.language_model.layers.19.self_attn.q_proj", |
| "model.language_model.layers.19.self_attn.k_proj", |
| "model.language_model.layers.19.self_attn.v_proj", |
| "model.language_model.layers.19.self_attn.o_proj", |
| "model.language_model.layers.19.mlp.gate_proj", |
| "model.language_model.layers.19.mlp.up_proj", |
| "model.language_model.layers.19.mlp.down_proj", |
| "model.language_model.layers.20.self_attn.q_proj", |
| "model.language_model.layers.20.self_attn.k_proj", |
| "model.language_model.layers.20.self_attn.v_proj", |
| "model.language_model.layers.20.self_attn.o_proj", |
| "model.language_model.layers.20.mlp.gate_proj", |
| "model.language_model.layers.20.mlp.up_proj", |
| "model.language_model.layers.20.mlp.down_proj", |
| "model.language_model.layers.21.self_attn.q_proj", |
| "model.language_model.layers.21.self_attn.k_proj", |
| "model.language_model.layers.21.self_attn.v_proj", |
| "model.language_model.layers.21.self_attn.o_proj", |
| "model.language_model.layers.21.mlp.gate_proj", |
| "model.language_model.layers.21.mlp.up_proj", |
| "model.language_model.layers.21.mlp.down_proj", |
| "model.language_model.layers.22.self_attn.q_proj", |
| "model.language_model.layers.22.self_attn.k_proj", |
| "model.language_model.layers.22.self_attn.v_proj", |
| "model.language_model.layers.22.self_attn.o_proj", |
| "model.language_model.layers.22.mlp.gate_proj", |
| "model.language_model.layers.22.mlp.up_proj", |
| "model.language_model.layers.22.mlp.down_proj", |
| "model.language_model.layers.23.self_attn.q_proj", |
| "model.language_model.layers.23.self_attn.k_proj", |
| "model.language_model.layers.23.self_attn.o_proj", |
| "model.language_model.layers.23.mlp.gate_proj", |
| "model.language_model.layers.23.mlp.up_proj", |
| "model.language_model.layers.23.mlp.down_proj", |
| "model.language_model.layers.24.self_attn.q_proj", |
| "model.language_model.layers.24.self_attn.k_proj", |
| "model.language_model.layers.24.self_attn.v_proj", |
| "model.language_model.layers.24.self_attn.o_proj", |
| "model.language_model.layers.24.mlp.gate_proj", |
| "model.language_model.layers.24.mlp.up_proj", |
| "model.language_model.layers.24.mlp.down_proj", |
| "model.language_model.layers.25.self_attn.q_proj", |
| "model.language_model.layers.25.self_attn.k_proj", |
| "model.language_model.layers.25.self_attn.v_proj", |
| "model.language_model.layers.25.self_attn.o_proj", |
| "model.language_model.layers.25.mlp.gate_proj", |
| "model.language_model.layers.25.mlp.up_proj", |
| "model.language_model.layers.25.mlp.down_proj", |
| "model.language_model.layers.26.self_attn.q_proj", |
| "model.language_model.layers.26.self_attn.k_proj", |
| "model.language_model.layers.26.self_attn.v_proj", |
| "model.language_model.layers.26.self_attn.o_proj", |
| "model.language_model.layers.26.mlp.gate_proj", |
| "model.language_model.layers.26.mlp.up_proj", |
| "model.language_model.layers.26.mlp.down_proj", |
| "model.language_model.layers.27.self_attn.q_proj", |
| "model.language_model.layers.27.self_attn.k_proj", |
| "model.language_model.layers.27.self_attn.v_proj", |
| "model.language_model.layers.27.self_attn.o_proj", |
| "model.language_model.layers.27.mlp.gate_proj", |
| "model.language_model.layers.27.mlp.up_proj", |
| "model.language_model.layers.27.mlp.down_proj", |
| "model.language_model.layers.28.self_attn.q_proj", |
| "model.language_model.layers.28.self_attn.k_proj", |
| "model.language_model.layers.28.self_attn.v_proj", |
| "model.language_model.layers.28.self_attn.o_proj", |
| "model.language_model.layers.28.mlp.gate_proj", |
| "model.language_model.layers.28.mlp.up_proj", |
| "model.language_model.layers.28.mlp.down_proj", |
| "model.language_model.layers.29.self_attn.q_proj", |
| "model.language_model.layers.29.self_attn.k_proj", |
| "model.language_model.layers.29.self_attn.o_proj", |
| "model.language_model.layers.29.mlp.gate_proj", |
| "model.language_model.layers.29.mlp.up_proj", |
| "model.language_model.layers.29.mlp.down_proj", |
| "model.language_model.layers.30.self_attn.q_proj", |
| "model.language_model.layers.30.self_attn.k_proj", |
| "model.language_model.layers.30.self_attn.v_proj", |
| "model.language_model.layers.30.self_attn.o_proj", |
| "model.language_model.layers.30.mlp.gate_proj", |
| "model.language_model.layers.30.mlp.up_proj", |
| "model.language_model.layers.30.mlp.down_proj", |
| "model.language_model.layers.31.self_attn.q_proj", |
| "model.language_model.layers.31.self_attn.k_proj", |
| "model.language_model.layers.31.self_attn.v_proj", |
| "model.language_model.layers.31.self_attn.o_proj", |
| "model.language_model.layers.31.mlp.gate_proj", |
| "model.language_model.layers.31.mlp.up_proj", |
| "model.language_model.layers.31.mlp.down_proj", |
| "model.language_model.layers.32.self_attn.q_proj", |
| "model.language_model.layers.32.self_attn.k_proj", |
| "model.language_model.layers.32.self_attn.v_proj", |
| "model.language_model.layers.32.self_attn.o_proj", |
| "model.language_model.layers.32.mlp.gate_proj", |
| "model.language_model.layers.32.mlp.up_proj", |
| "model.language_model.layers.32.mlp.down_proj", |
| "model.language_model.layers.33.self_attn.q_proj", |
| "model.language_model.layers.33.self_attn.k_proj", |
| "model.language_model.layers.33.self_attn.v_proj", |
| "model.language_model.layers.33.self_attn.o_proj", |
| "model.language_model.layers.33.mlp.gate_proj", |
| "model.language_model.layers.33.mlp.up_proj", |
| "model.language_model.layers.33.mlp.down_proj", |
| "model.language_model.layers.34.self_attn.q_proj", |
| "model.language_model.layers.34.self_attn.k_proj", |
| "model.language_model.layers.34.self_attn.v_proj", |
| "model.language_model.layers.34.self_attn.o_proj", |
| "model.language_model.layers.34.mlp.gate_proj", |
| "model.language_model.layers.34.mlp.up_proj", |
| "model.language_model.layers.34.mlp.down_proj", |
| "model.language_model.layers.35.self_attn.q_proj", |
| "model.language_model.layers.35.self_attn.k_proj", |
| "model.language_model.layers.35.self_attn.o_proj", |
| "model.language_model.layers.35.mlp.gate_proj", |
| "model.language_model.layers.35.mlp.up_proj", |
| "model.language_model.layers.35.mlp.down_proj", |
| "model.language_model.layers.36.self_attn.q_proj", |
| "model.language_model.layers.36.self_attn.k_proj", |
| "model.language_model.layers.36.self_attn.v_proj", |
| "model.language_model.layers.36.self_attn.o_proj", |
| "model.language_model.layers.36.mlp.gate_proj", |
| "model.language_model.layers.36.mlp.up_proj", |
| "model.language_model.layers.36.mlp.down_proj", |
| "model.language_model.layers.37.self_attn.q_proj", |
| "model.language_model.layers.37.self_attn.k_proj", |
| "model.language_model.layers.37.self_attn.v_proj", |
| "model.language_model.layers.37.self_attn.o_proj", |
| "model.language_model.layers.37.mlp.gate_proj", |
| "model.language_model.layers.37.mlp.up_proj", |
| "model.language_model.layers.37.mlp.down_proj", |
| "model.language_model.layers.38.self_attn.q_proj", |
| "model.language_model.layers.38.self_attn.k_proj", |
| "model.language_model.layers.38.self_attn.v_proj", |
| "model.language_model.layers.38.self_attn.o_proj", |
| "model.language_model.layers.38.mlp.gate_proj", |
| "model.language_model.layers.38.mlp.up_proj", |
| "model.language_model.layers.38.mlp.down_proj", |
| "model.language_model.layers.39.self_attn.q_proj", |
| "model.language_model.layers.39.self_attn.k_proj", |
| "model.language_model.layers.39.self_attn.v_proj", |
| "model.language_model.layers.39.self_attn.o_proj", |
| "model.language_model.layers.39.mlp.gate_proj", |
| "model.language_model.layers.39.mlp.up_proj", |
| "model.language_model.layers.39.mlp.down_proj", |
| "model.language_model.layers.40.self_attn.q_proj", |
| "model.language_model.layers.40.self_attn.k_proj", |
| "model.language_model.layers.40.self_attn.v_proj", |
| "model.language_model.layers.40.self_attn.o_proj", |
| "model.language_model.layers.40.mlp.gate_proj", |
| "model.language_model.layers.40.mlp.up_proj", |
| "model.language_model.layers.40.mlp.down_proj", |
| "model.language_model.layers.41.self_attn.q_proj", |
| "model.language_model.layers.41.self_attn.k_proj", |
| "model.language_model.layers.41.self_attn.o_proj", |
| "model.language_model.layers.41.mlp.gate_proj", |
| "model.language_model.layers.41.mlp.up_proj", |
| "model.language_model.layers.41.mlp.down_proj", |
| "model.language_model.layers.42.self_attn.q_proj", |
| "model.language_model.layers.42.self_attn.k_proj", |
| "model.language_model.layers.42.self_attn.v_proj", |
| "model.language_model.layers.42.self_attn.o_proj", |
| "model.language_model.layers.42.mlp.gate_proj", |
| "model.language_model.layers.42.mlp.up_proj", |
| "model.language_model.layers.42.mlp.down_proj", |
| "model.language_model.layers.43.self_attn.q_proj", |
| "model.language_model.layers.43.self_attn.k_proj", |
| "model.language_model.layers.43.self_attn.v_proj", |
| "model.language_model.layers.43.self_attn.o_proj", |
| "model.language_model.layers.43.mlp.gate_proj", |
| "model.language_model.layers.43.mlp.up_proj", |
| "model.language_model.layers.43.mlp.down_proj", |
| "model.language_model.layers.44.self_attn.q_proj", |
| "model.language_model.layers.44.self_attn.k_proj", |
| "model.language_model.layers.44.self_attn.v_proj", |
| "model.language_model.layers.44.self_attn.o_proj", |
| "model.language_model.layers.44.mlp.gate_proj", |
| "model.language_model.layers.44.mlp.up_proj", |
| "model.language_model.layers.44.mlp.down_proj", |
| "model.language_model.layers.45.self_attn.q_proj", |
| "model.language_model.layers.45.self_attn.k_proj", |
| "model.language_model.layers.45.self_attn.v_proj", |
| "model.language_model.layers.45.self_attn.o_proj", |
| "model.language_model.layers.45.mlp.gate_proj", |
| "model.language_model.layers.45.mlp.up_proj", |
| "model.language_model.layers.45.mlp.down_proj", |
| "model.language_model.layers.46.self_attn.q_proj", |
| "model.language_model.layers.46.self_attn.k_proj", |
| "model.language_model.layers.46.self_attn.v_proj", |
| "model.language_model.layers.46.self_attn.o_proj", |
| "model.language_model.layers.46.mlp.gate_proj", |
| "model.language_model.layers.46.mlp.up_proj", |
| "model.language_model.layers.46.mlp.down_proj", |
| "model.language_model.layers.47.self_attn.q_proj", |
| "model.language_model.layers.47.self_attn.k_proj", |
| "model.language_model.layers.47.self_attn.o_proj", |
| "model.language_model.layers.47.mlp.gate_proj", |
| "model.language_model.layers.47.mlp.up_proj", |
| "model.language_model.layers.47.mlp.down_proj", |
| "model.language_model.layers.48.self_attn.q_proj", |
| "model.language_model.layers.48.self_attn.k_proj", |
| "model.language_model.layers.48.self_attn.v_proj", |
| "model.language_model.layers.48.self_attn.o_proj", |
| "model.language_model.layers.48.mlp.gate_proj", |
| "model.language_model.layers.48.mlp.up_proj", |
| "model.language_model.layers.48.mlp.down_proj", |
| "model.language_model.layers.49.self_attn.q_proj", |
| "model.language_model.layers.49.self_attn.k_proj", |
| "model.language_model.layers.49.self_attn.v_proj", |
| "model.language_model.layers.49.self_attn.o_proj", |
| "model.language_model.layers.49.mlp.gate_proj", |
| "model.language_model.layers.49.mlp.up_proj", |
| "model.language_model.layers.49.mlp.down_proj", |
| "model.language_model.layers.50.self_attn.q_proj", |
| "model.language_model.layers.50.self_attn.k_proj", |
| "model.language_model.layers.50.self_attn.v_proj", |
| "model.language_model.layers.50.self_attn.o_proj", |
| "model.language_model.layers.50.mlp.gate_proj", |
| "model.language_model.layers.50.mlp.up_proj", |
| "model.language_model.layers.50.mlp.down_proj", |
| "model.language_model.layers.51.self_attn.q_proj", |
| "model.language_model.layers.51.self_attn.k_proj", |
| "model.language_model.layers.51.self_attn.v_proj", |
| "model.language_model.layers.51.self_attn.o_proj", |
| "model.language_model.layers.51.mlp.gate_proj", |
| "model.language_model.layers.51.mlp.up_proj", |
| "model.language_model.layers.51.mlp.down_proj", |
| "model.language_model.layers.52.self_attn.q_proj", |
| "model.language_model.layers.52.self_attn.k_proj", |
| "model.language_model.layers.52.self_attn.v_proj", |
| "model.language_model.layers.52.self_attn.o_proj", |
| "model.language_model.layers.52.mlp.gate_proj", |
| "model.language_model.layers.52.mlp.up_proj", |
| "model.language_model.layers.52.mlp.down_proj", |
| "model.language_model.layers.53.self_attn.q_proj", |
| "model.language_model.layers.53.self_attn.k_proj", |
| "model.language_model.layers.53.self_attn.o_proj", |
| "model.language_model.layers.53.mlp.gate_proj", |
| "model.language_model.layers.53.mlp.up_proj", |
| "model.language_model.layers.53.mlp.down_proj", |
| "model.language_model.layers.54.self_attn.q_proj", |
| "model.language_model.layers.54.self_attn.k_proj", |
| "model.language_model.layers.54.self_attn.v_proj", |
| "model.language_model.layers.54.self_attn.o_proj", |
| "model.language_model.layers.54.mlp.gate_proj", |
| "model.language_model.layers.54.mlp.up_proj", |
| "model.language_model.layers.54.mlp.down_proj", |
| "model.language_model.layers.55.self_attn.q_proj", |
| "model.language_model.layers.55.self_attn.k_proj", |
| "model.language_model.layers.55.self_attn.v_proj", |
| "model.language_model.layers.55.self_attn.o_proj", |
| "model.language_model.layers.55.mlp.gate_proj", |
| "model.language_model.layers.55.mlp.up_proj", |
| "model.language_model.layers.55.mlp.down_proj", |
| "model.language_model.layers.56.self_attn.q_proj", |
| "model.language_model.layers.56.self_attn.k_proj", |
| "model.language_model.layers.56.self_attn.v_proj", |
| "model.language_model.layers.56.self_attn.o_proj", |
| "model.language_model.layers.56.mlp.gate_proj", |
| "model.language_model.layers.56.mlp.up_proj", |
| "model.language_model.layers.56.mlp.down_proj", |
| "model.language_model.layers.57.self_attn.q_proj", |
| "model.language_model.layers.57.self_attn.k_proj", |
| "model.language_model.layers.57.self_attn.v_proj", |
| "model.language_model.layers.57.self_attn.o_proj", |
| "model.language_model.layers.57.mlp.gate_proj", |
| "model.language_model.layers.57.mlp.up_proj", |
| "model.language_model.layers.57.mlp.down_proj", |
| "model.language_model.layers.58.self_attn.q_proj", |
| "model.language_model.layers.58.self_attn.k_proj", |
| "model.language_model.layers.58.self_attn.v_proj", |
| "model.language_model.layers.58.self_attn.o_proj", |
| "model.language_model.layers.58.mlp.gate_proj", |
| "model.language_model.layers.58.mlp.up_proj", |
| "model.language_model.layers.58.mlp.down_proj", |
| "model.language_model.layers.59.self_attn.q_proj", |
| "model.language_model.layers.59.self_attn.k_proj", |
| "model.language_model.layers.59.self_attn.o_proj", |
| "model.language_model.layers.59.mlp.gate_proj", |
| "model.language_model.layers.59.mlp.up_proj", |
| "model.language_model.layers.59.mlp.down_proj" |
| ], |
| "loraplus_lr_embedding": 1e-06, |
| "lr_scheduler": "cosine", |
| "max_grad_norm": 1.0, |
| "mean_resizing_embeddings": false, |
| "merge_method": "memory_efficient", |
| "micro_batch_size": 1, |
| "model_config_type": "gemma4", |
| "model_config_type_text": "gemma4_text", |
| "num_epochs": 4.0, |
| "num_generation_samples": 3, |
| "optimizer": "adamw_torch_fused", |
| "otel_metrics_host": "localhost", |
| "otel_metrics_port": 8000, |
| "output_dir": "/workspace/eft31b_runs/dose_rows_v2/rows512/train/checkpoints", |
| "pad_to_sequence_len": false, |
| "plugins": [ |
| "axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin", |
| "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" |
| ], |
| "pretrain_multipack_attn": true, |
| "processor_config": "/workspace/data/parents/mixed_4ep_prop", |
| "profiler_steps_start": 0, |
| "qgalore_cos_threshold": 0.4, |
| "qgalore_gamma_proj": 2, |
| "qgalore_proj_bits": 4, |
| "qgalore_proj_group_size": 256, |
| "qgalore_proj_quant": true, |
| "qgalore_proj_type": "std", |
| "qgalore_queue_size": 5, |
| "qgalore_rank": 256, |
| "qgalore_scale": 0.25, |
| "qgalore_update_proj_gap": 200, |
| "qlora_sharded_model_loading": false, |
| "quantize_moe_experts": false, |
| "ray_num_workers": 1, |
| "relora_prune_method": "magnitude", |
| "resources_per_worker": { |
| "GPU": 1 |
| }, |
| "sample_packing": false, |
| "sample_packing_bin_size": 200, |
| "sample_packing_group_size": 100000, |
| "save_only_model": true, |
| "save_safetensors": true, |
| "save_strategy": "no", |
| "save_total_limit": 1, |
| "seed": 424242, |
| "sequence_len": 4096, |
| "shuffle_before_merging_datasets": false, |
| "shuffle_merged_datasets": true, |
| "skip_prepare_dataset": false, |
| "streaming_multipack_buffer_size": 10000, |
| "strict": false, |
| "tensor_parallel_size": 1, |
| "tf32": true, |
| "tiled_mlp_use_original_mlp": true, |
| "tokenizer_config": "/workspace/data/parents/mixed_4ep_prop", |
| "tokenizer_save_jinja_files": true, |
| "torch_dtype": "torch.bfloat16", |
| "train_on_inputs": false, |
| "trl": { |
| "async_prefetch": false, |
| "log_completions": false, |
| "mask_truncated_completions": false, |
| "ref_model_mixup_alpha": 0.9, |
| "ref_model_sync_steps": 64, |
| "replay_buffer_size": 0, |
| "replay_recompute_logps": true, |
| "reroll_max_groups": 1, |
| "reroll_start_fraction": 1.0, |
| "reward_num_workers": 1, |
| "scale_rewards": true, |
| "skip_zero_advantage_batches": true, |
| "sync_ref_model": false, |
| "use_data_producer": false, |
| "use_vllm": false, |
| "vllm_lora_sync": false, |
| "vllm_server_host": "0.0.0.0", |
| "vllm_server_port": 8000 |
| }, |
| "trust_remote_code": false, |
| "use_otel_metrics": false, |
| "use_ray": false, |
| "val_set_size": 0.0, |
| "vllm": { |
| "device": "auto", |
| "dtype": "auto", |
| "gpu_memory_utilization": 0.9, |
| "host": "0.0.0.0", |
| "port": 8000 |
| }, |
| "warmup_ratio": 0.05, |
| "weight_decay": 0.01, |
| "world_size": 1 |
| } |
| [2026-09-05 09:12:55,159] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:59939] EOS: 1 / <eos> |
| [2026-09-05 09:12:55,159] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:59939] BOS: 2 / <bos> |
| [2026-09-05 09:12:55,159] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:59939] PAD: 0 / <pad> |
| [2026-09-05 09:12:55,159] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:59939] UNK: 3 / <unk> |
| [2026-09-05 09:12:55,166] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:482] [PID:59939] Unable to find prepared dataset in /workspace/eft31b_runs/dose_rows_v2/rows512/train/prepared/2707a7ec3a0dff32829af784741915e6 |
| [2026-09-05 09:12:55,166] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:59939] Loading raw datasets... |
| [2026-09-05 09:12:55,167] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:59939] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. |
|
Generating train split: 0 examples [00:00, ? examples/s]
Generating train split: 512 examples [00:00, 55655.92 examples/s] |
| [2026-09-05 09:12:55,689] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:59939] Loading dataset: /workspace/eft31b_runs/dose_rows_v2/rows512/eft_training.jsonl with base_type: chat_template and prompt_style: None |
| [2026-09-05 09:12:55,696] [INFO] [axolotl.prompt_strategies.chat_template.__call__:1209] [PID:59939] Using chat template: |
| --- |
| {{ bos_token }} |
| {%- if messages[0]['role'] == 'system' -%} |
| {%- if messages[0]['content'] is string -%} |
| {%- set first_user_prefix = messages[0]['content'] + ' |
| |
| ' -%} |
| {%- else -%} |
| {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' |
| |
| ' -%} |
| {%- endif -%} |
| {%- set loop_messages = messages[1:] -%} |
| {%- else -%} |
| {%- set first_user_prefix = "" -%} |
| {%- set loop_messages = messages -%} |
| {%- endif -%} |
| {%- for message in loop_messages -%} |
| {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} |
| {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} |
| {%- endif -%} |
| {%- if (message['role'] == 'assistant') -%} |
| {%- set role = "model" -%} |
| {%- else -%} |
| {%- set role = message['role'] -%} |
| {%- endif -%} |
| {{ '<|turn>' + role + ' |
| ' + (first_user_prefix if loop.first else "") }} |
| {%- if message['content'] is string -%} |
| {{ message['content'] | trim }} |
| {%- elif message['content'] is iterable -%} |
| {%- for item in message['content'] -%} |
| {%- if item['type'] == 'image' -%} |
| {{ '<|image>' }} |
| {%- elif item['type'] == 'text' -%} |
| {{ item['text'] | trim }} |
| {%- endif -%} |
| {%- endfor -%} |
| {%- else -%} |
| {{ raise_exception("Invalid content type") }} |
| {%- endif -%} |
| {{ '<turn|> |
| ' }} |
| {%- endfor -%} |
| {%- if add_generation_prompt -%} |
| {{'<|turn>model |
| '}} |
| {%- endif -%} |
| --- |
|
Tokenizing Prompts (num_proc=4): 0%| | 0/512 [00:00<?, ? examples/s]
Tokenizing Prompts (num_proc=4): 25%|██▌ | 128/512 [00:05<00:15, 24.03 examples/s]
Tokenizing Prompts (num_proc=4): 50%|█████ | 256/512 [00:08<00:08, 29.47 examples/s]
Tokenizing Prompts (num_proc=4): 75%|███████▌ | 384/512 [00:12<00:04, 31.28 examples/s]
Tokenizing Prompts (num_proc=4): 100%|██████████| 512/512 [00:16<00:00, 32.46 examples/s]
Tokenizing Prompts (num_proc=4): 100%|██████████| 512/512 [00:16<00:00, 30.72 examples/s] |
| [2026-09-05 09:13:26,886] [INFO] [axolotl.utils.data.utils._log_dataset_stats:212] [PID:59939] min_input_len: 104 |
| [2026-09-05 09:13:26,887] [INFO] [axolotl.utils.data.utils._log_dataset_stats:213] [PID:59939] max_input_len: 3139 |
|
Dropping Invalid Sequences (<None or >4096) (num_proc=4): 0%| | 0/512 [00:00<?, ? examples/s]
Dropping Invalid Sequences (<None or >4096) (num_proc=4): 25%|██▌ | 128/512 [00:00<00:00, 1252.97 examples/s]
Dropping Invalid Sequences (<None or >4096) (num_proc=4): 100%|██████████| 512/512 [00:00<00:00, 2257.43 examples/s] |
|
Saving the dataset (0/2 shards): 0%| | 0/512 [00:00<?, ? examples/s]
Saving the dataset (0/2 shards): 50%|█████ | 256/512 [00:32<00:32, 7.98 examples/s]
Saving the dataset (1/2 shards): 50%|█████ | 256/512 [00:32<00:32, 7.98 examples/s]
Saving the dataset (2/2 shards): 100%|██████████| 512/512 [00:32<00:00, 7.98 examples/s]
Saving the dataset (2/2 shards): 100%|██████████| 512/512 [00:33<00:00, 15.42 examples/s] |
| [2026-09-05 09:14:00,449] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:420] [PID:59939] total_num_tokens: 339_038 |
| [2026-09-05 09:14:00,463] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:438] [PID:59939] `total_supervised_tokens: 152_473` |
| [2026-09-05 09:14:00,463] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:521] [PID:59939] total_num_steps: 64 |
| [2026-09-05 09:14:00,464] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:59939] Maximum number of steps set at 64 |
| [2026-09-05 09:14:00,606] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:59939] loading tokenizer... /workspace/data/parents/mixed_4ep_prop |
| [2026-09-05 09:14:02,558] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:59939] EOS: 1 / <eos> |
| [2026-09-05 09:14:02,558] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:59939] BOS: 2 / <bos> |
| [2026-09-05 09:14:02,558] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:59939] PAD: 0 / <pad> |
| [2026-09-05 09:14:02,558] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:59939] UNK: 3 / <unk> |
| [2026-09-05 09:14:04,824] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:59939] Loading model |
| [2026-09-05 09:14:04,831] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:75] [PID:59939] Patched OptimState8bit for torch.compile compatibility |
| [2026-09-05 09:14:04,831] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:122] [PID:59939] Patched OptimState4bit for torch.compile compatibility |
| [2026-09-05 09:14:04,831] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:154] [PID:59939] Patched OptimStateFp8 for torch.compile compatibility |
| [2026-09-05 09:14:04,839] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:59939] Patched Trainer.evaluation_loop with nanmean loss calculation |
| [2026-09-05 09:14:04,841] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:59939] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation |
| [2026-09-05 09:14:04,991] [INFO] [axolotl.monkeypatch.attention.flash_attn_4.patch_flash_attn_4:52] [PID:59939] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 |
| [2026-09-05 09:14:05,198] [INFO] [axolotl.monkeypatch.models.gemma4.fused_attn.patch_gemma4_fused_attn:194] [PID:59939] Patched Gemma4TextAttention.forward with fused RMSNorm+RoPE Triton kernels |
| [2026-09-05 09:14:05,198] [INFO] [axolotl.monkeypatch.models.gemma4.fused_attn.patch_gemma4_fused_attn:198] [PID:59939] Installed Gemma4 shared_kv_states side channel (PR #3611) |
| [2026-09-05 09:14:05,241] [INFO] [axolotl.integrations.cut_cross_entropy.pre_model_load:94] [PID:59939] Applying Cut Cross Entropy to model type: gemma4 |
|
Loading weights: 0%| | 0/1189 [00:00<?, ?it/s]
Loading weights: 66%|██████▌ | 785/1189 [00:00<00:00, 7842.90it/s]
Loading weights: 100%|██████████| 1189/1189 [00:00<00:00, 8687.20it/s] |
| [2026-09-05 09:14:07,407] [INFO] [axolotl.monkeypatch.gemma4_hybrid_mask.patch_gemma4_hybrid_mask:94] [PID:59939] gemma4_hybrid_mask: patched modeling_gemma4.create_causal_mask to force SDPA-format masks for full-attention layers |
| [2026-09-05 09:14:07,408] [INFO] [axolotl.loaders.patch_manager._apply_gemma_hybrid_attention:243] [PID:59939] gemma4_hybrid_attn_impl: patched 10 global layers to use SDPA (remaining 50 sliding layers use flash_attention_2) |
| [2026-09-05 09:14:09,700] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:433] [PID:59939] Converting modules to torch.bfloat16 |
| [2026-09-05 09:14:11,414] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:59939] Memory usage after model load 0.000GB () |
| trainable params: 489,717,760 || all params: 31,762,804,272 || trainable%: 1.5418 |
| [2026-09-05 09:14:14,971] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:59939] after adapters 0.000GB () |
| [2026-09-05 09:14:28,539] [INFO] [axolotl.train.save_initial_configs:450] [PID:59939] Pre-saving adapter config to /workspace/eft31b_runs/dose_rows_v2/rows512/train/checkpoints... |
| [2026-09-05 09:14:28,543] [INFO] [axolotl.train.save_initial_configs:454] [PID:59939] Pre-saving tokenizer to /workspace/eft31b_runs/dose_rows_v2/rows512/train/checkpoints... |
| [2026-09-05 09:14:28,834] [INFO] [axolotl.train.save_initial_configs:459] [PID:59939] Pre-saving model config to /workspace/eft31b_runs/dose_rows_v2/rows512/train/checkpoints... |
| [2026-09-05 09:14:28,840] [INFO] [axolotl.train.save_initial_configs:463] [PID:59939] Pre-saving processor to /workspace/eft31b_runs/dose_rows_v2/rows512/train/checkpoints... |
| [2026-09-05 09:14:29,108] [INFO] [axolotl.train.execute_training:226] [PID:59939] Starting trainer... |
|
0%| | 0/64 [00:00<?, ?it/s][2026-09-05 09:14:54,528] [INFO] [axolotl.kernels.autotune_telemetry.on_step_end:133] [PID:59939] Reported 2 fused-rope kernel autotune config(s) to telemetry. |
|
2%|▏ | 1/64 [00:25<26:17, 25.03s/it]
{'loss': '0.5444', 'grad_norm': '5.777', 'learning_rate': '0', 'ppl': '1.724', 'memory/max_active (GiB)': '67.6', 'memory/max_allocated (GiB)': '67.6', 'memory/device_reserved (GiB)': '69.44', 'tokens/train_per_sec_per_gpu': '10.67', 'tokens/total': 18949, 'tokens/trainable': 7682, 'epoch': '0.0625'} |
|
2%|▏ | 1/64 [00:25<26:17, 25.03s/it]
3%|▎ | 2/64 [00:44<22:29, 21.77s/it]
{'loss': '0.5539', 'grad_norm': '15.27', 'learning_rate': '3.333e-05', 'ppl': '1.74', 'memory/max_active (GiB)': '69.7', 'memory/max_allocated (GiB)': '69.7', 'memory/device_reserved (GiB)': '71.42', 'tokens/train_per_sec_per_gpu': '30.32', 'tokens/total': 38342, 'tokens/trainable': 16648, 'epoch': '0.125'} |
|
3%|▎ | 2/64 [00:44<22:29, 21.77s/it]
5%|▍ | 3/64 [01:03<21:03, 20.71s/it]
{'loss': '0.4981', 'grad_norm': '3.005', 'learning_rate': '6.667e-05', 'ppl': '1.646', 'memory/max_active (GiB)': '70.04', 'memory/max_allocated (GiB)': '70.04', 'memory/device_reserved (GiB)': '71.73', 'tokens/train_per_sec_per_gpu': '14.91', 'tokens/total': 58793, 'tokens/trainable': 25149, 'epoch': '0.1875'} |
|
5%|▍ | 3/64 [01:03<21:03, 20.71s/it]
6%|▋ | 4/64 [01:24<20:42, 20.70s/it]
{'loss': '0.3588', 'grad_norm': '1.661', 'learning_rate': '0.0001', 'ppl': '1.432', 'memory/max_active (GiB)': '78.37', 'memory/max_allocated (GiB)': '78.37', 'memory/device_reserved (GiB)': '81.17', 'tokens/train_per_sec_per_gpu': '11.31', 'tokens/total': 82000, 'tokens/trainable': 37026, 'epoch': '0.25'} |
|
6%|▋ | 4/64 [01:24<20:42, 20.70s/it]
8%|▊ | 5/64 [01:43<19:51, 20.20s/it]
{'loss': '0.496', 'grad_norm': '15.24', 'learning_rate': '9.994e-05', 'ppl': '1.642', 'memory/max_active (GiB)': '71.5', 'memory/max_allocated (GiB)': '71.5', 'memory/device_reserved (GiB)': '81.17', 'tokens/train_per_sec_per_gpu': '14.61', 'tokens/total': 103034, 'tokens/trainable': 45945, 'epoch': '0.3125'} |
|
8%|▊ | 5/64 [01:43<19:51, 20.20s/it]
9%|▉ | 6/64 [02:03<19:12, 19.87s/it]
{'loss': '0.4163', 'grad_norm': '3.385', 'learning_rate': '9.976e-05', 'ppl': '1.516', 'memory/max_active (GiB)': '71.02', 'memory/max_allocated (GiB)': '71.02', 'memory/device_reserved (GiB)': '81.17', 'tokens/train_per_sec_per_gpu': '4.367', 'tokens/total': 122673, 'tokens/trainable': 54919, 'epoch': '0.375'} |
|
9%|▉ | 6/64 [02:03<19:12, 19.87s/it]
11%|█ | 7/64 [02:22<18:48, 19.80s/it]
{'loss': '0.3743', 'grad_norm': '35.89', 'learning_rate': '9.946e-05', 'ppl': '1.454', 'memory/max_active (GiB)': '72.78', 'memory/max_allocated (GiB)': '72.78', 'memory/device_reserved (GiB)': '81.17', 'tokens/train_per_sec_per_gpu': '13.9', 'tokens/total': 144789, 'tokens/trainable': 64564, 'epoch': '0.4375'} |
|
11%|█ | 7/64 [02:22<18:48, 19.80s/it]
12%|█▎ | 8/64 [02:42<18:25, 19.75s/it]
{'loss': '0.4224', 'grad_norm': '1.879', 'learning_rate': '9.905e-05', 'ppl': '1.526', 'memory/max_active (GiB)': '72.13', 'memory/max_allocated (GiB)': '72.13', 'memory/device_reserved (GiB)': '81.17', 'tokens/train_per_sec_per_gpu': '10.44', 'tokens/total': 166214, 'tokens/trainable': 74241, 'epoch': '0.5'} |
|
12%|█▎ | 8/64 [02:42<18:25, 19.75s/it]
14%|█▍ | 9/64 [03:01<17:52, 19.49s/it]
{'loss': '0.5003', 'grad_norm': '3.737', 'learning_rate': '9.852e-05', 'ppl': '1.649', 'memory/max_active (GiB)': '70.62', 'memory/max_allocated (GiB)': '70.62', 'memory/device_reserved (GiB)': '81.17', 'tokens/train_per_sec_per_gpu': '9.982', 'tokens/total': 187400, 'tokens/trainable': 84993, 'epoch': '0.5625'} |
|
14%|█▍ | 9/64 [03:01<17:52, 19.49s/it]
16%|█▌ | 10/64 [03:20<17:26, 19.38s/it]
{'loss': '0.3622', 'grad_norm': '2.118', 'learning_rate': '9.787e-05', 'ppl': '1.436', 'memory/max_active (GiB)': '70.1', 'memory/max_allocated (GiB)': '70.1', 'memory/device_reserved (GiB)': '81.17', 'tokens/train_per_sec_per_gpu': '5.702', 'tokens/total': 209029, 'tokens/trainable': 93515, 'epoch': '0.625'} |
|
16%|█▌ | 10/64 [03:20<17:26, 19.38s/it]
17%|█▋ | 11/64 [03:40<17:09, 19.42s/it]
{'loss': '0.3441', 'grad_norm': '14.44', 'learning_rate': '9.711e-05', 'ppl': '1.411', 'memory/max_active (GiB)': '72.48', 'memory/max_allocated (GiB)': '72.48', 'memory/device_reserved (GiB)': '81.17', 'tokens/train_per_sec_per_gpu': '9.732', 'tokens/total': 231592, 'tokens/trainable': 103909, 'epoch': '0.6875'} |
|
17%|█▋ | 11/64 [03:40<17:09, 19.42s/it]
19%|█▉ | 12/64 [03:59<16:49, 19.41s/it]
{'loss': '0.3248', 'grad_norm': '1.012', 'learning_rate': '9.623e-05', 'ppl': '1.384', 'memory/max_active (GiB)': '70.9', 'memory/max_allocated (GiB)': '70.9', 'memory/device_reserved (GiB)': '81.17', 'tokens/train_per_sec_per_gpu': '3.871', 'tokens/total': 252381, 'tokens/trainable': 112916, 'epoch': '0.75'} |
|
19%|█▉ | 12/64 [03:59<16:49, 19.41s/it]
20%|██ | 13/64 [04:19<16:41, 19.65s/it]
{'loss': '0.2996', 'grad_norm': '0.9506', 'learning_rate': '9.525e-05', 'ppl': '1.349', 'memory/max_active (GiB)': '71.88', 'memory/max_allocated (GiB)': '71.88', 'memory/device_reserved (GiB)': '81.17', 'tokens/train_per_sec_per_gpu': '9.861', 'tokens/total': 276170, 'tokens/trainable': 123411, 'epoch': '0.8125'} |
|
20%|██ | 13/64 [04:19<16:41, 19.65s/it]
22%|██▏ | 14/64 [04:38<16:11, 19.42s/it]
{'loss': '0.3252', 'grad_norm': '0.8923', 'learning_rate': '9.416e-05', 'ppl': '1.384', 'memory/max_active (GiB)': '70.85', 'memory/max_allocated (GiB)': '70.85', 'memory/device_reserved (GiB)': '81.17', 'tokens/train_per_sec_per_gpu': '12.81', 'tokens/total': 297462, 'tokens/trainable': 132960, 'epoch': '0.875'} |
|
22%|██▏ | 14/64 [04:38<16:11, 19.42s/it]
23%|██▎ | 15/64 [04:58<15:59, 19.58s/it]
{'loss': '0.34', 'grad_norm': '13.13', 'learning_rate': '9.297e-05', 'ppl': '1.405', 'memory/max_active (GiB)': '71.07', 'memory/max_allocated (GiB)': '71.07', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '7.073', 'tokens/total': 318085, 'tokens/trainable': 142483, 'epoch': '0.9375'} |
|
23%|██▎ | 15/64 [04:58<15:59, 19.58s/it]
25%|██▌ | 16/64 [05:18<15:47, 19.75s/it]
{'loss': '0.4197', 'grad_norm': '1.599', 'learning_rate': '9.168e-05', 'ppl': '1.522', 'memory/max_active (GiB)': '70.8', 'memory/max_allocated (GiB)': '70.8', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '26.08', 'tokens/total': 339038, 'tokens/trainable': 152473, 'epoch': '1'} |
|
25%|██▌ | 16/64 [05:18<15:47, 19.75s/it]
27%|██▋ | 17/64 [05:38<15:29, 19.78s/it]
{'loss': '0.2899', 'grad_norm': '1.154', 'learning_rate': '9.029e-05', 'ppl': '1.336', 'memory/max_active (GiB)': '70.38', 'memory/max_allocated (GiB)': '70.38', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '21.82', 'tokens/total': 360344, 'tokens/trainable': 162294, 'epoch': '1.062'} |
|
27%|██▋ | 17/64 [05:38<15:29, 19.78s/it]
28%|██▊ | 18/64 [05:57<14:59, 19.54s/it]
{'loss': '0.2572', 'grad_norm': '0.7001', 'learning_rate': '8.88e-05', 'ppl': '1.293', 'memory/max_active (GiB)': '70.48', 'memory/max_allocated (GiB)': '70.48', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '3.474', 'tokens/total': 381505, 'tokens/trainable': 171149, 'epoch': '1.125'} |
|
28%|██▊ | 18/64 [05:57<14:59, 19.54s/it]
30%|██▉ | 19/64 [06:15<14:25, 19.23s/it]
{'loss': '0.3382', 'grad_norm': '1.352', 'learning_rate': '8.723e-05', 'ppl': '1.402', 'memory/max_active (GiB)': '70.61', 'memory/max_allocated (GiB)': '70.61', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '8.279', 'tokens/total': 401977, 'tokens/trainable': 180241, 'epoch': '1.188'} |
|
30%|██▉ | 19/64 [06:15<14:25, 19.23s/it]
31%|███▏ | 20/64 [06:35<14:08, 19.28s/it]
{'loss': '0.2548', 'grad_norm': '2.883', 'learning_rate': '8.557e-05', 'ppl': '1.29', 'memory/max_active (GiB)': '72.13', 'memory/max_allocated (GiB)': '72.13', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '3.869', 'tokens/total': 423227, 'tokens/trainable': 189826, 'epoch': '1.25'} |
|
31%|███▏ | 20/64 [06:35<14:08, 19.28s/it]
33%|███▎ | 21/64 [06:54<13:49, 19.30s/it]
{'loss': '0.2809', 'grad_norm': '582.5', 'learning_rate': '8.383e-05', 'ppl': '1.324', 'memory/max_active (GiB)': '72.48', 'memory/max_allocated (GiB)': '72.48', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '2.379', 'tokens/total': 445224, 'tokens/trainable': 198550, 'epoch': '1.312'} |
|
33%|███▎ | 21/64 [06:54<13:49, 19.30s/it]
34%|███▍ | 22/64 [07:13<13:24, 19.16s/it]
{'loss': '0.2745', 'grad_norm': '5.241', 'learning_rate': '8.201e-05', 'ppl': '1.316', 'memory/max_active (GiB)': '70.77', 'memory/max_allocated (GiB)': '70.77', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '38.68', 'tokens/total': 465626, 'tokens/trainable': 208382, 'epoch': '1.375'} |
|
34%|███▍ | 22/64 [07:13<13:24, 19.16s/it]
36%|███▌ | 23/64 [07:32<13:07, 19.21s/it]
{'loss': '0.3234', 'grad_norm': '1.644', 'learning_rate': '8.012e-05', 'ppl': '1.382', 'memory/max_active (GiB)': '71.35', 'memory/max_allocated (GiB)': '71.35', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '6.151', 'tokens/total': 486368, 'tokens/trainable': 217581, 'epoch': '1.438'} |
|
36%|███▌ | 23/64 [07:32<13:07, 19.21s/it]
38%|███▊ | 24/64 [07:51<12:42, 19.05s/it]
{'loss': '0.2969', 'grad_norm': '0.6482', 'learning_rate': '7.817e-05', 'ppl': '1.346', 'memory/max_active (GiB)': '70.21', 'memory/max_allocated (GiB)': '70.21', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '14.99', 'tokens/total': 507015, 'tokens/trainable': 226751, 'epoch': '1.5'} |
|
38%|███▊ | 24/64 [07:51<12:42, 19.05s/it]
39%|███▉ | 25/64 [08:11<12:33, 19.33s/it]
{'loss': '0.3383', 'grad_norm': '5.75', 'learning_rate': '7.615e-05', 'ppl': '1.403', 'memory/max_active (GiB)': '72.78', 'memory/max_allocated (GiB)': '72.78', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '12.02', 'tokens/total': 530979, 'tokens/trainable': 239008, 'epoch': '1.562'} |
|
39%|███▉ | 25/64 [08:11<12:33, 19.33s/it]
41%|████ | 26/64 [08:30<12:07, 19.14s/it]
{'loss': '0.2582', 'grad_norm': '3.245', 'learning_rate': '7.408e-05', 'ppl': '1.295', 'memory/max_active (GiB)': '69.78', 'memory/max_allocated (GiB)': '69.78', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '9.248', 'tokens/total': 551197, 'tokens/trainable': 248343, 'epoch': '1.625'} |
|
41%|████ | 26/64 [08:30<12:07, 19.14s/it]
42%|████▏ | 27/64 [08:49<11:44, 19.04s/it]
{'loss': '0.3365', 'grad_norm': '0.7615', 'learning_rate': '7.195e-05', 'ppl': '1.4', 'memory/max_active (GiB)': '70.61', 'memory/max_allocated (GiB)': '70.61', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '13.31', 'tokens/total': 573414, 'tokens/trainable': 259489, 'epoch': '1.688'} |
|
42%|████▏ | 27/64 [08:49<11:44, 19.04s/it]
44%|████▍ | 28/64 [09:08<11:27, 19.09s/it]
{'loss': '0.2393', 'grad_norm': '2.69', 'learning_rate': '6.978e-05', 'ppl': '1.27', 'memory/max_active (GiB)': '71.02', 'memory/max_allocated (GiB)': '71.02', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '10.67', 'tokens/total': 595006, 'tokens/trainable': 268962, 'epoch': '1.75'} |
|
44%|████▍ | 28/64 [09:08<11:27, 19.09s/it]
45%|████▌ | 29/64 [09:28<11:22, 19.50s/it]
{'loss': '0.2838', 'grad_norm': '1.12', 'learning_rate': '6.758e-05', 'ppl': '1.328', 'memory/max_active (GiB)': '78.37', 'memory/max_allocated (GiB)': '78.37', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '21.84', 'tokens/total': 618258, 'tokens/trainable': 280691, 'epoch': '1.812'} |
|
45%|████▌ | 29/64 [09:28<11:22, 19.50s/it]
47%|████▋ | 30/64 [09:47<10:57, 19.33s/it]
{'loss': '0.269', 'grad_norm': '2.455', 'learning_rate': '6.534e-05', 'ppl': '1.309', 'memory/max_active (GiB)': '71.7', 'memory/max_allocated (GiB)': '71.7', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '15.96', 'tokens/total': 638307, 'tokens/trainable': 288561, 'epoch': '1.875'} |
|
47%|████▋ | 30/64 [09:47<10:57, 19.33s/it]
48%|████▊ | 31/64 [10:06<10:31, 19.14s/it]
{'loss': '0.2551', 'grad_norm': '2.482', 'learning_rate': '6.307e-05', 'ppl': '1.291', 'memory/max_active (GiB)': '70.32', 'memory/max_allocated (GiB)': '70.32', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '6.847', 'tokens/total': 657152, 'tokens/trainable': 296385, 'epoch': '1.938'} |
|
48%|████▊ | 31/64 [10:06<10:31, 19.14s/it]
50%|█████ | 32/64 [10:25<10:11, 19.12s/it]
{'loss': '0.2861', 'grad_norm': '1.026', 'learning_rate': '6.078e-05', 'ppl': '1.331', 'memory/max_active (GiB)': '70.86', 'memory/max_allocated (GiB)': '70.86', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '33.09', 'tokens/total': 678076, 'tokens/trainable': 304946, 'epoch': '2'} |
|
50%|█████ | 32/64 [10:25<10:11, 19.12s/it]
52%|█████▏ | 33/64 [10:45<10:04, 19.51s/it]
{'loss': '0.1803', 'grad_norm': '1.762', 'learning_rate': '5.847e-05', 'ppl': '1.198', 'memory/max_active (GiB)': '70.86', 'memory/max_allocated (GiB)': '70.86', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '25.39', 'tokens/total': 698592, 'tokens/trainable': 314059, 'epoch': '2.062'} |
|
52%|█████▏ | 33/64 [10:45<10:04, 19.51s/it]
53%|█████▎ | 34/64 [11:04<09:37, 19.26s/it]
{'loss': '0.2389', 'grad_norm': '1.057', 'learning_rate': '5.616e-05', 'ppl': '1.27', 'memory/max_active (GiB)': '70.16', 'memory/max_allocated (GiB)': '70.16', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '6.649', 'tokens/total': 718262, 'tokens/trainable': 322819, 'epoch': '2.125'} |
|
53%|█████▎ | 34/64 [11:04<09:37, 19.26s/it]
55%|█████▍ | 35/64 [11:23<09:15, 19.14s/it]
{'loss': '0.2255', 'grad_norm': '1.355', 'learning_rate': '5.384e-05', 'ppl': '1.253', 'memory/max_active (GiB)': '72.13', 'memory/max_allocated (GiB)': '72.13', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '7.58', 'tokens/total': 739018, 'tokens/trainable': 332299, 'epoch': '2.188'} |
|
55%|█████▍ | 35/64 [11:23<09:15, 19.14s/it]
56%|█████▋ | 36/64 [11:42<08:54, 19.09s/it]
{'loss': '0.2085', 'grad_norm': '1.474', 'learning_rate': '5.153e-05', 'ppl': '1.232', 'memory/max_active (GiB)': '70.85', 'memory/max_allocated (GiB)': '70.85', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '9.543', 'tokens/total': 760017, 'tokens/trainable': 341460, 'epoch': '2.25'} |
|
56%|█████▋ | 36/64 [11:42<08:54, 19.09s/it]
58%|█████▊ | 37/64 [12:01<08:33, 19.04s/it]
{'loss': '0.2352', 'grad_norm': '0.8865', 'learning_rate': '4.922e-05', 'ppl': '1.265', 'memory/max_active (GiB)': '71.7', 'memory/max_allocated (GiB)': '71.7', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '21.53', 'tokens/total': 781916, 'tokens/trainable': 351355, 'epoch': '2.312'} |
|
58%|█████▊ | 37/64 [12:01<08:33, 19.04s/it]
59%|█████▉ | 38/64 [12:20<08:13, 18.98s/it]
{'loss': '0.2611', 'grad_norm': '1.828', 'learning_rate': '4.693e-05', 'ppl': '1.298', 'memory/max_active (GiB)': '70.04', 'memory/max_allocated (GiB)': '70.04', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '3.238', 'tokens/total': 802181, 'tokens/trainable': 359732, 'epoch': '2.375'} |
|
59%|█████▉ | 38/64 [12:20<08:13, 18.98s/it]
61%|██████ | 39/64 [12:38<07:53, 18.94s/it]
{'loss': '0.2062', 'grad_norm': '2.93', 'learning_rate': '4.466e-05', 'ppl': '1.229', 'memory/max_active (GiB)': '71.02', 'memory/max_allocated (GiB)': '71.02', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '29.57', 'tokens/total': 821689, 'tokens/trainable': 369538, 'epoch': '2.438'} |
|
61%|██████ | 39/64 [12:38<07:53, 18.94s/it]
62%|██████▎ | 40/64 [12:57<07:33, 18.89s/it]
{'loss': '0.2282', 'grad_norm': '1.265', 'learning_rate': '4.242e-05', 'ppl': '1.256', 'memory/max_active (GiB)': '70.32', 'memory/max_allocated (GiB)': '70.32', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '7.191', 'tokens/total': 843029, 'tokens/trainable': 379290, 'epoch': '2.5'} |
|
62%|██████▎ | 40/64 [12:57<07:33, 18.89s/it]
64%|██████▍ | 41/64 [13:17<07:20, 19.13s/it]
{'loss': '0.1848', 'grad_norm': '2.291', 'learning_rate': '4.022e-05', 'ppl': '1.203', 'memory/max_active (GiB)': '72.78', 'memory/max_allocated (GiB)': '72.78', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '10.87', 'tokens/total': 866092, 'tokens/trainable': 389494, 'epoch': '2.562'} |
|
64%|██████▍ | 41/64 [13:17<07:20, 19.13s/it]
66%|██████▌ | 42/64 [13:36<07:00, 19.10s/it]
{'loss': '0.1871', 'grad_norm': '3.856', 'learning_rate': '3.805e-05', 'ppl': '1.206', 'memory/max_active (GiB)': '71.35', 'memory/max_allocated (GiB)': '71.35', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '4.051', 'tokens/total': 886561, 'tokens/trainable': 398022, 'epoch': '2.625'} |
|
66%|██████▌ | 42/64 [13:36<07:00, 19.10s/it]
67%|██████▋ | 43/64 [13:55<06:41, 19.11s/it]
{'loss': '0.2673', 'grad_norm': '1.933', 'learning_rate': '3.592e-05', 'ppl': '1.306', 'memory/max_active (GiB)': '71.63', 'memory/max_allocated (GiB)': '71.63', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '3.137', 'tokens/total': 908071, 'tokens/trainable': 407054, 'epoch': '2.688'} |
|
67%|██████▋ | 43/64 [13:55<06:41, 19.11s/it]
69%|██████▉ | 44/64 [14:15<06:25, 19.27s/it]
{'loss': '0.2651', 'grad_norm': '1.731', 'learning_rate': '3.385e-05', 'ppl': '1.304', 'memory/max_active (GiB)': '71.25', 'memory/max_allocated (GiB)': '71.25', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '21.65', 'tokens/total': 929740, 'tokens/trainable': 417850, 'epoch': '2.75'} |
|
69%|██████▉ | 44/64 [14:15<06:25, 19.27s/it]
70%|███████ | 45/64 [14:34<06:04, 19.19s/it]
{'loss': '0.2483', 'grad_norm': '1.053', 'learning_rate': '3.183e-05', 'ppl': '1.282', 'memory/max_active (GiB)': '70.86', 'memory/max_allocated (GiB)': '70.86', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '10.99', 'tokens/total': 949830, 'tokens/trainable': 426621, 'epoch': '2.812'} |
|
70%|███████ | 45/64 [14:34<06:04, 19.19s/it]
72%|███████▏ | 46/64 [14:53<05:47, 19.30s/it]
{'loss': '0.2102', 'grad_norm': '6.798', 'learning_rate': '2.988e-05', 'ppl': '1.234', 'memory/max_active (GiB)': '71.07', 'memory/max_allocated (GiB)': '71.07', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '5.732', 'tokens/total': 972638, 'tokens/trainable': 436683, 'epoch': '2.875'} |
|
72%|███████▏ | 46/64 [14:53<05:47, 19.30s/it]
73%|███████▎ | 47/64 [15:12<05:26, 19.18s/it]
{'loss': '0.1847', 'grad_norm': '4.911', 'learning_rate': '2.799e-05', 'ppl': '1.203', 'memory/max_active (GiB)': '70.9', 'memory/max_allocated (GiB)': '70.9', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '12.85', 'tokens/total': 992954, 'tokens/trainable': 444773, 'epoch': '2.938'} |
|
73%|███████▎ | 47/64 [15:12<05:26, 19.18s/it]
75%|███████▌ | 48/64 [15:33<05:16, 19.78s/it]
{'loss': '0.2337', 'grad_norm': '1.143', 'learning_rate': '2.617e-05', 'ppl': '1.263', 'memory/max_active (GiB)': '78.37', 'memory/max_allocated (GiB)': '78.37', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '8.888', 'tokens/total': 1017114, 'tokens/trainable': 457419, 'epoch': '3'} |
|
75%|███████▌ | 48/64 [15:33<05:16, 19.78s/it]
77%|███████▋ | 49/64 [15:54<04:59, 19.98s/it]
{'loss': '0.1369', 'grad_norm': '0.796', 'learning_rate': '2.443e-05', 'ppl': '1.147', 'memory/max_active (GiB)': '70.8', 'memory/max_allocated (GiB)': '70.8', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '15.25', 'tokens/total': 1037823, 'tokens/trainable': 466311, 'epoch': '3.062'} |
|
77%|███████▋ | 49/64 [15:54<04:59, 19.98s/it]
78%|███████▊ | 50/64 [16:13<04:35, 19.67s/it]
{'loss': '0.1947', 'grad_norm': '1.118', 'learning_rate': '2.277e-05', 'ppl': '1.215', 'memory/max_active (GiB)': '71.7', 'memory/max_allocated (GiB)': '71.7', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '27.55', 'tokens/total': 1058482, 'tokens/trainable': 475960, 'epoch': '3.125'} |
|
78%|███████▊ | 50/64 [16:13<04:35, 19.67s/it]
80%|███████▉ | 51/64 [16:32<04:13, 19.49s/it]
{'loss': '0.1329', 'grad_norm': '26.31', 'learning_rate': '2.12e-05', 'ppl': '1.142', 'memory/max_active (GiB)': '70.86', 'memory/max_allocated (GiB)': '70.86', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '9.971', 'tokens/total': 1078561, 'tokens/trainable': 484604, 'epoch': '3.188'} |
|
80%|███████▉ | 51/64 [16:32<04:13, 19.49s/it]
81%|████████▏ | 52/64 [16:51<03:52, 19.38s/it]
{'loss': '0.1959', 'grad_norm': '1.293', 'learning_rate': '1.971e-05', 'ppl': '1.216', 'memory/max_active (GiB)': '71.25', 'memory/max_allocated (GiB)': '71.25', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '11.81', 'tokens/total': 1099878, 'tokens/trainable': 494786, 'epoch': '3.25'} |
|
81%|████████▏ | 52/64 [16:51<03:52, 19.38s/it]
83%|████████▎ | 53/64 [17:10<03:31, 19.24s/it]
{'loss': '0.1558', 'grad_norm': '0.6817', 'learning_rate': '1.832e-05', 'ppl': '1.169', 'memory/max_active (GiB)': '71.63', 'memory/max_allocated (GiB)': '71.63', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '37.54', 'tokens/total': 1122069, 'tokens/trainable': 504202, 'epoch': '3.312'} |
|
83%|████████▎ | 53/64 [17:10<03:31, 19.24s/it]
84%|████████▍ | 54/64 [17:29<03:11, 19.12s/it]
{'loss': '0.1703', 'grad_norm': '2.15', 'learning_rate': '1.703e-05', 'ppl': '1.186', 'memory/max_active (GiB)': '71.35', 'memory/max_allocated (GiB)': '71.35', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '11.11', 'tokens/total': 1141363, 'tokens/trainable': 511920, 'epoch': '3.375'} |
|
84%|████████▍ | 54/64 [17:29<03:11, 19.12s/it]
86%|████████▌ | 55/64 [17:48<02:53, 19.23s/it]
{'loss': '0.1685', 'grad_norm': '0.8789', 'learning_rate': '1.584e-05', 'ppl': '1.183', 'memory/max_active (GiB)': '72.78', 'memory/max_allocated (GiB)': '72.78', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '3.851', 'tokens/total': 1164043, 'tokens/trainable': 522944, 'epoch': '3.438'} |
|
86%|████████▌ | 55/64 [17:48<02:53, 19.23s/it]
88%|████████▊ | 56/64 [18:08<02:36, 19.53s/it]
{'loss': '0.1795', 'grad_norm': '2.179', 'learning_rate': '1.475e-05', 'ppl': '1.197', 'memory/max_active (GiB)': '78.37', 'memory/max_allocated (GiB)': '78.37', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '30.27', 'tokens/total': 1187482, 'tokens/trainable': 533763, 'epoch': '3.5'} |
|
88%|████████▊ | 56/64 [18:08<02:36, 19.53s/it]
89%|████████▉ | 57/64 [18:27<02:15, 19.30s/it]
{'loss': '0.1668', 'grad_norm': '1.579', 'learning_rate': '1.377e-05', 'ppl': '1.181', 'memory/max_active (GiB)': '70.32', 'memory/max_allocated (GiB)': '70.32', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '2.025', 'tokens/total': 1207986, 'tokens/trainable': 542583, 'epoch': '3.562'} |
|
89%|████████▉ | 57/64 [18:27<02:15, 19.30s/it]
91%|█████████ | 58/64 [18:46<01:55, 19.22s/it]
{'loss': '0.1558', 'grad_norm': '1.421', 'learning_rate': '1.289e-05', 'ppl': '1.169', 'memory/max_active (GiB)': '72.48', 'memory/max_allocated (GiB)': '72.48', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '11.42', 'tokens/total': 1228303, 'tokens/trainable': 551886, 'epoch': '3.625'} |
|
91%|█████████ | 58/64 [18:46<01:55, 19.22s/it]
92%|█████████▏| 59/64 [19:05<01:35, 19.01s/it]
{'loss': '0.1949', 'grad_norm': '1.786', 'learning_rate': '1.213e-05', 'ppl': '1.215', 'memory/max_active (GiB)': '70.04', 'memory/max_allocated (GiB)': '70.04', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '19.7', 'tokens/total': 1246283, 'tokens/trainable': 559997, 'epoch': '3.688'} |
|
92%|█████████▏| 59/64 [19:05<01:35, 19.01s/it]
94%|█████████▍| 60/64 [19:24<01:16, 19.14s/it]
{'loss': '0.1658', 'grad_norm': '1.611', 'learning_rate': '1.148e-05', 'ppl': '1.18', 'memory/max_active (GiB)': '72.13', 'memory/max_allocated (GiB)': '72.13', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '8.08', 'tokens/total': 1270523, 'tokens/trainable': 569759, 'epoch': '3.75'} |
|
94%|█████████▍| 60/64 [19:24<01:16, 19.14s/it]
95%|█████████▌| 61/64 [19:43<00:57, 19.04s/it]
{'loss': '0.3363', 'grad_norm': '3.11', 'learning_rate': '1.095e-05', 'ppl': '1.4', 'memory/max_active (GiB)': '70.9', 'memory/max_allocated (GiB)': '70.9', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '14.03', 'tokens/total': 1293319, 'tokens/trainable': 582167, 'epoch': '3.812'} |
|
95%|█████████▌| 61/64 [19:43<00:57, 19.04s/it]
97%|█████████▋| 62/64 [20:02<00:38, 19.04s/it]
{'loss': '0.1499', 'grad_norm': '1.247', 'learning_rate': '1.054e-05', 'ppl': '1.162', 'memory/max_active (GiB)': '71.02', 'memory/max_allocated (GiB)': '71.02', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '40.21', 'tokens/total': 1313661, 'tokens/trainable': 590725, 'epoch': '3.875'} |
|
97%|█████████▋| 62/64 [20:02<00:38, 19.04s/it]
98%|█████████▊| 63/64 [20:21<00:18, 18.90s/it]
{'loss': '0.1347', 'grad_norm': '3.047', 'learning_rate': '1.024e-05', 'ppl': '1.144', 'memory/max_active (GiB)': '70.85', 'memory/max_allocated (GiB)': '70.85', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '2.587', 'tokens/total': 1332967, 'tokens/trainable': 599904, 'epoch': '3.938'} |
|
98%|█████████▊| 63/64 [20:21<00:18, 18.90s/it]
100%|██████████| 64/64 [20:40<00:00, 19.13s/it]
{'loss': '0.1617', 'grad_norm': '14.64', 'learning_rate': '1.006e-05', 'ppl': '1.176', 'memory/max_active (GiB)': '71.07', 'memory/max_allocated (GiB)': '71.07', 'memory/device_reserved (GiB)': '81.18', 'tokens/train_per_sec_per_gpu': '6.541', 'tokens/total': 1356152, 'tokens/trainable': 609892, 'epoch': '4'} |
|
100%|██████████| 64/64 [20:40<00:00, 19.13s/it][2026-09-05 09:35:10,228] [INFO] [axolotl.core.trainers.base._save:828] [PID:59939] Saving model checkpoint to /workspace/eft31b_runs/dose_rows_v2/rows512/train/checkpoints/checkpoint-64 |
|
{'train_runtime': '1245', 'train_samples_per_second': '1.645', 'train_steps_per_second': '0.051', 'train_loss': '0.2739', 'memory/max_active (GiB)': '63.86', 'memory/max_allocated (GiB)': '63.86', 'memory/device_reserved (GiB)': '81.18', 'epoch': '4', 'tokens/train_per_sec_per_gpu': '0'} |
|
100%|██████████| 64/64 [20:45<00:00, 19.13s/it]
100%|██████████| 64/64 [20:45<00:00, 19.46s/it] |
| [2026-09-05 09:35:14,807] [INFO] [axolotl.train.save_trained_model:267] [PID:59939] Training completed! Saving trained model to /workspace/eft31b_runs/dose_rows_v2/rows512/train/checkpoints. |
| [2026-09-05 09:35:18,186] [INFO] [axolotl.train.save_trained_model:388] [PID:59939] Model successfully saved to /workspace/eft31b_runs/dose_rows_v2/rows512/train/checkpoints |
|
|
