File size: 54,244 Bytes
f56e17a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 | [2026-09-05 08:56:31,883] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:58343] baseline 0.000GB ()
[2026-09-05 08:56:31,885] [INFO] [axolotl.cli.config.load_cfg:333] [PID:58343] config:
{
"activation_offloading": false,
"adapter": "lora",
"attn_implementation": "flash_attention_2",
"attn_needs_dtype_cast": true,
"attn_supports_packing": true,
"attn_uses_flash_lib": true,
"axolotl_config_path": "/workspace/eft31b_runs/dose_rows_v2/rows256/train/axolotl.yaml",
"base_model": "/workspace/data/parents/mixed_4ep_prop",
"base_model_config": "/workspace/data/parents/mixed_4ep_prop",
"batch_size": 32,
"bf16": true,
"capabilities": {
"bf16": true,
"compute_capability": "sm_90",
"fp8": true,
"n_gpu": 1,
"n_node": 1,
"tf32": true
},
"chat_template": "jinja",
"chat_template_jinja": "/workspace/work/src/scimt/train/stages/assets/gemma4_chat_template.jinja",
"checkpoint_schedule": [
32
],
"context_parallel_size": 1,
"cosine_min_lr_ratio": 0.1,
"cut_cross_entropy": true,
"dataloader_num_workers": 1,
"dataloader_pin_memory": true,
"dataloader_prefetch_factor": 256,
"dataset_num_proc": 4,
"dataset_prepared_path": "/workspace/eft31b_runs/dose_rows_v2/rows256/train/prepared",
"datasets": [
{
"chat_template": "tokenizer_default",
"field_messages": "messages",
"message_property_mappings": {
"content": "content",
"role": "role"
},
"path": "/workspace/eft31b_runs/dose_rows_v2/rows256/eft_training.jsonl",
"trust_remote_code": false,
"type": "chat_template"
}
],
"ddp": false,
"ddp_find_unused_parameters": true,
"device": "cuda:0",
"dion_rank_fraction": 1.0,
"dion_rank_multiple_of": 1,
"eaft_alpha": 1.0,
"eaft_k": 20,
"env_capabilities": {
"torch_version": "2.12.1"
},
"eot_tokens": [
"<turn|>"
],
"eval_batch_size": 1,
"eval_causal_lm_metrics": [
"sacrebleu",
"comet",
"ter",
"chrf"
],
"eval_max_new_tokens": 128,
"eval_table_size": 0,
"experimental_skip_move_to_device": true,
"fp16": false,
"gemma4_hybrid_attn_impl": true,
"generate_samples": false,
"generation_do_sample": true,
"generation_max_new_tokens": 50,
"generation_prompt_ratio": 0.5,
"generation_temperature": 0.7,
"gradient_accumulation_steps": 32,
"gradient_checkpointing": true,
"gradient_checkpointing_kwargs": {
"use_reentrant": false
},
"include_tkps": true,
"is_multimodal": true,
"layer_offloading": false,
"learning_rate": 0.0001,
"lisa_layers_attribute": "model.layers",
"load_best_model_at_end": false,
"load_in_4bit": false,
"load_in_8bit": false,
"local_rank": 0,
"logging_steps": 1,
"lora_alpha": 128,
"lora_dropout": 0.0,
"lora_mlp_kernel": false,
"lora_o_kernel": false,
"lora_qkv_kernel": false,
"lora_r": 64,
"lora_target_modules": [
"model.language_model.layers.0.self_attn.q_proj",
"model.language_model.layers.0.self_attn.k_proj",
"model.language_model.layers.0.self_attn.v_proj",
"model.language_model.layers.0.self_attn.o_proj",
"model.language_model.layers.0.mlp.gate_proj",
"model.language_model.layers.0.mlp.up_proj",
"model.language_model.layers.0.mlp.down_proj",
"model.language_model.layers.1.self_attn.q_proj",
"model.language_model.layers.1.self_attn.k_proj",
"model.language_model.layers.1.self_attn.v_proj",
"model.language_model.layers.1.self_attn.o_proj",
"model.language_model.layers.1.mlp.gate_proj",
"model.language_model.layers.1.mlp.up_proj",
"model.language_model.layers.1.mlp.down_proj",
"model.language_model.layers.2.self_attn.q_proj",
"model.language_model.layers.2.self_attn.k_proj",
"model.language_model.layers.2.self_attn.v_proj",
"model.language_model.layers.2.self_attn.o_proj",
"model.language_model.layers.2.mlp.gate_proj",
"model.language_model.layers.2.mlp.up_proj",
"model.language_model.layers.2.mlp.down_proj",
"model.language_model.layers.3.self_attn.q_proj",
"model.language_model.layers.3.self_attn.k_proj",
"model.language_model.layers.3.self_attn.v_proj",
"model.language_model.layers.3.self_attn.o_proj",
"model.language_model.layers.3.mlp.gate_proj",
"model.language_model.layers.3.mlp.up_proj",
"model.language_model.layers.3.mlp.down_proj",
"model.language_model.layers.4.self_attn.q_proj",
"model.language_model.layers.4.self_attn.k_proj",
"model.language_model.layers.4.self_attn.v_proj",
"model.language_model.layers.4.self_attn.o_proj",
"model.language_model.layers.4.mlp.gate_proj",
"model.language_model.layers.4.mlp.up_proj",
"model.language_model.layers.4.mlp.down_proj",
"model.language_model.layers.5.self_attn.q_proj",
"model.language_model.layers.5.self_attn.k_proj",
"model.language_model.layers.5.self_attn.o_proj",
"model.language_model.layers.5.mlp.gate_proj",
"model.language_model.layers.5.mlp.up_proj",
"model.language_model.layers.5.mlp.down_proj",
"model.language_model.layers.6.self_attn.q_proj",
"model.language_model.layers.6.self_attn.k_proj",
"model.language_model.layers.6.self_attn.v_proj",
"model.language_model.layers.6.self_attn.o_proj",
"model.language_model.layers.6.mlp.gate_proj",
"model.language_model.layers.6.mlp.up_proj",
"model.language_model.layers.6.mlp.down_proj",
"model.language_model.layers.7.self_attn.q_proj",
"model.language_model.layers.7.self_attn.k_proj",
"model.language_model.layers.7.self_attn.v_proj",
"model.language_model.layers.7.self_attn.o_proj",
"model.language_model.layers.7.mlp.gate_proj",
"model.language_model.layers.7.mlp.up_proj",
"model.language_model.layers.7.mlp.down_proj",
"model.language_model.layers.8.self_attn.q_proj",
"model.language_model.layers.8.self_attn.k_proj",
"model.language_model.layers.8.self_attn.v_proj",
"model.language_model.layers.8.self_attn.o_proj",
"model.language_model.layers.8.mlp.gate_proj",
"model.language_model.layers.8.mlp.up_proj",
"model.language_model.layers.8.mlp.down_proj",
"model.language_model.layers.9.self_attn.q_proj",
"model.language_model.layers.9.self_attn.k_proj",
"model.language_model.layers.9.self_attn.v_proj",
"model.language_model.layers.9.self_attn.o_proj",
"model.language_model.layers.9.mlp.gate_proj",
"model.language_model.layers.9.mlp.up_proj",
"model.language_model.layers.9.mlp.down_proj",
"model.language_model.layers.10.self_attn.q_proj",
"model.language_model.layers.10.self_attn.k_proj",
"model.language_model.layers.10.self_attn.v_proj",
"model.language_model.layers.10.self_attn.o_proj",
"model.language_model.layers.10.mlp.gate_proj",
"model.language_model.layers.10.mlp.up_proj",
"model.language_model.layers.10.mlp.down_proj",
"model.language_model.layers.11.self_attn.q_proj",
"model.language_model.layers.11.self_attn.k_proj",
"model.language_model.layers.11.self_attn.o_proj",
"model.language_model.layers.11.mlp.gate_proj",
"model.language_model.layers.11.mlp.up_proj",
"model.language_model.layers.11.mlp.down_proj",
"model.language_model.layers.12.self_attn.q_proj",
"model.language_model.layers.12.self_attn.k_proj",
"model.language_model.layers.12.self_attn.v_proj",
"model.language_model.layers.12.self_attn.o_proj",
"model.language_model.layers.12.mlp.gate_proj",
"model.language_model.layers.12.mlp.up_proj",
"model.language_model.layers.12.mlp.down_proj",
"model.language_model.layers.13.self_attn.q_proj",
"model.language_model.layers.13.self_attn.k_proj",
"model.language_model.layers.13.self_attn.v_proj",
"model.language_model.layers.13.self_attn.o_proj",
"model.language_model.layers.13.mlp.gate_proj",
"model.language_model.layers.13.mlp.up_proj",
"model.language_model.layers.13.mlp.down_proj",
"model.language_model.layers.14.self_attn.q_proj",
"model.language_model.layers.14.self_attn.k_proj",
"model.language_model.layers.14.self_attn.v_proj",
"model.language_model.layers.14.self_attn.o_proj",
"model.language_model.layers.14.mlp.gate_proj",
"model.language_model.layers.14.mlp.up_proj",
"model.language_model.layers.14.mlp.down_proj",
"model.language_model.layers.15.self_attn.q_proj",
"model.language_model.layers.15.self_attn.k_proj",
"model.language_model.layers.15.self_attn.v_proj",
"model.language_model.layers.15.self_attn.o_proj",
"model.language_model.layers.15.mlp.gate_proj",
"model.language_model.layers.15.mlp.up_proj",
"model.language_model.layers.15.mlp.down_proj",
"model.language_model.layers.16.self_attn.q_proj",
"model.language_model.layers.16.self_attn.k_proj",
"model.language_model.layers.16.self_attn.v_proj",
"model.language_model.layers.16.self_attn.o_proj",
"model.language_model.layers.16.mlp.gate_proj",
"model.language_model.layers.16.mlp.up_proj",
"model.language_model.layers.16.mlp.down_proj",
"model.language_model.layers.17.self_attn.q_proj",
"model.language_model.layers.17.self_attn.k_proj",
"model.language_model.layers.17.self_attn.o_proj",
"model.language_model.layers.17.mlp.gate_proj",
"model.language_model.layers.17.mlp.up_proj",
"model.language_model.layers.17.mlp.down_proj",
"model.language_model.layers.18.self_attn.q_proj",
"model.language_model.layers.18.self_attn.k_proj",
"model.language_model.layers.18.self_attn.v_proj",
"model.language_model.layers.18.self_attn.o_proj",
"model.language_model.layers.18.mlp.gate_proj",
"model.language_model.layers.18.mlp.up_proj",
"model.language_model.layers.18.mlp.down_proj",
"model.language_model.layers.19.self_attn.q_proj",
"model.language_model.layers.19.self_attn.k_proj",
"model.language_model.layers.19.self_attn.v_proj",
"model.language_model.layers.19.self_attn.o_proj",
"model.language_model.layers.19.mlp.gate_proj",
"model.language_model.layers.19.mlp.up_proj",
"model.language_model.layers.19.mlp.down_proj",
"model.language_model.layers.20.self_attn.q_proj",
"model.language_model.layers.20.self_attn.k_proj",
"model.language_model.layers.20.self_attn.v_proj",
"model.language_model.layers.20.self_attn.o_proj",
"model.language_model.layers.20.mlp.gate_proj",
"model.language_model.layers.20.mlp.up_proj",
"model.language_model.layers.20.mlp.down_proj",
"model.language_model.layers.21.self_attn.q_proj",
"model.language_model.layers.21.self_attn.k_proj",
"model.language_model.layers.21.self_attn.v_proj",
"model.language_model.layers.21.self_attn.o_proj",
"model.language_model.layers.21.mlp.gate_proj",
"model.language_model.layers.21.mlp.up_proj",
"model.language_model.layers.21.mlp.down_proj",
"model.language_model.layers.22.self_attn.q_proj",
"model.language_model.layers.22.self_attn.k_proj",
"model.language_model.layers.22.self_attn.v_proj",
"model.language_model.layers.22.self_attn.o_proj",
"model.language_model.layers.22.mlp.gate_proj",
"model.language_model.layers.22.mlp.up_proj",
"model.language_model.layers.22.mlp.down_proj",
"model.language_model.layers.23.self_attn.q_proj",
"model.language_model.layers.23.self_attn.k_proj",
"model.language_model.layers.23.self_attn.o_proj",
"model.language_model.layers.23.mlp.gate_proj",
"model.language_model.layers.23.mlp.up_proj",
"model.language_model.layers.23.mlp.down_proj",
"model.language_model.layers.24.self_attn.q_proj",
"model.language_model.layers.24.self_attn.k_proj",
"model.language_model.layers.24.self_attn.v_proj",
"model.language_model.layers.24.self_attn.o_proj",
"model.language_model.layers.24.mlp.gate_proj",
"model.language_model.layers.24.mlp.up_proj",
"model.language_model.layers.24.mlp.down_proj",
"model.language_model.layers.25.self_attn.q_proj",
"model.language_model.layers.25.self_attn.k_proj",
"model.language_model.layers.25.self_attn.v_proj",
"model.language_model.layers.25.self_attn.o_proj",
"model.language_model.layers.25.mlp.gate_proj",
"model.language_model.layers.25.mlp.up_proj",
"model.language_model.layers.25.mlp.down_proj",
"model.language_model.layers.26.self_attn.q_proj",
"model.language_model.layers.26.self_attn.k_proj",
"model.language_model.layers.26.self_attn.v_proj",
"model.language_model.layers.26.self_attn.o_proj",
"model.language_model.layers.26.mlp.gate_proj",
"model.language_model.layers.26.mlp.up_proj",
"model.language_model.layers.26.mlp.down_proj",
"model.language_model.layers.27.self_attn.q_proj",
"model.language_model.layers.27.self_attn.k_proj",
"model.language_model.layers.27.self_attn.v_proj",
"model.language_model.layers.27.self_attn.o_proj",
"model.language_model.layers.27.mlp.gate_proj",
"model.language_model.layers.27.mlp.up_proj",
"model.language_model.layers.27.mlp.down_proj",
"model.language_model.layers.28.self_attn.q_proj",
"model.language_model.layers.28.self_attn.k_proj",
"model.language_model.layers.28.self_attn.v_proj",
"model.language_model.layers.28.self_attn.o_proj",
"model.language_model.layers.28.mlp.gate_proj",
"model.language_model.layers.28.mlp.up_proj",
"model.language_model.layers.28.mlp.down_proj",
"model.language_model.layers.29.self_attn.q_proj",
"model.language_model.layers.29.self_attn.k_proj",
"model.language_model.layers.29.self_attn.o_proj",
"model.language_model.layers.29.mlp.gate_proj",
"model.language_model.layers.29.mlp.up_proj",
"model.language_model.layers.29.mlp.down_proj",
"model.language_model.layers.30.self_attn.q_proj",
"model.language_model.layers.30.self_attn.k_proj",
"model.language_model.layers.30.self_attn.v_proj",
"model.language_model.layers.30.self_attn.o_proj",
"model.language_model.layers.30.mlp.gate_proj",
"model.language_model.layers.30.mlp.up_proj",
"model.language_model.layers.30.mlp.down_proj",
"model.language_model.layers.31.self_attn.q_proj",
"model.language_model.layers.31.self_attn.k_proj",
"model.language_model.layers.31.self_attn.v_proj",
"model.language_model.layers.31.self_attn.o_proj",
"model.language_model.layers.31.mlp.gate_proj",
"model.language_model.layers.31.mlp.up_proj",
"model.language_model.layers.31.mlp.down_proj",
"model.language_model.layers.32.self_attn.q_proj",
"model.language_model.layers.32.self_attn.k_proj",
"model.language_model.layers.32.self_attn.v_proj",
"model.language_model.layers.32.self_attn.o_proj",
"model.language_model.layers.32.mlp.gate_proj",
"model.language_model.layers.32.mlp.up_proj",
"model.language_model.layers.32.mlp.down_proj",
"model.language_model.layers.33.self_attn.q_proj",
"model.language_model.layers.33.self_attn.k_proj",
"model.language_model.layers.33.self_attn.v_proj",
"model.language_model.layers.33.self_attn.o_proj",
"model.language_model.layers.33.mlp.gate_proj",
"model.language_model.layers.33.mlp.up_proj",
"model.language_model.layers.33.mlp.down_proj",
"model.language_model.layers.34.self_attn.q_proj",
"model.language_model.layers.34.self_attn.k_proj",
"model.language_model.layers.34.self_attn.v_proj",
"model.language_model.layers.34.self_attn.o_proj",
"model.language_model.layers.34.mlp.gate_proj",
"model.language_model.layers.34.mlp.up_proj",
"model.language_model.layers.34.mlp.down_proj",
"model.language_model.layers.35.self_attn.q_proj",
"model.language_model.layers.35.self_attn.k_proj",
"model.language_model.layers.35.self_attn.o_proj",
"model.language_model.layers.35.mlp.gate_proj",
"model.language_model.layers.35.mlp.up_proj",
"model.language_model.layers.35.mlp.down_proj",
"model.language_model.layers.36.self_attn.q_proj",
"model.language_model.layers.36.self_attn.k_proj",
"model.language_model.layers.36.self_attn.v_proj",
"model.language_model.layers.36.self_attn.o_proj",
"model.language_model.layers.36.mlp.gate_proj",
"model.language_model.layers.36.mlp.up_proj",
"model.language_model.layers.36.mlp.down_proj",
"model.language_model.layers.37.self_attn.q_proj",
"model.language_model.layers.37.self_attn.k_proj",
"model.language_model.layers.37.self_attn.v_proj",
"model.language_model.layers.37.self_attn.o_proj",
"model.language_model.layers.37.mlp.gate_proj",
"model.language_model.layers.37.mlp.up_proj",
"model.language_model.layers.37.mlp.down_proj",
"model.language_model.layers.38.self_attn.q_proj",
"model.language_model.layers.38.self_attn.k_proj",
"model.language_model.layers.38.self_attn.v_proj",
"model.language_model.layers.38.self_attn.o_proj",
"model.language_model.layers.38.mlp.gate_proj",
"model.language_model.layers.38.mlp.up_proj",
"model.language_model.layers.38.mlp.down_proj",
"model.language_model.layers.39.self_attn.q_proj",
"model.language_model.layers.39.self_attn.k_proj",
"model.language_model.layers.39.self_attn.v_proj",
"model.language_model.layers.39.self_attn.o_proj",
"model.language_model.layers.39.mlp.gate_proj",
"model.language_model.layers.39.mlp.up_proj",
"model.language_model.layers.39.mlp.down_proj",
"model.language_model.layers.40.self_attn.q_proj",
"model.language_model.layers.40.self_attn.k_proj",
"model.language_model.layers.40.self_attn.v_proj",
"model.language_model.layers.40.self_attn.o_proj",
"model.language_model.layers.40.mlp.gate_proj",
"model.language_model.layers.40.mlp.up_proj",
"model.language_model.layers.40.mlp.down_proj",
"model.language_model.layers.41.self_attn.q_proj",
"model.language_model.layers.41.self_attn.k_proj",
"model.language_model.layers.41.self_attn.o_proj",
"model.language_model.layers.41.mlp.gate_proj",
"model.language_model.layers.41.mlp.up_proj",
"model.language_model.layers.41.mlp.down_proj",
"model.language_model.layers.42.self_attn.q_proj",
"model.language_model.layers.42.self_attn.k_proj",
"model.language_model.layers.42.self_attn.v_proj",
"model.language_model.layers.42.self_attn.o_proj",
"model.language_model.layers.42.mlp.gate_proj",
"model.language_model.layers.42.mlp.up_proj",
"model.language_model.layers.42.mlp.down_proj",
"model.language_model.layers.43.self_attn.q_proj",
"model.language_model.layers.43.self_attn.k_proj",
"model.language_model.layers.43.self_attn.v_proj",
"model.language_model.layers.43.self_attn.o_proj",
"model.language_model.layers.43.mlp.gate_proj",
"model.language_model.layers.43.mlp.up_proj",
"model.language_model.layers.43.mlp.down_proj",
"model.language_model.layers.44.self_attn.q_proj",
"model.language_model.layers.44.self_attn.k_proj",
"model.language_model.layers.44.self_attn.v_proj",
"model.language_model.layers.44.self_attn.o_proj",
"model.language_model.layers.44.mlp.gate_proj",
"model.language_model.layers.44.mlp.up_proj",
"model.language_model.layers.44.mlp.down_proj",
"model.language_model.layers.45.self_attn.q_proj",
"model.language_model.layers.45.self_attn.k_proj",
"model.language_model.layers.45.self_attn.v_proj",
"model.language_model.layers.45.self_attn.o_proj",
"model.language_model.layers.45.mlp.gate_proj",
"model.language_model.layers.45.mlp.up_proj",
"model.language_model.layers.45.mlp.down_proj",
"model.language_model.layers.46.self_attn.q_proj",
"model.language_model.layers.46.self_attn.k_proj",
"model.language_model.layers.46.self_attn.v_proj",
"model.language_model.layers.46.self_attn.o_proj",
"model.language_model.layers.46.mlp.gate_proj",
"model.language_model.layers.46.mlp.up_proj",
"model.language_model.layers.46.mlp.down_proj",
"model.language_model.layers.47.self_attn.q_proj",
"model.language_model.layers.47.self_attn.k_proj",
"model.language_model.layers.47.self_attn.o_proj",
"model.language_model.layers.47.mlp.gate_proj",
"model.language_model.layers.47.mlp.up_proj",
"model.language_model.layers.47.mlp.down_proj",
"model.language_model.layers.48.self_attn.q_proj",
"model.language_model.layers.48.self_attn.k_proj",
"model.language_model.layers.48.self_attn.v_proj",
"model.language_model.layers.48.self_attn.o_proj",
"model.language_model.layers.48.mlp.gate_proj",
"model.language_model.layers.48.mlp.up_proj",
"model.language_model.layers.48.mlp.down_proj",
"model.language_model.layers.49.self_attn.q_proj",
"model.language_model.layers.49.self_attn.k_proj",
"model.language_model.layers.49.self_attn.v_proj",
"model.language_model.layers.49.self_attn.o_proj",
"model.language_model.layers.49.mlp.gate_proj",
"model.language_model.layers.49.mlp.up_proj",
"model.language_model.layers.49.mlp.down_proj",
"model.language_model.layers.50.self_attn.q_proj",
"model.language_model.layers.50.self_attn.k_proj",
"model.language_model.layers.50.self_attn.v_proj",
"model.language_model.layers.50.self_attn.o_proj",
"model.language_model.layers.50.mlp.gate_proj",
"model.language_model.layers.50.mlp.up_proj",
"model.language_model.layers.50.mlp.down_proj",
"model.language_model.layers.51.self_attn.q_proj",
"model.language_model.layers.51.self_attn.k_proj",
"model.language_model.layers.51.self_attn.v_proj",
"model.language_model.layers.51.self_attn.o_proj",
"model.language_model.layers.51.mlp.gate_proj",
"model.language_model.layers.51.mlp.up_proj",
"model.language_model.layers.51.mlp.down_proj",
"model.language_model.layers.52.self_attn.q_proj",
"model.language_model.layers.52.self_attn.k_proj",
"model.language_model.layers.52.self_attn.v_proj",
"model.language_model.layers.52.self_attn.o_proj",
"model.language_model.layers.52.mlp.gate_proj",
"model.language_model.layers.52.mlp.up_proj",
"model.language_model.layers.52.mlp.down_proj",
"model.language_model.layers.53.self_attn.q_proj",
"model.language_model.layers.53.self_attn.k_proj",
"model.language_model.layers.53.self_attn.o_proj",
"model.language_model.layers.53.mlp.gate_proj",
"model.language_model.layers.53.mlp.up_proj",
"model.language_model.layers.53.mlp.down_proj",
"model.language_model.layers.54.self_attn.q_proj",
"model.language_model.layers.54.self_attn.k_proj",
"model.language_model.layers.54.self_attn.v_proj",
"model.language_model.layers.54.self_attn.o_proj",
"model.language_model.layers.54.mlp.gate_proj",
"model.language_model.layers.54.mlp.up_proj",
"model.language_model.layers.54.mlp.down_proj",
"model.language_model.layers.55.self_attn.q_proj",
"model.language_model.layers.55.self_attn.k_proj",
"model.language_model.layers.55.self_attn.v_proj",
"model.language_model.layers.55.self_attn.o_proj",
"model.language_model.layers.55.mlp.gate_proj",
"model.language_model.layers.55.mlp.up_proj",
"model.language_model.layers.55.mlp.down_proj",
"model.language_model.layers.56.self_attn.q_proj",
"model.language_model.layers.56.self_attn.k_proj",
"model.language_model.layers.56.self_attn.v_proj",
"model.language_model.layers.56.self_attn.o_proj",
"model.language_model.layers.56.mlp.gate_proj",
"model.language_model.layers.56.mlp.up_proj",
"model.language_model.layers.56.mlp.down_proj",
"model.language_model.layers.57.self_attn.q_proj",
"model.language_model.layers.57.self_attn.k_proj",
"model.language_model.layers.57.self_attn.v_proj",
"model.language_model.layers.57.self_attn.o_proj",
"model.language_model.layers.57.mlp.gate_proj",
"model.language_model.layers.57.mlp.up_proj",
"model.language_model.layers.57.mlp.down_proj",
"model.language_model.layers.58.self_attn.q_proj",
"model.language_model.layers.58.self_attn.k_proj",
"model.language_model.layers.58.self_attn.v_proj",
"model.language_model.layers.58.self_attn.o_proj",
"model.language_model.layers.58.mlp.gate_proj",
"model.language_model.layers.58.mlp.up_proj",
"model.language_model.layers.58.mlp.down_proj",
"model.language_model.layers.59.self_attn.q_proj",
"model.language_model.layers.59.self_attn.k_proj",
"model.language_model.layers.59.self_attn.o_proj",
"model.language_model.layers.59.mlp.gate_proj",
"model.language_model.layers.59.mlp.up_proj",
"model.language_model.layers.59.mlp.down_proj"
],
"loraplus_lr_embedding": 1e-06,
"lr_scheduler": "cosine",
"max_grad_norm": 1.0,
"mean_resizing_embeddings": false,
"merge_method": "memory_efficient",
"micro_batch_size": 1,
"model_config_type": "gemma4",
"model_config_type_text": "gemma4_text",
"num_epochs": 4.0,
"num_generation_samples": 3,
"optimizer": "adamw_torch_fused",
"otel_metrics_host": "localhost",
"otel_metrics_port": 8000,
"output_dir": "/workspace/eft31b_runs/dose_rows_v2/rows256/train/checkpoints",
"pad_to_sequence_len": false,
"plugins": [
"axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin",
"scimt.train.axolotl_plugins.CheckpointSchedulePlugin"
],
"pretrain_multipack_attn": true,
"processor_config": "/workspace/data/parents/mixed_4ep_prop",
"profiler_steps_start": 0,
"qgalore_cos_threshold": 0.4,
"qgalore_gamma_proj": 2,
"qgalore_proj_bits": 4,
"qgalore_proj_group_size": 256,
"qgalore_proj_quant": true,
"qgalore_proj_type": "std",
"qgalore_queue_size": 5,
"qgalore_rank": 256,
"qgalore_scale": 0.25,
"qgalore_update_proj_gap": 200,
"qlora_sharded_model_loading": false,
"quantize_moe_experts": false,
"ray_num_workers": 1,
"relora_prune_method": "magnitude",
"resources_per_worker": {
"GPU": 1
},
"sample_packing": false,
"sample_packing_bin_size": 200,
"sample_packing_group_size": 100000,
"save_only_model": true,
"save_safetensors": true,
"save_strategy": "no",
"save_total_limit": 1,
"seed": 424242,
"sequence_len": 4096,
"shuffle_before_merging_datasets": false,
"shuffle_merged_datasets": true,
"skip_prepare_dataset": false,
"streaming_multipack_buffer_size": 10000,
"strict": false,
"tensor_parallel_size": 1,
"tf32": true,
"tiled_mlp_use_original_mlp": true,
"tokenizer_config": "/workspace/data/parents/mixed_4ep_prop",
"tokenizer_save_jinja_files": true,
"torch_dtype": "torch.bfloat16",
"train_on_inputs": false,
"trl": {
"async_prefetch": false,
"log_completions": false,
"mask_truncated_completions": false,
"ref_model_mixup_alpha": 0.9,
"ref_model_sync_steps": 64,
"replay_buffer_size": 0,
"replay_recompute_logps": true,
"reroll_max_groups": 1,
"reroll_start_fraction": 1.0,
"reward_num_workers": 1,
"scale_rewards": true,
"skip_zero_advantage_batches": true,
"sync_ref_model": false,
"use_data_producer": false,
"use_vllm": false,
"vllm_lora_sync": false,
"vllm_server_host": "0.0.0.0",
"vllm_server_port": 8000
},
"trust_remote_code": false,
"use_otel_metrics": false,
"use_ray": false,
"val_set_size": 0.0,
"vllm": {
"device": "auto",
"dtype": "auto",
"gpu_memory_utilization": 0.9,
"host": "0.0.0.0",
"port": 8000
},
"warmup_ratio": 0.05,
"weight_decay": 0.01,
"world_size": 1
}
[2026-09-05 08:56:34,251] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:58343] EOS: 1 / <eos>
[2026-09-05 08:56:34,252] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:58343] BOS: 2 / <bos>
[2026-09-05 08:56:34,252] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:58343] PAD: 0 / <pad>
[2026-09-05 08:56:34,252] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:58343] UNK: 3 / <unk>
[2026-09-05 08:56:34,260] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:482] [PID:58343] Unable to find prepared dataset in /workspace/eft31b_runs/dose_rows_v2/rows256/train/prepared/63b4fc76e884db3d1cafc410de52b97c
[2026-09-05 08:56:34,260] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:58343] Loading raw datasets...
[2026-09-05 08:56:34,260] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:58343] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`.
Generating train split: 0 examples [00:00, ? examples/s]
Generating train split: 256 examples [00:00, 20424.60 examples/s]
[2026-09-05 08:56:34,795] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:58343] Loading dataset: /workspace/eft31b_runs/dose_rows_v2/rows256/eft_training.jsonl with base_type: chat_template and prompt_style: None
[2026-09-05 08:56:34,800] [INFO] [axolotl.prompt_strategies.chat_template.__call__:1209] [PID:58343] Using chat template:
---
{{ bos_token }}
{%- if messages[0]['role'] == 'system' -%}
{%- if messages[0]['content'] is string -%}
{%- set first_user_prefix = messages[0]['content'] + '
' -%}
{%- else -%}
{%- set first_user_prefix = messages[0]['content'][0]['text'] + '
' -%}
{%- endif -%}
{%- set loop_messages = messages[1:] -%}
{%- else -%}
{%- set first_user_prefix = "" -%}
{%- set loop_messages = messages -%}
{%- endif -%}
{%- for message in loop_messages -%}
{%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
{{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
{%- endif -%}
{%- if (message['role'] == 'assistant') -%}
{%- set role = "model" -%}
{%- else -%}
{%- set role = message['role'] -%}
{%- endif -%}
{{ '<|turn>' + role + '
' + (first_user_prefix if loop.first else "") }}
{%- if message['content'] is string -%}
{{ message['content'] | trim }}
{%- elif message['content'] is iterable -%}
{%- for item in message['content'] -%}
{%- if item['type'] == 'image' -%}
{{ '<|image>' }}
{%- elif item['type'] == 'text' -%}
{{ item['text'] | trim }}
{%- endif -%}
{%- endfor -%}
{%- else -%}
{{ raise_exception("Invalid content type") }}
{%- endif -%}
{{ '<turn|>
' }}
{%- endfor -%}
{%- if add_generation_prompt -%}
{{'<|turn>model
'}}
{%- endif -%}
---
Tokenizing Prompts (num_proc=4): 0%| | 0/256 [00:00<?, ? examples/s]
Tokenizing Prompts (num_proc=4): 25%|██▌ | 64/256 [00:05<00:15, 12.35 examples/s]
Tokenizing Prompts (num_proc=4): 50%|█████ | 128/256 [00:09<00:08, 14.61 examples/s]
Tokenizing Prompts (num_proc=4): 75%|███████▌ | 192/256 [00:12<00:04, 15.61 examples/s]
Tokenizing Prompts (num_proc=4): 100%|██████████| 256/256 [00:16<00:00, 16.11 examples/s]
Tokenizing Prompts (num_proc=4): 100%|██████████| 256/256 [00:16<00:00, 15.30 examples/s]
[2026-09-05 08:57:06,444] [INFO] [axolotl.utils.data.utils._log_dataset_stats:212] [PID:58343] min_input_len: 104
[2026-09-05 08:57:06,446] [INFO] [axolotl.utils.data.utils._log_dataset_stats:213] [PID:58343] max_input_len: 1764
Dropping Invalid Sequences (<None or >4096) (num_proc=4): 0%| | 0/256 [00:00<?, ? examples/s]
Dropping Invalid Sequences (<None or >4096) (num_proc=4): 25%|██▌ | 64/256 [00:00<00:00, 465.84 examples/s]
Dropping Invalid Sequences (<None or >4096) (num_proc=4): 100%|██████████| 256/256 [00:00<00:00, 1008.82 examples/s]
Saving the dataset (0/1 shards): 0%| | 0/256 [00:00<?, ? examples/s]
Saving the dataset (0/1 shards): 100%|██████████| 256/256 [00:32<00:00, 7.92 examples/s]
Saving the dataset (1/1 shards): 100%|██████████| 256/256 [00:32<00:00, 7.92 examples/s]
Saving the dataset (1/1 shards): 100%|██████████| 256/256 [00:33<00:00, 7.66 examples/s]
[2026-09-05 08:57:40,288] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:420] [PID:58343] total_num_tokens: 164_297
[2026-09-05 08:57:40,296] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:438] [PID:58343] `total_supervised_tokens: 72_046`
[2026-09-05 08:57:40,296] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:521] [PID:58343] total_num_steps: 32
[2026-09-05 08:57:40,297] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:58343] Maximum number of steps set at 32
[2026-09-05 08:57:40,298] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:58343] loading tokenizer... /workspace/data/parents/mixed_4ep_prop
[2026-09-05 08:57:42,507] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:58343] EOS: 1 / <eos>
[2026-09-05 08:57:42,507] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:58343] BOS: 2 / <bos>
[2026-09-05 08:57:42,507] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:58343] PAD: 0 / <pad>
[2026-09-05 08:57:42,507] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:58343] UNK: 3 / <unk>
[2026-09-05 08:57:44,744] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:58343] Loading model
[2026-09-05 08:57:44,751] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:75] [PID:58343] Patched OptimState8bit for torch.compile compatibility
[2026-09-05 08:57:44,751] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:122] [PID:58343] Patched OptimState4bit for torch.compile compatibility
[2026-09-05 08:57:44,751] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:154] [PID:58343] Patched OptimStateFp8 for torch.compile compatibility
[2026-09-05 08:57:44,761] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:58343] Patched Trainer.evaluation_loop with nanmean loss calculation
[2026-09-05 08:57:44,763] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:58343] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation
[2026-09-05 08:57:44,942] [INFO] [axolotl.monkeypatch.attention.flash_attn_4.patch_flash_attn_4:52] [PID:58343] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4
[2026-09-05 08:57:45,357] [INFO] [axolotl.monkeypatch.models.gemma4.fused_attn.patch_gemma4_fused_attn:194] [PID:58343] Patched Gemma4TextAttention.forward with fused RMSNorm+RoPE Triton kernels
[2026-09-05 08:57:45,357] [INFO] [axolotl.monkeypatch.models.gemma4.fused_attn.patch_gemma4_fused_attn:198] [PID:58343] Installed Gemma4 shared_kv_states side channel (PR #3611)
[2026-09-05 08:57:45,422] [INFO] [axolotl.integrations.cut_cross_entropy.pre_model_load:94] [PID:58343] Applying Cut Cross Entropy to model type: gemma4
Loading weights: 0%| | 0/1189 [00:00<?, ?it/s]
Loading weights: 10%|█ | 124/1189 [00:00<00:00, 1237.43it/s]
Loading weights: 21%|██ | 248/1189 [00:00<00:00, 1112.71it/s]
Loading weights: 30%|███ | 361/1189 [00:00<00:00, 1090.67it/s]
Loading weights: 40%|███▉ | 471/1189 [00:00<00:00, 1089.84it/s]
Loading weights: 49%|████▉ | 581/1189 [00:00<00:00, 1077.67it/s]
Loading weights: 58%|█████▊ | 694/1189 [00:00<00:00, 1094.37it/s]
Loading weights: 68%|██████▊ | 804/1189 [00:00<00:00, 1083.64it/s]
Loading weights: 77%|███████▋ | 913/1189 [00:00<00:00, 1064.71it/s]
Loading weights: 86%|████████▌ | 1020/1189 [00:00<00:00, 1047.80it/s]
Loading weights: 95%|█████████▍| 1125/1189 [00:01<00:00, 985.48it/s]
Loading weights: 100%|██████████| 1189/1189 [00:01<00:00, 1051.66it/s]
[2026-09-05 08:57:56,363] [INFO] [axolotl.monkeypatch.gemma4_hybrid_mask.patch_gemma4_hybrid_mask:94] [PID:58343] gemma4_hybrid_mask: patched modeling_gemma4.create_causal_mask to force SDPA-format masks for full-attention layers
[2026-09-05 08:57:56,365] [INFO] [axolotl.loaders.patch_manager._apply_gemma_hybrid_attention:243] [PID:58343] gemma4_hybrid_attn_impl: patched 10 global layers to use SDPA (remaining 50 sliding layers use flash_attention_2)
[2026-09-05 08:57:58,853] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:433] [PID:58343] Converting modules to torch.bfloat16
[2026-09-05 08:58:00,666] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:58343] Memory usage after model load 0.000GB ()
trainable params: 489,717,760 || all params: 31,762,804,272 || trainable%: 1.5418
[2026-09-05 08:58:04,279] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:58343] after adapters 0.000GB ()
[2026-09-05 08:59:50,501] [INFO] [axolotl.train.save_initial_configs:450] [PID:58343] Pre-saving adapter config to /workspace/eft31b_runs/dose_rows_v2/rows256/train/checkpoints...
[2026-09-05 08:59:50,506] [INFO] [axolotl.train.save_initial_configs:454] [PID:58343] Pre-saving tokenizer to /workspace/eft31b_runs/dose_rows_v2/rows256/train/checkpoints...
[2026-09-05 08:59:50,841] [INFO] [axolotl.train.save_initial_configs:459] [PID:58343] Pre-saving model config to /workspace/eft31b_runs/dose_rows_v2/rows256/train/checkpoints...
[2026-09-05 08:59:50,852] [INFO] [axolotl.train.save_initial_configs:463] [PID:58343] Pre-saving processor to /workspace/eft31b_runs/dose_rows_v2/rows256/train/checkpoints...
[2026-09-05 08:59:51,156] [INFO] [axolotl.train.execute_training:226] [PID:58343] Starting trainer...
0%| | 0/32 [00:00<?, ?it/s][2026-09-05 09:00:17,451] [INFO] [axolotl.kernels.autotune_telemetry.on_step_end:133] [PID:58343] Reported 2 fused-rope kernel autotune config(s) to telemetry.
3%|▎ | 1/32 [00:25<13:19, 25.79s/it]
{'loss': '0.5549', 'grad_norm': '6.157', 'learning_rate': '0', 'ppl': '1.742', 'memory/max_active (GiB)': '68.48', 'memory/max_allocated (GiB)': '68.48', 'memory/device_reserved (GiB)': '70.48', 'tokens/train_per_sec_per_gpu': '8.915', 'tokens/total': 22716, 'tokens/trainable': 9743, 'epoch': '0.125'}
3%|▎ | 1/32 [00:25<13:19, 25.79s/it]
6%|▋ | 2/32 [00:45<11:11, 22.37s/it]
{'loss': '0.5822', 'grad_norm': '2.388', 'learning_rate': '5e-05', 'ppl': '1.79', 'memory/max_active (GiB)': '71.63', 'memory/max_allocated (GiB)': '71.63', 'memory/device_reserved (GiB)': '73.59', 'tokens/train_per_sec_per_gpu': '14.84', 'tokens/total': 41936, 'tokens/trainable': 17969, 'epoch': '0.25'}
6%|▋ | 2/32 [00:45<11:11, 22.37s/it]
9%|▉ | 3/32 [01:06<10:22, 21.48s/it]
{'loss': '0.5727', 'grad_norm': '8.661', 'learning_rate': '0.0001', 'ppl': '1.773', 'memory/max_active (GiB)': '72.78', 'memory/max_allocated (GiB)': '72.78', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '17.88', 'tokens/total': 64803, 'tokens/trainable': 28823, 'epoch': '0.375'}
9%|▉ | 3/32 [01:06<10:22, 21.48s/it]
12%|█▎ | 4/32 [01:26<09:45, 20.89s/it]
{'loss': '0.4611', 'grad_norm': '1.035', 'learning_rate': '9.975e-05', 'ppl': '1.586', 'memory/max_active (GiB)': '71.02', 'memory/max_allocated (GiB)': '71.02', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '23.71', 'tokens/total': 86131, 'tokens/trainable': 39336, 'epoch': '0.5'}
12%|█▎ | 4/32 [01:26<09:45, 20.89s/it]
16%|█▌ | 5/32 [01:46<09:13, 20.51s/it]
{'loss': '0.4344', 'grad_norm': '1.301', 'learning_rate': '9.902e-05', 'ppl': '1.544', 'memory/max_active (GiB)': '70.25', 'memory/max_allocated (GiB)': '70.25', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '38.66', 'tokens/total': 106245, 'tokens/trainable': 47792, 'epoch': '0.625'}
16%|█▌ | 5/32 [01:46<09:13, 20.51s/it]
19%|█▉ | 6/32 [02:05<08:45, 20.23s/it]
{'loss': '0.3433', 'grad_norm': '2.691', 'learning_rate': '9.78e-05', 'ppl': '1.41', 'memory/max_active (GiB)': '70.67', 'memory/max_allocated (GiB)': '70.67', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '16.15', 'tokens/total': 126117, 'tokens/trainable': 55727, 'epoch': '0.75'}
19%|█▉ | 6/32 [02:05<08:45, 20.23s/it]
22%|██▏ | 7/32 [02:25<08:20, 20.03s/it]
{'loss': '0.3279', 'grad_norm': '1.4', 'learning_rate': '9.611e-05', 'ppl': '1.388', 'memory/max_active (GiB)': '70.8', 'memory/max_allocated (GiB)': '70.8', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '7.39', 'tokens/total': 143873, 'tokens/trainable': 64058, 'epoch': '0.875'}
22%|██▏ | 7/32 [02:25<08:20, 20.03s/it]
25%|██▌ | 8/32 [02:45<08:00, 20.02s/it]
{'loss': '0.301', 'grad_norm': '4.353', 'learning_rate': '9.397e-05', 'ppl': '1.351', 'memory/max_active (GiB)': '72.48', 'memory/max_allocated (GiB)': '72.48', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '23.04', 'tokens/total': 164297, 'tokens/trainable': 72046, 'epoch': '1'}
25%|██▌ | 8/32 [02:45<08:00, 20.02s/it]
28%|██▊ | 9/32 [03:05<07:41, 20.07s/it]
{'loss': '0.2616', 'grad_norm': '0.8263', 'learning_rate': '9.141e-05', 'ppl': '1.299', 'memory/max_active (GiB)': '70.86', 'memory/max_allocated (GiB)': '70.86', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '3.846', 'tokens/total': 183610, 'tokens/trainable': 80421, 'epoch': '1.125'}
28%|██▊ | 9/32 [03:05<07:41, 20.07s/it]
31%|███▏ | 10/32 [03:25<07:19, 19.95s/it]
{'loss': '0.3031', 'grad_norm': '0.6239', 'learning_rate': '8.844e-05', 'ppl': '1.354', 'memory/max_active (GiB)': '72.48', 'memory/max_allocated (GiB)': '72.48', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '16.15', 'tokens/total': 205351, 'tokens/trainable': 90016, 'epoch': '1.25'}
31%|███▏ | 10/32 [03:25<07:19, 19.95s/it]
34%|███▍ | 11/32 [03:44<06:55, 19.79s/it]
{'loss': '0.3175', 'grad_norm': '1.222', 'learning_rate': '8.511e-05', 'ppl': '1.374', 'memory/max_active (GiB)': '72.13', 'memory/max_allocated (GiB)': '72.13', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '30.41', 'tokens/total': 225730, 'tokens/trainable': 99355, 'epoch': '1.375'}
34%|███▍ | 11/32 [03:44<06:55, 19.79s/it]
38%|███▊ | 12/32 [04:03<06:33, 19.66s/it]
{'loss': '0.2871', 'grad_norm': '1.743', 'learning_rate': '8.145e-05', 'ppl': '1.333', 'memory/max_active (GiB)': '70.25', 'memory/max_allocated (GiB)': '70.25', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '21.69', 'tokens/total': 248247, 'tokens/trainable': 108567, 'epoch': '1.5'}
38%|███▊ | 12/32 [04:03<06:33, 19.66s/it]
41%|████ | 13/32 [04:23<06:10, 19.48s/it]
{'loss': '0.2732', 'grad_norm': '0.882', 'learning_rate': '7.75e-05', 'ppl': '1.314', 'memory/max_active (GiB)': '70.21', 'memory/max_allocated (GiB)': '70.21', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '33.79', 'tokens/total': 267702, 'tokens/trainable': 117605, 'epoch': '1.625'}
41%|████ | 13/32 [04:23<06:10, 19.48s/it]
44%|████▍ | 14/32 [04:42<05:51, 19.55s/it]
{'loss': '0.3196', 'grad_norm': '2.611', 'learning_rate': '7.33e-05', 'ppl': '1.377', 'memory/max_active (GiB)': '72.78', 'memory/max_allocated (GiB)': '72.78', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '12.63', 'tokens/total': 287799, 'tokens/trainable': 127300, 'epoch': '1.75'}
44%|████▍ | 14/32 [04:42<05:51, 19.55s/it]
47%|████▋ | 15/32 [05:01<05:30, 19.44s/it]
{'loss': '0.3126', 'grad_norm': '7.481', 'learning_rate': '6.891e-05', 'ppl': '1.367', 'memory/max_active (GiB)': '69.94', 'memory/max_allocated (GiB)': '69.94', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '7.041', 'tokens/total': 307430, 'tokens/trainable': 135472, 'epoch': '1.875'}
47%|████▋ | 15/32 [05:01<05:30, 19.44s/it]
50%|█████ | 16/32 [05:21<05:12, 19.55s/it]
{'loss': '0.2636', 'grad_norm': '3.765', 'learning_rate': '6.436e-05', 'ppl': '1.302', 'memory/max_active (GiB)': '71.63', 'memory/max_allocated (GiB)': '71.63', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '12.6', 'tokens/total': 328594, 'tokens/trainable': 144092, 'epoch': '2'}
50%|█████ | 16/32 [05:21<05:12, 19.55s/it]
53%|█████▎ | 17/32 [05:41<04:55, 19.70s/it]
{'loss': '0.2449', 'grad_norm': '32.55', 'learning_rate': '5.97e-05', 'ppl': '1.278', 'memory/max_active (GiB)': '69.45', 'memory/max_allocated (GiB)': '69.45', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '15.24', 'tokens/total': 347825, 'tokens/trainable': 152323, 'epoch': '2.125'}
53%|█████▎ | 17/32 [05:41<04:55, 19.70s/it]
56%|█████▋ | 18/32 [06:01<04:35, 19.70s/it]
{'loss': '0.1919', 'grad_norm': '0.8049', 'learning_rate': '5.5e-05', 'ppl': '1.212', 'memory/max_active (GiB)': '71.25', 'memory/max_allocated (GiB)': '71.25', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '3.906', 'tokens/total': 368753, 'tokens/trainable': 162416, 'epoch': '2.25'}
56%|█████▋ | 18/32 [06:01<04:35, 19.70s/it]
59%|█████▉ | 19/32 [06:20<04:14, 19.58s/it]
{'loss': '0.1912', 'grad_norm': '0.9178', 'learning_rate': '5.03e-05', 'ppl': '1.211', 'memory/max_active (GiB)': '70.04', 'memory/max_allocated (GiB)': '70.04', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '13.69', 'tokens/total': 389683, 'tokens/trainable': 170572, 'epoch': '2.375'}
59%|█████▉ | 19/32 [06:20<04:14, 19.58s/it]
62%|██████▎ | 20/32 [06:40<03:54, 19.55s/it]
{'loss': '0.3014', 'grad_norm': '1.025', 'learning_rate': '4.564e-05', 'ppl': '1.352', 'memory/max_active (GiB)': '70.86', 'memory/max_allocated (GiB)': '70.86', 'memory/device_reserved (GiB)': '74.81', 'tokens/train_per_sec_per_gpu': '8.426', 'tokens/total': 410688, 'tokens/trainable': 179712, 'epoch': '2.5'}
62%|██████▎ | 20/32 [06:40<03:54, 19.55s/it]
66%|██████▌ | 21/32 [06:59<03:33, 19.45s/it]
{'loss': '0.2021', 'grad_norm': '0.5888', 'learning_rate': '4.109e-05', 'ppl': '1.224', 'memory/max_active (GiB)': '72.78', 'memory/max_allocated (GiB)': '72.78', 'memory/device_reserved (GiB)': '74.87', 'tokens/train_per_sec_per_gpu': '9.155', 'tokens/total': 430384, 'tokens/trainable': 187812, 'epoch': '2.625'}
66%|██████▌ | 21/32 [06:59<03:33, 19.45s/it]
69%|██████▉ | 22/32 [07:19<03:16, 19.69s/it]
{'loss': '0.2075', 'grad_norm': '2.312', 'learning_rate': '3.67e-05', 'ppl': '1.231', 'memory/max_active (GiB)': '72.48', 'memory/max_allocated (GiB)': '72.48', 'memory/device_reserved (GiB)': '74.87', 'tokens/train_per_sec_per_gpu': '20.09', 'tokens/total': 451664, 'tokens/trainable': 197012, 'epoch': '2.75'}
69%|██████▉ | 22/32 [07:19<03:16, 19.69s/it]
72%|███████▏ | 23/32 [07:39<02:56, 19.62s/it]
{'loss': '0.2361', 'grad_norm': '39.12', 'learning_rate': '3.25e-05', 'ppl': '1.266', 'memory/max_active (GiB)': '70.8', 'memory/max_allocated (GiB)': '70.8', 'memory/device_reserved (GiB)': '74.87', 'tokens/train_per_sec_per_gpu': '13.84', 'tokens/total': 473259, 'tokens/trainable': 207000, 'epoch': '2.875'}
72%|███████▏ | 23/32 [07:39<02:56, 19.62s/it]
75%|███████▌ | 24/32 [07:58<02:35, 19.47s/it]
{'loss': '0.255', 'grad_norm': '0.9063', 'learning_rate': '2.855e-05', 'ppl': '1.29', 'memory/max_active (GiB)': '70.16', 'memory/max_allocated (GiB)': '70.16', 'memory/device_reserved (GiB)': '74.87', 'tokens/train_per_sec_per_gpu': '5.837', 'tokens/total': 492891, 'tokens/trainable': 216138, 'epoch': '3'}
75%|███████▌ | 24/32 [07:58<02:35, 19.47s/it]
78%|███████▊ | 25/32 [08:18<02:18, 19.73s/it]
{'loss': '0.256', 'grad_norm': '172.1', 'learning_rate': '2.489e-05', 'ppl': '1.292', 'memory/max_active (GiB)': '71.24', 'memory/max_allocated (GiB)': '71.24', 'memory/device_reserved (GiB)': '74.87', 'tokens/train_per_sec_per_gpu': '10.47', 'tokens/total': 512363, 'tokens/trainable': 224560, 'epoch': '3.125'}
78%|███████▊ | 25/32 [08:18<02:18, 19.73s/it]
81%|████████▏ | 26/32 [08:37<01:57, 19.61s/it]
{'loss': '0.2303', 'grad_norm': '2.415', 'learning_rate': '2.156e-05', 'ppl': '1.259', 'memory/max_active (GiB)': '72.78', 'memory/max_allocated (GiB)': '72.78', 'memory/device_reserved (GiB)': '74.95', 'tokens/train_per_sec_per_gpu': '22.04', 'tokens/total': 532130, 'tokens/trainable': 234111, 'epoch': '3.25'}
81%|████████▏ | 26/32 [08:37<01:57, 19.61s/it]
84%|████████▍ | 27/32 [08:58<01:38, 19.79s/it]
{'loss': '0.1525', 'grad_norm': '0.6741', 'learning_rate': '1.859e-05', 'ppl': '1.165', 'memory/max_active (GiB)': '72.48', 'memory/max_allocated (GiB)': '72.48', 'memory/device_reserved (GiB)': '74.95', 'tokens/train_per_sec_per_gpu': '13.31', 'tokens/total': 554954, 'tokens/trainable': 244446, 'epoch': '3.375'}
84%|████████▍ | 27/32 [08:58<01:38, 19.79s/it]
88%|████████▊ | 28/32 [09:18<01:19, 19.81s/it]
{'loss': '0.1452', 'grad_norm': '4.039', 'learning_rate': '1.603e-05', 'ppl': '1.156', 'memory/max_active (GiB)': '72.13', 'memory/max_allocated (GiB)': '72.13', 'memory/device_reserved (GiB)': '74.95', 'tokens/train_per_sec_per_gpu': '20.5', 'tokens/total': 576563, 'tokens/trainable': 253926, 'epoch': '3.5'}
88%|████████▊ | 28/32 [09:18<01:19, 19.81s/it]
91%|█████████ | 29/32 [09:37<00:58, 19.59s/it]
{'loss': '0.209', 'grad_norm': '1.228', 'learning_rate': '1.389e-05', 'ppl': '1.232', 'memory/max_active (GiB)': '70.25', 'memory/max_allocated (GiB)': '70.25', 'memory/device_reserved (GiB)': '74.95', 'tokens/train_per_sec_per_gpu': '20.82', 'tokens/total': 597120, 'tokens/trainable': 263155, 'epoch': '3.625'}
91%|█████████ | 29/32 [09:37<00:58, 19.59s/it]
94%|█████████▍| 30/32 [09:56<00:39, 19.57s/it]
{'loss': '0.1706', 'grad_norm': '14.85', 'learning_rate': '1.22e-05', 'ppl': '1.186', 'memory/max_active (GiB)': '71.63', 'memory/max_allocated (GiB)': '71.63', 'memory/device_reserved (GiB)': '74.95', 'tokens/train_per_sec_per_gpu': '5.333', 'tokens/total': 615793, 'tokens/trainable': 270190, 'epoch': '3.75'}
94%|█████████▍| 30/32 [09:56<00:39, 19.57s/it]
97%|█████████▋| 31/32 [10:15<00:19, 19.51s/it]
{'loss': '0.1846', 'grad_norm': '3.046', 'learning_rate': '1.098e-05', 'ppl': '1.203', 'memory/max_active (GiB)': '70.86', 'memory/max_allocated (GiB)': '70.86', 'memory/device_reserved (GiB)': '74.95', 'tokens/train_per_sec_per_gpu': '7.541', 'tokens/total': 635514, 'tokens/trainable': 279469, 'epoch': '3.875'}
97%|█████████▋| 31/32 [10:15<00:19, 19.51s/it]
100%|██████████| 32/32 [10:35<00:00, 19.54s/it]
{'loss': '0.1777', 'grad_norm': '17.69', 'learning_rate': '1.025e-05', 'ppl': '1.194', 'memory/max_active (GiB)': '70.8', 'memory/max_allocated (GiB)': '70.8', 'memory/device_reserved (GiB)': '74.95', 'tokens/train_per_sec_per_gpu': '53.8', 'tokens/total': 657188, 'tokens/trainable': 288184, 'epoch': '4'}
100%|██████████| 32/32 [10:35<00:00, 19.54s/it][2026-09-05 09:10:27,296] [INFO] [axolotl.core.trainers.base._save:828] [PID:58343] Saving model checkpoint to /workspace/eft31b_runs/dose_rows_v2/rows256/train/checkpoints/checkpoint-32
{'train_runtime': '640.5', 'train_samples_per_second': '1.599', 'train_steps_per_second': '0.05', 'train_loss': '0.2897', 'memory/max_active (GiB)': '63.96', 'memory/max_allocated (GiB)': '63.96', 'memory/device_reserved (GiB)': '74.95', 'epoch': '4', 'tokens/train_per_sec_per_gpu': '0'}
100%|██████████| 32/32 [10:40<00:00, 19.54s/it]
100%|██████████| 32/32 [10:40<00:00, 20.01s/it]
[2026-09-05 09:10:32,109] [INFO] [axolotl.train.save_trained_model:267] [PID:58343] Training completed! Saving trained model to /workspace/eft31b_runs/dose_rows_v2/rows256/train/checkpoints.
[2026-09-05 09:10:36,110] [INFO] [axolotl.train.save_trained_model:388] [PID:58343] Model successfully saved to /workspace/eft31b_runs/dose_rows_v2/rows256/train/checkpoints
|