Fornace's picture
Upload Eugeny seed SFT LoRA adapter
235714e verified
Raw History Blame Contribute Delete
33.5 kB
[2026-06-24 12:52:11,377] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:37810] baseline 0.000GB ()
[2026-06-24 12:52:11,378] [INFO] [axolotl.cli.config.load_cfg:333] [PID:37810] config:
{
"activation_offloading": false,
"adapter": "lora",
"attn_needs_dtype_cast": false,
"attn_supports_packing": false,
"attn_uses_flash_lib": false,
"axolotl_config_path": "fine-tuning/configs/axolotl_qwen36_27b_seed_sft.yml",
"base_model": "Qwen/Qwen3.6-27B",
"base_model_config": "Qwen/Qwen3.6-27B",
"batch_size": 16,
"bf16": true,
"capabilities": {
"bf16": true,
"compute_capability": "sm_120",
"fp8": true,
"n_gpu": 1,
"n_node": 1,
"tf32": true
},
"chat_template": "tokenizer_default",
"context_parallel_size": 1,
"dataloader_num_workers": 1,
"dataloader_pin_memory": true,
"dataloader_prefetch_factor": 256,
"dataset_num_proc": 384,
"dataset_prepared_path": "./fine-tuning/data/seed/axolotl_prepared_qwen36_27b_seed",
"datasets": [
{
"chat_template": "tokenizer_default",
"field_messages": "messages",
"message_property_mappings": {
"content": "content",
"role": "role"
},
"path": "./fine-tuning/data/seed/sft_train.jsonl",
"roles_to_train": [
"assistant"
],
"train_on_eos": "turn",
"trust_remote_code": false,
"type": "chat_template"
}
],
"ddp": false,
"device": "cuda:0",
"dion_rank_fraction": 1.0,
"dion_rank_multiple_of": 1,
"eaft_alpha": 1.0,
"eaft_k": 20,
"env_capabilities": {
"torch_version": "2.12.1"
},
"eval_batch_size": 1,
"eval_causal_lm_metrics": [
"sacrebleu",
"comet",
"ter",
"chrf"
],
"eval_max_new_tokens": 128,
"eval_steps": 200,
"eval_table_size": 0,
"experimental_skip_move_to_device": true,
"fp16": false,
"generate_samples": false,
"generation_do_sample": true,
"generation_max_new_tokens": 50,
"generation_prompt_ratio": 0.5,
"generation_temperature": 0.7,
"gradient_accumulation_steps": 16,
"gradient_checkpointing": true,
"gradient_checkpointing_kwargs": {
"use_reentrant": true
},
"hub_model_id": "fornace-ml/eugeny-seed-adapters",
"hub_strategy": "every_save",
"include_tkps": true,
"is_falcon_derived_model": false,
"is_llama_derived_model": false,
"is_mistral_derived_model": false,
"is_multimodal": true,
"layer_offloading": false,
"learning_rate": 0.0001,
"lisa_layers_attribute": "model.layers",
"load_best_model_at_end": false,
"load_in_4bit": false,
"load_in_8bit": false,
"local_rank": 0,
"logging_steps": 5,
"lora_alpha": 128,
"lora_dropout": 0.05,
"lora_r": 64,
"lora_target_linear": true,
"loraplus_lr_embedding": 1e-06,
"lr_scheduler": "cosine",
"mean_resizing_embeddings": false,
"merge_method": "memory_efficient",
"micro_batch_size": 1,
"model_config_type": "qwen3_5",
"model_config_type_text": "qwen3_5_text",
"num_epochs": 2.0,
"num_generation_samples": 3,
"optimizer": "adamw_torch",
"otel_metrics_host": "localhost",
"otel_metrics_port": 8000,
"output_dir": "./outputs/qwen36-27b-agent-memory-sft-lora-axolotl",
"pad_to_sequence_len": true,
"pretrain_multipack_attn": true,
"processor_config": "Qwen/Qwen3.6-27B",
"profiler_steps_start": 0,
"qgalore_cos_threshold": 0.4,
"qgalore_gamma_proj": 2,
"qgalore_proj_bits": 4,
"qgalore_proj_group_size": 256,
"qgalore_proj_quant": true,
"qgalore_proj_type": "std",
"qgalore_queue_size": 5,
"qgalore_rank": 256,
"qgalore_scale": 0.25,
"qgalore_update_proj_gap": 200,
"qlora_sharded_model_loading": false,
"quantize_moe_experts": false,
"ray_num_workers": 1,
"relora_prune_method": "magnitude",
"resources_per_worker": {
"GPU": 1
},
"sample_packing": false,
"sample_packing_bin_size": 200,
"sample_packing_group_size": 100000,
"save_only_model": false,
"save_safetensors": true,
"save_steps": 200,
"save_total_limit": 2,
"seed": 42,
"sequence_len": 4096,
"shuffle_before_merging_datasets": false,
"shuffle_merged_datasets": true,
"skip_prepare_dataset": false,
"streaming_multipack_buffer_size": 10000,
"strict": false,
"tensor_parallel_size": 1,
"test_datasets": [
{
"chat_template": "tokenizer_default",
"field_messages": "messages",
"message_property_mappings": {
"content": "content",
"role": "role"
},
"path": "./fine-tuning/data/seed/sft_eval.jsonl",
"roles_to_train": [
"assistant"
],
"split": "train",
"train_on_eos": "turn",
"trust_remote_code": false,
"type": "chat_template"
}
],
"tf32": true,
"tiled_mlp_use_original_mlp": true,
"tokenizer_config": "Qwen/Qwen3.6-27B",
"tokenizer_save_jinja_files": true,
"tokenizer_type": "AutoTokenizer",
"torch_dtype": "torch.bfloat16",
"train_on_inputs": false,
"trl": {
"async_prefetch": false,
"log_completions": false,
"mask_truncated_completions": false,
"ref_model_mixup_alpha": 0.9,
"ref_model_sync_steps": 64,
"replay_buffer_size": 0,
"replay_recompute_logps": true,
"reroll_max_groups": 1,
"reroll_start_fraction": 1.0,
"reward_num_workers": 1,
"scale_rewards": true,
"skip_zero_advantage_batches": true,
"sync_ref_model": false,
"use_data_producer": false,
"use_vllm": false,
"vllm_lora_sync": false,
"vllm_server_host": "0.0.0.0",
"vllm_server_port": 8000
},
"trust_remote_code": true,
"type_of_model": "AutoModelForCausalLM",
"use_otel_metrics": false,
"use_ray": false,
"use_wandb": true,
"val_set_size": 0.0,
"vllm": {
"device": "auto",
"dtype": "auto",
"gpu_memory_utilization": 0.9,
"host": "0.0.0.0",
"port": 8000
},
"wandb_log_model": "checkpoint",
"wandb_name": "eugeny-qwen36-27b-seed-sft",
"wandb_project": "eugeny",
"warmup_ratio": 0.03,
"weight_decay": 0.01,
"world_size": 1
}
[2026-06-24 12:52:12,530] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:37810] EOS: 248046 / <|im_end|>
[2026-06-24 12:52:12,530] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:37810] BOS: None / None
[2026-06-24 12:52:12,530] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:37810] PAD: 248044 / <|endoftext|>
[2026-06-24 12:52:12,530] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:37810] UNK: None / None
[2026-06-24 12:52:12,531] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:477] [PID:37810] Loading prepared dataset from disk at fine-tuning/data/seed/axolotl_prepared_qwen36_27b_seed/ddaef6b4b116d96bae21a39214f33062...
[2026-06-24 12:52:12,535] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:477] [PID:37810] Loading prepared dataset from disk at fine-tuning/data/seed/axolotl_prepared_qwen36_27b_seed/063e5073d5d03e6d44b00418133c74c4...
[2026-06-24 12:52:12,539] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:420] [PID:37810] total_num_tokens: 77_178
[2026-06-24 12:52:12,540] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:438] [PID:37810] `total_supervised_tokens: 27_423`
[2026-06-24 12:52:12,540] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:521] [PID:37810] total_num_steps: 45
[2026-06-24 12:52:12,540] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:37810] Maximum number of steps set at 45
[2026-06-24 12:52:12,587] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:37810] loading tokenizer... Qwen/Qwen3.6-27B
[2026-06-24 12:52:13,600] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:37810] EOS: 248046 / <|im_end|>
[2026-06-24 12:52:13,600] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:37810] BOS: None / None
[2026-06-24 12:52:13,600] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:37810] PAD: 248044 / <|endoftext|>
[2026-06-24 12:52:13,600] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:37810] UNK: None / None
[2026-06-24 12:52:16,066] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:37810] Loading model
[2026-06-24 12:52:16,147] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:75] [PID:37810] Patched OptimState8bit for torch.compile compatibility
[2026-06-24 12:52:16,148] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:122] [PID:37810] Patched OptimState4bit for torch.compile compatibility
[2026-06-24 12:52:16,148] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:154] [PID:37810] Patched OptimStateFp8 for torch.compile compatibility
[2026-06-24 12:52:16,153] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:37810] Patched Trainer.evaluation_loop with nanmean loss calculation
[2026-06-24 12:52:16,154] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:37810] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation
[2026-06-24 12:52:16,160] [WARNING] [fla.utils.check_environments:63] [PID:37810] Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience.
Loading weights: 0%| | 0/1184 [00:00<?, ?it/s] Loading weights: 39%|β–ˆβ–ˆβ–ˆβ–‰ | 460/1184 [00:00<00:00, 4597.10it/s] Loading weights: 78%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 920/1184 [00:00<00:00, 4567.21it/s] Loading weights: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 1184/1184 [00:00<00:00, 4620.02it/s]
[2026-06-24 12:52:21,952] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:433] [PID:37810] Converting modules to torch.bfloat16
[2026-06-24 12:52:25,193] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:37810] Memory usage after model load 0.000GB (+0.000GB allocated, +0.002GB reserved)
[2026-06-24 12:52:25,217] [INFO] [axolotl.loaders.adapter._build_peft_lora_config:170] [PID:37810] found linear modules: ['down_proj', 'gate_proj', 'in_proj_a', 'in_proj_b', 'in_proj_qkv', 'in_proj_z', 'k_proj', 'linear_fc1', 'linear_fc2', 'o_proj', 'out_proj', 'proj', 'q_proj', 'qkv', 'up_proj', 'v_proj']
trainable params: 499,095,552 || all params: 27,855,824,112 || trainable%: 1.7917
[2026-06-24 12:52:28,890] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:37810] after adapters 0.000GB ()
[2026-06-24 12:52:37,669] [INFO] [axolotl.train.save_initial_configs:450] [PID:37810] Pre-saving adapter config to ./outputs/qwen36-27b-agent-memory-sft-lora-axolotl...
[2026-06-24 12:52:37,670] [INFO] [axolotl.train.save_initial_configs:454] [PID:37810] Pre-saving tokenizer to ./outputs/qwen36-27b-agent-memory-sft-lora-axolotl...
[2026-06-24 12:52:37,804] [INFO] [axolotl.train.save_initial_configs:459] [PID:37810] Pre-saving model config to ./outputs/qwen36-27b-agent-memory-sft-lora-axolotl...
[2026-06-24 12:52:37,812] [INFO] [axolotl.train.save_initial_configs:463] [PID:37810] Pre-saving processor to ./outputs/qwen36-27b-agent-memory-sft-lora-axolotl...
[2026-06-24 12:52:37,930] [INFO] [axolotl.train.execute_training:226] [PID:37810] Starting trainer...
wandb: Loading settings from /root/.config/wandb/settings
wandb: [wandb.login()] Loaded credentials for https://wandb.fornace.net from WANDB_API_KEY.
wandb: Currently logged in as: frappo to https://wandb.fornace.net. Use `wandb login --relogin` to force relogin
wandb: Tracking run with wandb version 0.28.0
wandb: Run data is saved locally in /root/sky_workdir/fornace-eugeny/eugeny/wandb/run-20260624_125239-mv6s5u0t
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run eugeny-qwen36-27b-seed-sft
wandb: ⭐️ View project at https://wandb.fornace.net/frappo/eugeny
wandb: πŸš€ View run at https://wandb.fornace.net/frappo/eugeny/runs/mv6s5u0t
wandb: WARNING Saving files without folders. If you want to preserve subdirectories pass base_path to wandb.save, i.e. wandb.save("/mnt/folder/file.h5", base_path="/mnt")
wandb: WARNING Symlinked 1 file into the W&B run directory; call wandb.save again to sync new files.
[2026-06-24 12:52:42,117] [INFO] [axolotl.utils.callbacks.on_train_begin:807] [PID:37810] The Axolotl config has been saved to the WandB run under files.
0%| | 0/45 [00:00<?, ?it/s][2026-06-24 12:52:42,127] [INFO] [axolotl.core.trainers.base.evaluate:475] [PID:37810] Running evaluation step...
0%| | 0/30 [00:00<?, ?it/s]
7%|β–‹ | 2/30 [00:00<00:11, 2.38it/s]
10%|β–ˆ | 3/30 [00:01<00:17, 1.52it/s]
13%|β–ˆβ–Ž | 4/30 [00:02<00:20, 1.27it/s]
17%|β–ˆβ–‹ | 5/30 [00:03<00:21, 1.17it/s]
20%|β–ˆβ–ˆ | 6/30 [00:04<00:21, 1.11it/s]
23%|β–ˆβ–ˆβ–Ž | 7/30 [00:05<00:21, 1.07it/s]
27%|β–ˆβ–ˆβ–‹ | 8/30 [00:06<00:20, 1.05it/s]
30%|β–ˆβ–ˆβ–ˆ | 9/30 [00:07<00:20, 1.03it/s]
33%|β–ˆβ–ˆβ–ˆβ–Ž | 10/30 [00:08<00:19, 1.02it/s]
37%|β–ˆβ–ˆβ–ˆβ–‹ | 11/30 [00:09<00:18, 1.02it/s]
40%|β–ˆβ–ˆβ–ˆβ–ˆ | 12/30 [00:10<00:17, 1.01it/s]
43%|β–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 13/30 [00:11<00:16, 1.01it/s]
47%|β–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 14/30 [00:12<00:15, 1.01it/s]
50%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 15/30 [00:13<00:14, 1.00it/s]
53%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 16/30 [00:14<00:13, 1.00it/s]
57%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 17/30 [00:16<00:14, 1.08s/it]
60%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 18/30 [00:17<00:12, 1.04s/it]
63%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 19/30 [00:18<00:11, 1.03s/it]
67%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 20/30 [00:19<00:10, 1.02s/it]
70%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 21/30 [00:20<00:09, 1.02s/it]
73%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 22/30 [00:21<00:08, 1.01s/it]
77%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 23/30 [00:22<00:07, 1.01s/it]
80%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 24/30 [00:23<00:06, 1.01s/it]
83%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 25/30 [00:24<00:05, 1.01s/it]
87%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 26/30 [00:25<00:04, 1.01s/it]
90%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 27/30 [00:26<00:03, 1.01s/it]
93%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž| 28/30 [00:27<00:02, 1.01s/it]
97%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹| 29/30 [00:28<00:01, 1.01s/it]
100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 30/30 [00:29<00:00, 1.04s/it]
{'eval_loss': '2.203', 'eval_runtime': '72.32', 'eval_samples_per_second': '0.415', 'eval_steps_per_second': '0.415', 'eval_ppl': '9.056', 'memory/max_active (GiB)': '63.23', 'memory/max_allocated (GiB)': '63.23', 'memory/device_reserved (GiB)': '63.62', 'epoch': 0}
0%| | 0/45 [01:12<?, ?it/s]
100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 30/30 [00:29<00:00, 1.04s/it]
 2%|▏ | 1/45 [02:42<1:59:23, 162.80s/it] 4%|▍ | 2/45 [03:41<1:12:58, 101.82s/it] 7%|β–‹ | 3/45 [04:41<57:48, 82.58s/it] 9%|β–‰ | 4/45 [05:41<50:18, 73.62s/it] 11%|β–ˆ | 5/45 [06:41<45:45, 68.64s/it] {'loss': '1.389', 'grad_norm': '2.801', 'learning_rate': '9.947e-05', 'ppl': '4.01', 'memory/max_active (GiB)': '72.06', 'memory/max_allocated (GiB)': '72.06', 'memory/device_reserved (GiB)': '75.09', 'tokens/train_per_sec_per_gpu': '2.492', 'tokens/total': 327680, 'tokens/trainable': 5821, 'epoch': '0.2228'}
11%|β–ˆ | 5/45 [06:41<45:45, 68.64s/it] 13%|β–ˆβ–Ž | 6/45 [07:41<42:39, 65.62s/it] 16%|β–ˆβ–Œ | 7/45 [08:41<40:25, 63.83s/it] 18%|β–ˆβ–Š | 8/45 [09:41<38:34, 62.54s/it] 20%|β–ˆβ–ˆ | 9/45 [10:40<37:00, 61.68s/it] 22%|β–ˆβ–ˆβ– | 10/45 [11:40<35:38, 61.10s/it] {'loss': '1.158', 'grad_norm': '0.9068', 'learning_rate': '9.36e-05', 'ppl': '3.182', 'memory/max_active (GiB)': '72.06', 'memory/max_allocated (GiB)': '72.06', 'memory/device_reserved (GiB)': '75.09', 'tokens/train_per_sec_per_gpu': '1.288', 'tokens/total': 655360, 'tokens/trainable': 12183, 'epoch': '0.4457'}
22%|β–ˆβ–ˆβ– | 10/45 [11:40<35:38, 61.10s/it] 24%|β–ˆβ–ˆβ– | 11/45 [12:40<34:23, 60.69s/it] 27%|β–ˆβ–ˆβ–‹ | 12/45 [13:40<33:13, 60.41s/it] 29%|β–ˆβ–ˆβ–‰ | 13/45 [14:39<32:06, 60.22s/it] 31%|β–ˆβ–ˆβ–ˆ | 14/45 [15:39<31:02, 60.08s/it] 33%|β–ˆβ–ˆβ–ˆβ–Ž | 15/45 [16:39<29:59, 59.99s/it] {'loss': '1.025', 'grad_norm': '0.7377', 'learning_rate': '8.198e-05', 'ppl': '2.788', 'memory/max_active (GiB)': '72.06', 'memory/max_allocated (GiB)': '72.06', 'memory/device_reserved (GiB)': '75.09', 'tokens/train_per_sec_per_gpu': '1.974', 'tokens/total': 983040, 'tokens/trainable': 18682, 'epoch': '0.6685'}
33%|β–ˆβ–ˆβ–ˆβ–Ž | 15/45 [16:39<29:59, 59.99s/it] 36%|β–ˆβ–ˆβ–ˆβ–Œ | 16/45 [17:39<28:57, 59.92s/it] 38%|β–ˆβ–ˆβ–ˆβ–Š | 17/45 [18:38<27:56, 59.88s/it] 40%|β–ˆβ–ˆβ–ˆβ–ˆ | 18/45 [19:38<26:56, 59.87s/it] 42%|β–ˆβ–ˆβ–ˆβ–ˆβ– | 19/45 [20:38<25:56, 59.86s/it] 44%|β–ˆβ–ˆβ–ˆβ–ˆβ– | 20/45 [21:38<24:56, 59.85s/it] {'loss': '0.9663', 'grad_norm': '0.9196', 'learning_rate': '6.614e-05', 'ppl': '2.628', 'memory/max_active (GiB)': '72.06', 'memory/max_allocated (GiB)': '72.06', 'memory/device_reserved (GiB)': '75.09', 'tokens/train_per_sec_per_gpu': '2.29', 'tokens/total': 1310720, 'tokens/trainable': 24574, 'epoch': '0.8914'}
44%|β–ˆβ–ˆβ–ˆβ–ˆβ– | 20/45 [21:38<24:56, 59.85s/it] 47%|β–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 21/45 [22:38<23:55, 59.83s/it] 49%|β–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 22/45 [23:38<22:55, 59.81s/it] 51%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 23/45 [24:04<18:14, 49.75s/it] 53%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 24/45 [25:05<18:34, 53.07s/it] 56%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 25/45 [26:04<18:21, 55.08s/it] {'loss': '0.8469', 'grad_norm': '0.7954', 'learning_rate': '4.817e-05', 'ppl': '2.332', 'memory/max_active (GiB)': '72.06', 'memory/max_allocated (GiB)': '72.06', 'memory/device_reserved (GiB)': '75.09', 'tokens/train_per_sec_per_gpu': '0.5188', 'tokens/total': 1601536, 'tokens/trainable': 29932, 'epoch': '1.089'}
56%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 25/45 [26:04<18:21, 55.08s/it] 58%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 26/45 [27:04<17:53, 56.50s/it] 60%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 27/45 [28:04<17:14, 57.48s/it] 62%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 28/45 [29:04<16:28, 58.17s/it] 64%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 29/45 [30:04<15:38, 58.65s/it] 67%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 30/45 [31:03<14:44, 59.00s/it] {'loss': '0.617', 'grad_norm': '0.7464', 'learning_rate': '3.044e-05', 'ppl': '1.853', 'memory/max_active (GiB)': '72.06', 'memory/max_allocated (GiB)': '72.06', 'memory/device_reserved (GiB)': '75.09', 'tokens/train_per_sec_per_gpu': '0.7193', 'tokens/total': 1929216, 'tokens/trainable': 35957, 'epoch': '1.312'}
67%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 30/45 [31:03<14:44, 59.00s/it] 69%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 31/45 [32:03<13:49, 59.23s/it] 71%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 32/45 [33:03<12:52, 59.39s/it] 73%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 33/45 [34:03<11:54, 59.53s/it] 76%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 34/45 [35:03<10:56, 59.65s/it] 78%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 35/45 [36:02<09:56, 59.69s/it] {'loss': '0.5985', 'grad_norm': '0.8065', 'learning_rate': '1.53e-05', 'ppl': '1.819', 'memory/max_active (GiB)': '72.06', 'memory/max_allocated (GiB)': '72.06', 'memory/device_reserved (GiB)': '75.09', 'tokens/train_per_sec_per_gpu': '1.405', 'tokens/total': 2256896, 'tokens/trainable': 41669, 'epoch': '1.535'}
78%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 35/45 [36:02<09:56, 59.69s/it] 80%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 36/45 [37:02<08:57, 59.70s/it] 82%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 37/45 [38:02<07:57, 59.73s/it] 84%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 38/45 [39:02<06:58, 59.78s/it] 87%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 39/45 [40:02<05:58, 59.79s/it] 89%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 40/45 [41:01<04:58, 59.78s/it] {'loss': '0.5577', 'grad_norm': '0.8635', 'learning_rate': '4.728e-06', 'ppl': '1.747', 'memory/max_active (GiB)': '72.06', 'memory/max_allocated (GiB)': '72.06', 'memory/device_reserved (GiB)': '75.09', 'tokens/train_per_sec_per_gpu': '2.108', 'tokens/total': 2584576, 'tokens/trainable': 48077, 'epoch': '1.758'}
89%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 40/45 [41:01<04:58, 59.78s/it] 91%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 41/45 [42:01<03:59, 59.77s/it] 93%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž| 42/45 [43:01<02:59, 59.77s/it] 96%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ| 43/45 [44:01<01:59, 59.78s/it] 98%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š| 44/45 [45:01<00:59, 59.77s/it] 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 45/45 [46:00<00:00, 59.77s/it] {'loss': '0.5623', 'grad_norm': '0.9089', 'learning_rate': '1.334e-07', 'ppl': '1.755', 'memory/max_active (GiB)': '72.06', 'memory/max_allocated (GiB)': '72.06', 'memory/device_reserved (GiB)': '75.09', 'tokens/train_per_sec_per_gpu': '1.808', 'tokens/total': 2912256, 'tokens/trainable': 54318, 'epoch': '1.981'}
100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 45/45 [46:00<00:00, 59.77s/it][2026-06-24 13:38:42,908] [INFO] [axolotl.core.trainers.base.evaluate:475] [PID:37810] Running evaluation step...
0%| | 0/30 [00:00<?, ?it/s]
7%|β–‹ | 2/30 [00:01<00:14, 1.88it/s]
10%|β–ˆ | 3/30 [00:02<00:20, 1.32it/s]
13%|β–ˆβ–Ž | 4/30 [00:03<00:22, 1.15it/s]
17%|β–ˆβ–‹ | 5/30 [00:04<00:23, 1.06it/s]
20%|β–ˆβ–ˆ | 6/30 [00:05<00:23, 1.02it/s]
23%|β–ˆβ–ˆβ–Ž | 7/30 [00:06<00:23, 1.01s/it]
27%|β–ˆβ–ˆβ–‹ | 8/30 [00:07<00:22, 1.03s/it]
30%|β–ˆβ–ˆβ–ˆ | 9/30 [00:08<00:21, 1.04s/it]
33%|β–ˆβ–ˆβ–ˆβ–Ž | 10/30 [00:09<00:21, 1.05s/it]
37%|β–ˆβ–ˆβ–ˆβ–‹ | 11/30 [00:10<00:20, 1.06s/it]
40%|β–ˆβ–ˆβ–ˆβ–ˆ | 12/30 [00:11<00:19, 1.06s/it]
43%|β–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 13/30 [00:12<00:18, 1.06s/it]
47%|β–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 14/30 [00:13<00:17, 1.07s/it]
50%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 15/30 [00:14<00:16, 1.07s/it]
53%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 16/30 [00:16<00:14, 1.07s/it]
57%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 17/30 [00:17<00:15, 1.20s/it]
60%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 18/30 [00:18<00:13, 1.14s/it]
63%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 19/30 [00:19<00:12, 1.12s/it]
67%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 20/30 [00:20<00:11, 1.11s/it]
70%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 21/30 [00:21<00:09, 1.10s/it]
73%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 22/30 [00:22<00:08, 1.09s/it]
77%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 23/30 [00:23<00:07, 1.08s/it]
80%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 24/30 [00:24<00:06, 1.08s/it]
83%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 25/30 [00:26<00:05, 1.08s/it]
87%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 26/30 [00:27<00:04, 1.07s/it]
90%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 27/30 [00:28<00:03, 1.07s/it]
93%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž| 28/30 [00:29<00:02, 1.07s/it]
97%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹| 29/30 [00:30<00:01, 1.07s/it]
100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 30/30 [00:31<00:00, 1.10s/it]
{'eval_loss': '0.8184', 'eval_runtime': '32.84', 'eval_samples_per_second': '0.914', 'eval_steps_per_second': '0.914', 'eval_ppl': '2.267', 'memory/max_active (GiB)': '66.74', 'memory/max_allocated (GiB)': '66.74', 'memory/device_reserved (GiB)': '75.09', 'epoch': '1.981', 'tokens/train_per_sec_per_gpu': '0'}
100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 45/45 [46:33<00:00, 59.77s/it]
100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 30/30 [00:31<00:00, 1.10s/it]
[2026-06-24 13:39:15,755] [INFO] [axolotl.core.trainers.base._save:828] [PID:37810] Saving model checkpoint to ./outputs/qwen36-27b-agent-memory-sft-lora-axolotl/checkpoint-45
wandb: Adding directory to artifact (outputs/qwen36-27b-agent-memory-sft-lora-axolotl/checkpoint-45)... Done. 6.0s
{'train_runtime': '2809', 'train_samples_per_second': '0.256', 'train_steps_per_second': '0.016', 'train_loss': '0.8578', 'memory/max_active (GiB)': '56.36', 'memory/max_allocated (GiB)': '56.36', 'memory/device_reserved (GiB)': '67.61', 'epoch': '1.981', 'tokens/train_per_sec_per_gpu': '0'}
100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 45/45 [46:44<00:00, 59.77s/it] 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 45/45 [47:01<00:00, 62.69s/it]
[2026-06-24 13:39:43,320] [INFO] [axolotl.train.save_trained_model:267] [PID:37810] Training completed! Saving trained model to ./outputs/qwen36-27b-agent-memory-sft-lora-axolotl.
[2026-06-24 13:39:44,079] [INFO] [axolotl.train.save_trained_model:388] [PID:37810] Model successfully saved to ./outputs/qwen36-27b-agent-memory-sft-lora-axolotl
[2026-06-24 13:39:44,270] [INFO] [axolotl.core.trainers.base._save:828] [PID:37810] Saving model checkpoint to ./outputs/qwen36-27b-agent-memory-sft-lora-axolotl
[2026-06-24 13:39:45,109] [ERROR] [axolotl.telemetry.errors.wrapper:158] [PID:37810] Error captured in telemetry. Run ID: 37a2bad5-0b99-441f-a93c-4b23f0b23d19
Traceback (most recent call last):
File "/root/miniconda3/lib/python3.10/site-packages/huggingface_hub/utils/_http.py", line 761, in hf_raise_for_status
response.raise_for_status()
File "/root/miniconda3/lib/python3.10/site-packages/httpx/_models.py", line 829, in raise_for_status
raise HTTPStatusError(message, request=request, response=self)
httpx.HTTPStatusError: Client error '400 Bad Request' for url 'https://huggingface.co/api/validate-yaml'
For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/400
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/root/miniconda3/lib/python3.10/site-packages/huggingface_hub/hf_api.py", line 11191, in _validate_yaml
hf_raise_for_status(response)
File "/root/miniconda3/lib/python3.10/site-packages/huggingface_hub/utils/_http.py", line 841, in hf_raise_for_status
raise _format(BadRequestError, message, response) from e
huggingface_hub.errors.BadRequestError: (Request ID: Root=1-6a3bde21-0ec1b10c4b5c85cb56bb36a3;a4c1317d-0162-4f09-ae83-92201a21fe1f)
Bad request:
"datasets[0]" with value "./fine-tuning/data/seed/sft_train.jsonl" is not valid. If possible, use a dataset id from https://hf.co/datasets.
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/root/miniconda3/lib/python3.10/runpy.py", line 196, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/root/miniconda3/lib/python3.10/runpy.py", line 86, in _run_code
exec(code, run_globals)
File "/root/miniconda3/lib/python3.10/site-packages/axolotl/cli/train.py", line 154, in <module>
fire.Fire(do_cli)
File "/root/miniconda3/lib/python3.10/site-packages/fire/core.py", line 135, in Fire
component_trace = _Fire(component, args, parsed_flag_args, context, name)
File "/root/miniconda3/lib/python3.10/site-packages/fire/core.py", line 468, in _Fire
component, remaining_args = _CallAndUpdateTrace(
File "/root/miniconda3/lib/python3.10/site-packages/fire/core.py", line 684, in _CallAndUpdateTrace
component = fn(*varargs, **kwargs)
File "/root/miniconda3/lib/python3.10/site-packages/axolotl/cli/train.py", line 96, in do_cli
do_train(parsed_cfg, parsed_cli_args)
File "/root/miniconda3/lib/python3.10/site-packages/axolotl/cli/train.py", line 50, in do_train
model, tokenizer, trainer = train(cfg=cfg, dataset_meta=dataset_meta)
File "/root/miniconda3/lib/python3.10/site-packages/axolotl/telemetry/errors.py", line 127, in wrapper
return func(*args, **kwargs)
File "/root/miniconda3/lib/python3.10/site-packages/axolotl/train.py", line 639, in train
create_model_card(cfg, trainer)
File "/root/miniconda3/lib/python3.10/site-packages/axolotl/train.py", line 424, in create_model_card
trainer.push_to_hub()
File "/root/miniconda3/lib/python3.10/site-packages/axolotl/core/trainers/base.py", line 675, in push_to_hub
return super().push_to_hub(*args, **kwargs)
File "/root/miniconda3/lib/python3.10/site-packages/transformers/trainer.py", line 4073, in push_to_hub
return upload_folder(
File "/root/miniconda3/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py", line 88, in _inner_fn
return fn(*args, **kwargs)
File "/root/miniconda3/lib/python3.10/site-packages/huggingface_hub/hf_api.py", line 2113, in _inner
return fn(self, *args, **kwargs)
File "/root/miniconda3/lib/python3.10/site-packages/huggingface_hub/hf_api.py", line 5803, in upload_folder
add_operations = self._prepare_upload_folder_additions(
File "/root/miniconda3/lib/python3.10/site-packages/huggingface_hub/hf_api.py", line 11130, in _prepare_upload_folder_additions
self._validate_yaml(
File "/root/miniconda3/lib/python3.10/site-packages/huggingface_hub/hf_api.py", line 11195, in _validate_yaml
raise ValueError(f"Invalid metadata in README.md.\n{message}") from e
ValueError: Invalid metadata in README.md.
- "datasets[0]" with value "./fine-tuning/data/seed/sft_train.jsonl" is not valid. If possible, use a dataset id from https://hf.co/datasets.
Traceback (most recent call last):
File "/root/miniconda3/lib/python3.10/site-packages/huggingface_hub/utils/_http.py", line 761, in hf_raise_for_status
response.raise_for_status()
File "/root/miniconda3/lib/python3.10/site-packages/httpx/_models.py", line 829, in raise_for_status
raise HTTPStatusError(message, request=request, response=self)
httpx.HTTPStatusError: Client error '400 Bad Request' for url 'https://huggingface.co/api/validate-yaml'
For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/400
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/root/miniconda3/lib/python3.10/site-packages/huggingface_hub/hf_api.py", line 11191, in _validate_yaml
hf_raise_for_status(response)
File "/root/miniconda3/lib/python3.10/site-packages/huggingface_hub/utils/_http.py", line 841, in hf_raise_for_status
raise _format(BadRequestError, message, response) from e
huggingface_hub.errors.BadRequestError: (Request ID: Root=1-6a3bde21-0ec1b10c4b5c85cb56bb36a3;a4c1317d-0162-4f09-ae83-92201a21fe1f)
Bad request:
"datasets[0]" with value "./fine-tuning/data/seed/sft_train.jsonl" is not valid. If possible, use a dataset id from https://hf.co/datasets.
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/root/miniconda3/lib/python3.10/runpy.py", line 196, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/root/miniconda3/lib/python3.10/runpy.py", line 86, in _run_code
exec(code, run_globals)
File "/root/miniconda3/lib/python3.10/site-packages/axolotl/cli/train.py", line 154, in <module>
fire.Fire(do_cli)
File "/root/miniconda3/lib/python3.10/site-packages/fire/core.py", line 135, in Fire
component_trace = _Fire(component, args, parsed_flag_args, context, name)
File "/root/miniconda3/lib/python3.10/site-packages/fire/core.py", line 468, in _Fire
component, remaining_args = _CallAndUpdateTrace(
File "/root/miniconda3/lib/python3.10/site-packages/fire/core.py", line 684, in _CallAndUpdateTrace
component = fn(*varargs, **kwargs)
File "/root/miniconda3/lib/python3.10/site-packages/axolotl/cli/train.py", line 96, in do_cli
do_train(parsed_cfg, parsed_cli_args)
File "/root/miniconda3/lib/python3.10/site-packages/axolotl/cli/train.py", line 50, in do_train
model, tokenizer, trainer = train(cfg=cfg, dataset_meta=dataset_meta)
File "/root/miniconda3/lib/python3.10/site-packages/axolotl/telemetry/errors.py", line 127, in wrapper
return func(*args, **kwargs)
File "/root/miniconda3/lib/python3.10/site-packages/axolotl/train.py", line 639, in train
create_model_card(cfg, trainer)
File "/root/miniconda3/lib/python3.10/site-packages/axolotl/train.py", line 424, in create_model_card
trainer.push_to_hub()
File "/root/miniconda3/lib/python3.10/site-packages/axolotl/core/trainers/base.py", line 675, in push_to_hub
return super().push_to_hub(*args, **kwargs)
File "/root/miniconda3/lib/python3.10/site-packages/transformers/trainer.py", line 4073, in push_to_hub
return upload_folder(
File "/root/miniconda3/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py", line 88, in _inner_fn
return fn(*args, **kwargs)
File "/root/miniconda3/lib/python3.10/site-packages/huggingface_hub/hf_api.py", line 2113, in _inner
return fn(self, *args, **kwargs)
File "/root/miniconda3/lib/python3.10/site-packages/huggingface_hub/hf_api.py", line 5803, in upload_folder
add_operations = self._prepare_upload_folder_additions(
File "/root/miniconda3/lib/python3.10/site-packages/huggingface_hub/hf_api.py", line 11130, in _prepare_upload_folder_additions
self._validate_yaml(
File "/root/miniconda3/lib/python3.10/site-packages/huggingface_hub/hf_api.py", line 11195, in _validate_yaml
raise ValueError(f"Invalid metadata in README.md.\n{message}") from e
ValueError: Invalid metadata in README.md.
- "datasets[0]" with value "./fine-tuning/data/seed/sft_train.jsonl" is not valid. If possible, use a dataset id from https://hf.co/datasets.