diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..c68e018dd309b4926322e1acfd559e2eaed4bd45 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,15 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +qwen2.5-1.5b-instruct/checkpoint-142/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen2.5-1.5b-instruct/checkpoint-213/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen2.5-1.5b-instruct/checkpoint-71/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen2.5-1.5b-instruct/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen2.5-3b-instruct/checkpoint-142/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen2.5-3b-instruct/checkpoint-213/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen2.5-3b-instruct/checkpoint-71/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen2.5-3b-instruct/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen2.5-7b-instruct/checkpoint-142/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen2.5-7b-instruct/checkpoint-213/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen2.5-7b-instruct/checkpoint-71/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen2.5-7b-instruct/tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/qwen2.5-1.5b-instruct/README.md b/qwen2.5-1.5b-instruct/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2845afbd43ef7aded0222e393c53b061edb88ded --- /dev/null +++ b/qwen2.5-1.5b-instruct/README.md @@ -0,0 +1,69 @@ +--- +library_name: transformers +license: other +base_model: Qwen/Qwen2.5-1.5B-Instruct +tags: +- llama-factory +- full +- generated_from_trainer +model-index: +- name: qwen2.5-1.5b-instruct + results: [] +--- + + + +# qwen2.5-1.5b-instruct + +This model is a fine-tuned version of [Qwen/Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct) on the codealpaca_train_20k_trace_generation_valid dataset. +It achieves the following results on the evaluation set: +- Loss: 0.5171 + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 1e-05 +- train_batch_size: 1 +- eval_batch_size: 1 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 2 +- gradient_accumulation_steps: 128 +- total_train_batch_size: 256 +- total_eval_batch_size: 2 +- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_steps: 0.1 +- num_epochs: 3 + +### Training results + +| Training Loss | Epoch | Step | Validation Loss | +|:-------------:|:------:|:----:|:---------------:| +| 0.5380 | 0.7111 | 50 | 0.5587 | +| 0.5060 | 1.4124 | 100 | 0.5292 | +| 0.4743 | 2.1138 | 150 | 0.5197 | +| 0.4765 | 2.8249 | 200 | 0.5171 | + + +### Framework versions + +- Transformers 5.2.0 +- Pytorch 2.11.0+cu128 +- Datasets 4.0.0 +- Tokenizers 0.22.2 diff --git a/qwen2.5-1.5b-instruct/all_results.json b/qwen2.5-1.5b-instruct/all_results.json new file mode 100644 index 0000000000000000000000000000000000000000..471eecadc651aa7dbdf4169141c09a9081917ccc --- /dev/null +++ b/qwen2.5-1.5b-instruct/all_results.json @@ -0,0 +1,12 @@ +{ + "epoch": 3.0, + "eval_loss": 0.5171241164207458, + "eval_runtime": 8.31, + "eval_samples_per_second": 24.067, + "eval_steps_per_second": 12.034, + "total_flos": 2.4727435500270387e+17, + "train_loss": 0.5265021298972654, + "train_runtime": 6756.1203, + "train_samples_per_second": 7.993, + "train_steps_per_second": 0.032 +} \ No newline at end of file diff --git a/qwen2.5-1.5b-instruct/chat_template.jinja b/qwen2.5-1.5b-instruct/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e --- /dev/null +++ b/qwen2.5-1.5b-instruct/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/qwen2.5-1.5b-instruct/checkpoint-142/chat_template.jinja b/qwen2.5-1.5b-instruct/checkpoint-142/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-142/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/qwen2.5-1.5b-instruct/checkpoint-142/config.json b/qwen2.5-1.5b-instruct/checkpoint-142/config.json new file mode 100644 index 0000000000000000000000000000000000000000..499065fd744a64fc8ad2884da26e85a6e99d468a --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-142/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 21, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/qwen2.5-1.5b-instruct/checkpoint-142/generation_config.json b/qwen2.5-1.5b-instruct/checkpoint-142/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c5b4e11411fed3e7a5e6557165f888a7b0b1248b --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-142/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.2.0" +} diff --git a/qwen2.5-1.5b-instruct/checkpoint-142/model.safetensors b/qwen2.5-1.5b-instruct/checkpoint-142/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fb66704e881025cbcbb2cab7d8055b305c4027b8 --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-142/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bc57373d2a4ede43535c112481a822694621c0d07fe2d1a9ffed15afd3cd6d9 +size 6174895536 diff --git a/qwen2.5-1.5b-instruct/checkpoint-142/optimizer.pt b/qwen2.5-1.5b-instruct/checkpoint-142/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..228937b778845d73feb5df8e81303f7929db9f6f --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-142/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0f047a0b7066f7bc7eb1e6f175b6ea18ddf4b1d0ee7cf376e26fb4fbf7b071c +size 12350013801 diff --git a/qwen2.5-1.5b-instruct/checkpoint-142/rng_state_0.pth b/qwen2.5-1.5b-instruct/checkpoint-142/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..870021e3fa5ac35c2f711adf0c93a556ab4842da --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-142/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95e5fc2074c0df31522a514f862c86cb00d71c946a7f15cc9ec0e53a69fb28a7 +size 14917 diff --git a/qwen2.5-1.5b-instruct/checkpoint-142/rng_state_1.pth b/qwen2.5-1.5b-instruct/checkpoint-142/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..21f20da5eb1da017f08aaa88bd19cf24d40e3fbf --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-142/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e7153eae67b6c9232a41bc996a2bf5b83229b8c7230d61911ac0fd40e64154e +size 14917 diff --git a/qwen2.5-1.5b-instruct/checkpoint-142/scheduler.pt b/qwen2.5-1.5b-instruct/checkpoint-142/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..2967e7243507a115f9424b1b386bf3657f047c75 --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-142/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:922deb71175797a96c4bddc5e1cef1c097df38c0dfa52334e97c89e0f573f2bb +size 1465 diff --git a/qwen2.5-1.5b-instruct/checkpoint-142/tokenizer.json b/qwen2.5-1.5b-instruct/checkpoint-142/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..34510ff0037cd50428af467a17ead5a96140a32c --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-142/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/qwen2.5-1.5b-instruct/checkpoint-142/tokenizer_config.json b/qwen2.5-1.5b-instruct/checkpoint-142/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..145e2c7a68b109ae0253dda29fc2e66cb01b4b85 --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-142/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen2.5-1.5b-instruct/checkpoint-142/trainer_state.json b/qwen2.5-1.5b-instruct/checkpoint-142/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c52409cade84ee95b6ef94f09ed2dfc2f3c177b8 --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-142/trainer_state.json @@ -0,0 +1,1044 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 50, + "global_step": 142, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.014222222222222223, + "grad_norm": 0.03586147725582123, + "learning_rate": 0.0, + "loss": 0.9098724722862244, + "step": 1 + }, + { + "epoch": 0.028444444444444446, + "grad_norm": 0.03577122464776039, + "learning_rate": 4.5454545454545457e-07, + "loss": 0.9084522724151611, + "step": 2 + }, + { + "epoch": 0.042666666666666665, + "grad_norm": 0.031653374433517456, + "learning_rate": 9.090909090909091e-07, + "loss": 0.8834755420684814, + "step": 3 + }, + { + "epoch": 0.05688888888888889, + "grad_norm": 0.025831686332821846, + "learning_rate": 1.3636363636363636e-06, + "loss": 0.871230959892273, + "step": 4 + }, + { + "epoch": 0.07111111111111111, + "grad_norm": 0.019876576960086823, + "learning_rate": 1.8181818181818183e-06, + "loss": 0.7917206287384033, + "step": 5 + }, + { + "epoch": 0.08533333333333333, + "grad_norm": 0.018677273765206337, + "learning_rate": 2.2727272727272728e-06, + "loss": 0.8436126708984375, + "step": 6 + }, + { + "epoch": 0.09955555555555555, + "grad_norm": 0.017528358846902847, + "learning_rate": 2.7272727272727272e-06, + "loss": 0.8075190782546997, + "step": 7 + }, + { + "epoch": 0.11377777777777778, + "grad_norm": 0.01399513240903616, + "learning_rate": 3.181818181818182e-06, + "loss": 0.7754716873168945, + "step": 8 + }, + { + "epoch": 0.128, + "grad_norm": 0.012644859962165356, + "learning_rate": 3.6363636363636366e-06, + "loss": 0.7459083199501038, + "step": 9 + }, + { + "epoch": 0.14222222222222222, + "grad_norm": 0.01209743320941925, + "learning_rate": 4.0909090909090915e-06, + "loss": 0.7365868091583252, + "step": 10 + }, + { + "epoch": 0.15644444444444444, + "grad_norm": 0.012133832089602947, + "learning_rate": 4.5454545454545455e-06, + "loss": 0.7119501829147339, + "step": 11 + }, + { + "epoch": 0.17066666666666666, + "grad_norm": 0.010017997585237026, + "learning_rate": 5e-06, + "loss": 0.6901087760925293, + "step": 12 + }, + { + "epoch": 0.18488888888888888, + "grad_norm": 0.009233366698026657, + "learning_rate": 5.4545454545454545e-06, + "loss": 0.6611638069152832, + "step": 13 + }, + { + "epoch": 0.1991111111111111, + "grad_norm": 0.00782010331749916, + "learning_rate": 5.90909090909091e-06, + "loss": 0.6715224981307983, + "step": 14 + }, + { + "epoch": 0.21333333333333335, + "grad_norm": 0.007341764867305756, + "learning_rate": 6.363636363636364e-06, + "loss": 0.6277110576629639, + "step": 15 + }, + { + "epoch": 0.22755555555555557, + "grad_norm": 0.00713470159098506, + "learning_rate": 6.818181818181818e-06, + "loss": 0.6689565181732178, + "step": 16 + }, + { + "epoch": 0.24177777777777779, + "grad_norm": 0.0067572277039289474, + "learning_rate": 7.272727272727273e-06, + "loss": 0.6363875865936279, + "step": 17 + }, + { + "epoch": 0.256, + "grad_norm": 0.006228511221706867, + "learning_rate": 7.727272727272727e-06, + "loss": 0.6257224678993225, + "step": 18 + }, + { + "epoch": 0.2702222222222222, + "grad_norm": 0.006048806011676788, + "learning_rate": 8.181818181818183e-06, + "loss": 0.6355900764465332, + "step": 19 + }, + { + "epoch": 0.28444444444444444, + "grad_norm": 0.005988306365907192, + "learning_rate": 8.636363636363637e-06, + "loss": 0.6338369846343994, + "step": 20 + }, + { + "epoch": 0.2986666666666667, + "grad_norm": 0.005441761109977961, + "learning_rate": 9.090909090909091e-06, + "loss": 0.6133573055267334, + "step": 21 + }, + { + "epoch": 0.3128888888888889, + "grad_norm": 0.005174445919692516, + "learning_rate": 9.545454545454547e-06, + "loss": 0.6042410135269165, + "step": 22 + }, + { + "epoch": 0.32711111111111113, + "grad_norm": 0.005199746694415808, + "learning_rate": 1e-05, + "loss": 0.602219820022583, + "step": 23 + }, + { + "epoch": 0.3413333333333333, + "grad_norm": 0.005485028959810734, + "learning_rate": 9.999323662872998e-06, + "loss": 0.6062123775482178, + "step": 24 + }, + { + "epoch": 0.35555555555555557, + "grad_norm": 0.004896003287285566, + "learning_rate": 9.99729483446475e-06, + "loss": 0.6077945232391357, + "step": 25 + }, + { + "epoch": 0.36977777777777776, + "grad_norm": 0.00495151337236166, + "learning_rate": 9.993914063644053e-06, + "loss": 0.5923135280609131, + "step": 26 + }, + { + "epoch": 0.384, + "grad_norm": 0.004744194447994232, + "learning_rate": 9.989182265027232e-06, + "loss": 0.5860621929168701, + "step": 27 + }, + { + "epoch": 0.3982222222222222, + "grad_norm": 0.004471767693758011, + "learning_rate": 9.98310071873072e-06, + "loss": 0.5823097229003906, + "step": 28 + }, + { + "epoch": 0.41244444444444445, + "grad_norm": 0.004788320977240801, + "learning_rate": 9.975671070024741e-06, + "loss": 0.5872600078582764, + "step": 29 + }, + { + "epoch": 0.4266666666666667, + "grad_norm": 0.0044632842764258385, + "learning_rate": 9.966895328888195e-06, + "loss": 0.6070278286933899, + "step": 30 + }, + { + "epoch": 0.4408888888888889, + "grad_norm": 0.004317905753850937, + "learning_rate": 9.956775869464901e-06, + "loss": 0.5780316591262817, + "step": 31 + }, + { + "epoch": 0.45511111111111113, + "grad_norm": 0.004291852004826069, + "learning_rate": 9.945315429421307e-06, + "loss": 0.5697590112686157, + "step": 32 + }, + { + "epoch": 0.4693333333333333, + "grad_norm": 0.003922830801457167, + "learning_rate": 9.932517109205849e-06, + "loss": 0.5747107267379761, + "step": 33 + }, + { + "epoch": 0.48355555555555557, + "grad_norm": 0.0041889771819114685, + "learning_rate": 9.918384371210178e-06, + "loss": 0.6021177172660828, + "step": 34 + }, + { + "epoch": 0.49777777777777776, + "grad_norm": 0.004147357773035765, + "learning_rate": 9.902921038832456e-06, + "loss": 0.5805741548538208, + "step": 35 + }, + { + "epoch": 0.512, + "grad_norm": 0.0040724254213273525, + "learning_rate": 9.886131295443003e-06, + "loss": 0.5483200550079346, + "step": 36 + }, + { + "epoch": 0.5262222222222223, + "grad_norm": 0.004085644613951445, + "learning_rate": 9.868019683252543e-06, + "loss": 0.557392954826355, + "step": 37 + }, + { + "epoch": 0.5404444444444444, + "grad_norm": 0.004289102274924517, + "learning_rate": 9.848591102083375e-06, + "loss": 0.590857207775116, + "step": 38 + }, + { + "epoch": 0.5546666666666666, + "grad_norm": 0.0040345704182982445, + "learning_rate": 9.82785080804381e-06, + "loss": 0.5750541090965271, + "step": 39 + }, + { + "epoch": 0.5688888888888889, + "grad_norm": 0.004034748766571283, + "learning_rate": 9.805804412106197e-06, + "loss": 0.5949826240539551, + "step": 40 + }, + { + "epoch": 0.5831111111111111, + "grad_norm": 0.003911909181624651, + "learning_rate": 9.782457878588977e-06, + "loss": 0.5623177289962769, + "step": 41 + }, + { + "epoch": 0.5973333333333334, + "grad_norm": 0.0037001846358180046, + "learning_rate": 9.75781752354311e-06, + "loss": 0.5465081930160522, + "step": 42 + }, + { + "epoch": 0.6115555555555555, + "grad_norm": 0.0037095139268785715, + "learning_rate": 9.731890013043367e-06, + "loss": 0.5752100944519043, + "step": 43 + }, + { + "epoch": 0.6257777777777778, + "grad_norm": 0.003655617358162999, + "learning_rate": 9.704682361384941e-06, + "loss": 0.5452020168304443, + "step": 44 + }, + { + "epoch": 0.64, + "grad_norm": 0.003784338477998972, + "learning_rate": 9.676201929185809e-06, + "loss": 0.5616332292556763, + "step": 45 + }, + { + "epoch": 0.6542222222222223, + "grad_norm": 0.003718078136444092, + "learning_rate": 9.646456421395447e-06, + "loss": 0.5558297634124756, + "step": 46 + }, + { + "epoch": 0.6684444444444444, + "grad_norm": 0.00395169947296381, + "learning_rate": 9.615453885210368e-06, + "loss": 0.5600595474243164, + "step": 47 + }, + { + "epoch": 0.6826666666666666, + "grad_norm": 0.003555440343916416, + "learning_rate": 9.583202707897075e-06, + "loss": 0.5372895002365112, + "step": 48 + }, + { + "epoch": 0.6968888888888889, + "grad_norm": 0.0037386869080364704, + "learning_rate": 9.549711614523007e-06, + "loss": 0.5322733521461487, + "step": 49 + }, + { + "epoch": 0.7111111111111111, + "grad_norm": 0.003739988896995783, + "learning_rate": 9.514989665596114e-06, + "loss": 0.5380419492721558, + "step": 50 + }, + { + "epoch": 0.7111111111111111, + "eval_loss": 0.5587145090103149, + "eval_runtime": 8.7676, + "eval_samples_per_second": 22.811, + "eval_steps_per_second": 11.406, + "step": 50 + }, + { + "epoch": 0.7253333333333334, + "grad_norm": 0.0037539806216955185, + "learning_rate": 9.479046254613673e-06, + "loss": 0.5472225546836853, + "step": 51 + }, + { + "epoch": 0.7395555555555555, + "grad_norm": 0.003411181503906846, + "learning_rate": 9.441891105521005e-06, + "loss": 0.5134032964706421, + "step": 52 + }, + { + "epoch": 0.7537777777777778, + "grad_norm": 0.0037538670003414154, + "learning_rate": 9.40353427008083e-06, + "loss": 0.5361013412475586, + "step": 53 + }, + { + "epoch": 0.768, + "grad_norm": 0.0036495670210570097, + "learning_rate": 9.3639861251539e-06, + "loss": 0.5299779772758484, + "step": 54 + }, + { + "epoch": 0.7822222222222223, + "grad_norm": 0.0037342547439038754, + "learning_rate": 9.323257369891702e-06, + "loss": 0.5607945919036865, + "step": 55 + }, + { + "epoch": 0.7964444444444444, + "grad_norm": 0.003572584595531225, + "learning_rate": 9.281359022841966e-06, + "loss": 0.5528460741043091, + "step": 56 + }, + { + "epoch": 0.8106666666666666, + "grad_norm": 0.003770101349800825, + "learning_rate": 9.238302418967757e-06, + "loss": 0.5440288186073303, + "step": 57 + }, + { + "epoch": 0.8248888888888889, + "grad_norm": 0.0036679303739219904, + "learning_rate": 9.194099206580981e-06, + "loss": 0.5564252138137817, + "step": 58 + }, + { + "epoch": 0.8391111111111111, + "grad_norm": 0.003834612900391221, + "learning_rate": 9.14876134419111e-06, + "loss": 0.5473330020904541, + "step": 59 + }, + { + "epoch": 0.8533333333333334, + "grad_norm": 0.003683249931782484, + "learning_rate": 9.102301097269974e-06, + "loss": 0.5463743209838867, + "step": 60 + }, + { + "epoch": 0.8675555555555555, + "grad_norm": 0.0036777963396161795, + "learning_rate": 9.05473103493355e-06, + "loss": 0.524878740310669, + "step": 61 + }, + { + "epoch": 0.8817777777777778, + "grad_norm": 0.0038230482023209333, + "learning_rate": 9.006064026541549e-06, + "loss": 0.5563522577285767, + "step": 62 + }, + { + "epoch": 0.896, + "grad_norm": 0.003645201213657856, + "learning_rate": 8.956313238215824e-06, + "loss": 0.5059154033660889, + "step": 63 + }, + { + "epoch": 0.9102222222222223, + "grad_norm": 0.0035886459518224, + "learning_rate": 8.905492129278478e-06, + "loss": 0.5312881469726562, + "step": 64 + }, + { + "epoch": 0.9244444444444444, + "grad_norm": 0.003721872577443719, + "learning_rate": 8.85361444861063e-06, + "loss": 0.5358959436416626, + "step": 65 + }, + { + "epoch": 0.9386666666666666, + "grad_norm": 0.003683999879285693, + "learning_rate": 8.800694230932885e-06, + "loss": 0.5442053079605103, + "step": 66 + }, + { + "epoch": 0.9528888888888889, + "grad_norm": 0.0036851842887699604, + "learning_rate": 8.74674579300843e-06, + "loss": 0.5263723731040955, + "step": 67 + }, + { + "epoch": 0.9671111111111111, + "grad_norm": 0.003754258854314685, + "learning_rate": 8.691783729769874e-06, + "loss": 0.5200538039207458, + "step": 68 + }, + { + "epoch": 0.9813333333333333, + "grad_norm": 0.0038373274728655815, + "learning_rate": 8.635822910370793e-06, + "loss": 0.5452108383178711, + "step": 69 + }, + { + "epoch": 0.9955555555555555, + "grad_norm": 0.0039480882696807384, + "learning_rate": 8.578878474163115e-06, + "loss": 0.5066030621528625, + "step": 70 + }, + { + "epoch": 1.0, + "grad_norm": 0.005897017661482096, + "learning_rate": 8.520965826601394e-06, + "loss": 0.5380499958992004, + "step": 71 + }, + { + "epoch": 1.0142222222222221, + "grad_norm": 0.0037728557363152504, + "learning_rate": 8.462100635075097e-06, + "loss": 0.5328541398048401, + "step": 72 + }, + { + "epoch": 1.0284444444444445, + "grad_norm": 0.003953218925744295, + "learning_rate": 8.40229882467003e-06, + "loss": 0.5096005201339722, + "step": 73 + }, + { + "epoch": 1.0426666666666666, + "grad_norm": 0.0036587677896022797, + "learning_rate": 8.341576573860049e-06, + "loss": 0.4889724850654602, + "step": 74 + }, + { + "epoch": 1.056888888888889, + "grad_norm": 0.0038338187150657177, + "learning_rate": 8.279950310130218e-06, + "loss": 0.5157173871994019, + "step": 75 + }, + { + "epoch": 1.0711111111111111, + "grad_norm": 0.003540106350556016, + "learning_rate": 8.2174367055326e-06, + "loss": 0.49691659212112427, + "step": 76 + }, + { + "epoch": 1.0853333333333333, + "grad_norm": 0.0037315969821065664, + "learning_rate": 8.154052672175888e-06, + "loss": 0.4873769283294678, + "step": 77 + }, + { + "epoch": 1.0995555555555556, + "grad_norm": 0.003963674418628216, + "learning_rate": 8.08981535765009e-06, + "loss": 0.5266215801239014, + "step": 78 + }, + { + "epoch": 1.1137777777777778, + "grad_norm": 0.0037685995921492577, + "learning_rate": 8.024742140387506e-06, + "loss": 0.5039150714874268, + "step": 79 + }, + { + "epoch": 1.1280000000000001, + "grad_norm": 0.003613364417105913, + "learning_rate": 7.95885062496126e-06, + "loss": 0.4896891117095947, + "step": 80 + }, + { + "epoch": 1.1422222222222222, + "grad_norm": 0.0037347134202718735, + "learning_rate": 7.892158637322647e-06, + "loss": 0.45997822284698486, + "step": 81 + }, + { + "epoch": 1.1564444444444444, + "grad_norm": 0.0037277438677847385, + "learning_rate": 7.824684219978591e-06, + "loss": 0.4917220175266266, + "step": 82 + }, + { + "epoch": 1.1706666666666667, + "grad_norm": 0.0034888957161456347, + "learning_rate": 7.756445627110523e-06, + "loss": 0.5024806261062622, + "step": 83 + }, + { + "epoch": 1.1848888888888889, + "grad_norm": 0.004098390229046345, + "learning_rate": 7.68746131963598e-06, + "loss": 0.5110447406768799, + "step": 84 + }, + { + "epoch": 1.199111111111111, + "grad_norm": 0.0036922753788530827, + "learning_rate": 7.6177499602143e-06, + "loss": 0.5031975507736206, + "step": 85 + }, + { + "epoch": 1.2133333333333334, + "grad_norm": 0.0035749904345721006, + "learning_rate": 7.547330408197695e-06, + "loss": 0.4899296164512634, + "step": 86 + }, + { + "epoch": 1.2275555555555555, + "grad_norm": 0.003764625173062086, + "learning_rate": 7.476221714529167e-06, + "loss": 0.5040560960769653, + "step": 87 + }, + { + "epoch": 1.2417777777777779, + "grad_norm": 0.003831087611615658, + "learning_rate": 7.404443116588548e-06, + "loss": 0.5307536125183105, + "step": 88 + }, + { + "epoch": 1.256, + "grad_norm": 0.003740105777978897, + "learning_rate": 7.332014032988123e-06, + "loss": 0.5025057792663574, + "step": 89 + }, + { + "epoch": 1.2702222222222221, + "grad_norm": 0.0038286929484456778, + "learning_rate": 7.2589540583192165e-06, + "loss": 0.5043497085571289, + "step": 90 + }, + { + "epoch": 1.2844444444444445, + "grad_norm": 0.003593199420720339, + "learning_rate": 7.185282957851175e-06, + "loss": 0.4883122444152832, + "step": 91 + }, + { + "epoch": 1.2986666666666666, + "grad_norm": 0.003721701679751277, + "learning_rate": 7.111020662184174e-06, + "loss": 0.5055882930755615, + "step": 92 + }, + { + "epoch": 1.3128888888888888, + "grad_norm": 0.0036744684912264347, + "learning_rate": 7.036187261857289e-06, + "loss": 0.4962804615497589, + "step": 93 + }, + { + "epoch": 1.3271111111111111, + "grad_norm": 0.004013169556856155, + "learning_rate": 6.960803001913315e-06, + "loss": 0.524259090423584, + "step": 94 + }, + { + "epoch": 1.3413333333333333, + "grad_norm": 0.003746299771592021, + "learning_rate": 6.884888276421766e-06, + "loss": 0.5073269009590149, + "step": 95 + }, + { + "epoch": 1.3555555555555556, + "grad_norm": 0.0036679194308817387, + "learning_rate": 6.8084636229615786e-06, + "loss": 0.4886914789676666, + "step": 96 + }, + { + "epoch": 1.3697777777777778, + "grad_norm": 0.00375540042296052, + "learning_rate": 6.731549717064975e-06, + "loss": 0.4920746088027954, + "step": 97 + }, + { + "epoch": 1.384, + "grad_norm": 0.003901587100699544, + "learning_rate": 6.654167366624009e-06, + "loss": 0.503207266330719, + "step": 98 + }, + { + "epoch": 1.3982222222222223, + "grad_norm": 0.003911704756319523, + "learning_rate": 6.576337506261314e-06, + "loss": 0.5012996196746826, + "step": 99 + }, + { + "epoch": 1.4124444444444444, + "grad_norm": 0.0035876361653208733, + "learning_rate": 6.498081191666549e-06, + "loss": 0.5059630274772644, + "step": 100 + }, + { + "epoch": 1.4124444444444444, + "eval_loss": 0.5291631817817688, + "eval_runtime": 10.3556, + "eval_samples_per_second": 19.313, + "eval_steps_per_second": 9.657, + "step": 100 + }, + { + "epoch": 1.4266666666666667, + "grad_norm": 0.003729900810867548, + "learning_rate": 6.419419593900109e-06, + "loss": 0.4896056056022644, + "step": 101 + }, + { + "epoch": 1.4408888888888889, + "grad_norm": 0.003648703685030341, + "learning_rate": 6.340373993665607e-06, + "loss": 0.48181772232055664, + "step": 102 + }, + { + "epoch": 1.455111111111111, + "grad_norm": 0.0035914399195462465, + "learning_rate": 6.2609657755527135e-06, + "loss": 0.4775124192237854, + "step": 103 + }, + { + "epoch": 1.4693333333333334, + "grad_norm": 0.0035880866926163435, + "learning_rate": 6.1812164222518626e-06, + "loss": 0.5167958736419678, + "step": 104 + }, + { + "epoch": 1.4835555555555555, + "grad_norm": 0.003788268193602562, + "learning_rate": 6.101147508742456e-06, + "loss": 0.4946171045303345, + "step": 105 + }, + { + "epoch": 1.4977777777777779, + "grad_norm": 0.0036002870183438063, + "learning_rate": 6.020780696456059e-06, + "loss": 0.4849076569080353, + "step": 106 + }, + { + "epoch": 1.512, + "grad_norm": 0.0036388293374329805, + "learning_rate": 5.940137727416247e-06, + "loss": 0.4958712160587311, + "step": 107 + }, + { + "epoch": 1.5262222222222221, + "grad_norm": 0.0034927036613225937, + "learning_rate": 5.859240418356614e-06, + "loss": 0.4822971820831299, + "step": 108 + }, + { + "epoch": 1.5404444444444443, + "grad_norm": 0.0038439796771854162, + "learning_rate": 5.778110654818602e-06, + "loss": 0.5280419588088989, + "step": 109 + }, + { + "epoch": 1.5546666666666666, + "grad_norm": 0.003715698141604662, + "learning_rate": 5.696770385230679e-06, + "loss": 0.4952104687690735, + "step": 110 + }, + { + "epoch": 1.568888888888889, + "grad_norm": 0.0036385883577167988, + "learning_rate": 5.615241614970546e-06, + "loss": 0.4887908101081848, + "step": 111 + }, + { + "epoch": 1.5831111111111111, + "grad_norm": 0.0036841081455349922, + "learning_rate": 5.533546400411899e-06, + "loss": 0.4928241968154907, + "step": 112 + }, + { + "epoch": 1.5973333333333333, + "grad_norm": 0.0036721359938383102, + "learning_rate": 5.4517068429574215e-06, + "loss": 0.5109080076217651, + "step": 113 + }, + { + "epoch": 1.6115555555555554, + "grad_norm": 0.0037413975223898888, + "learning_rate": 5.3697450830595775e-06, + "loss": 0.48911499977111816, + "step": 114 + }, + { + "epoch": 1.6257777777777778, + "grad_norm": 0.003966668155044317, + "learning_rate": 5.287683294230855e-06, + "loss": 0.49537116289138794, + "step": 115 + }, + { + "epoch": 1.6400000000000001, + "grad_norm": 0.0036018763203173876, + "learning_rate": 5.20554367704505e-06, + "loss": 0.4613257348537445, + "step": 116 + }, + { + "epoch": 1.6542222222222223, + "grad_norm": 0.0038206754252314568, + "learning_rate": 5.123348453131242e-06, + "loss": 0.5043697357177734, + "step": 117 + }, + { + "epoch": 1.6684444444444444, + "grad_norm": 0.0037831810768693686, + "learning_rate": 5.041119859162068e-06, + "loss": 0.48968735337257385, + "step": 118 + }, + { + "epoch": 1.6826666666666665, + "grad_norm": 0.0038370522670447826, + "learning_rate": 4.958880140837934e-06, + "loss": 0.49605250358581543, + "step": 119 + }, + { + "epoch": 1.696888888888889, + "grad_norm": 0.003586502280086279, + "learning_rate": 4.876651546868759e-06, + "loss": 0.4839916229248047, + "step": 120 + }, + { + "epoch": 1.7111111111111112, + "grad_norm": 0.003694413462653756, + "learning_rate": 4.794456322954953e-06, + "loss": 0.5173561573028564, + "step": 121 + }, + { + "epoch": 1.7253333333333334, + "grad_norm": 0.003933500964194536, + "learning_rate": 4.7123167057691446e-06, + "loss": 0.5177811980247498, + "step": 122 + }, + { + "epoch": 1.7395555555555555, + "grad_norm": 0.0036068507470190525, + "learning_rate": 4.630254916940424e-06, + "loss": 0.5164836049079895, + "step": 123 + }, + { + "epoch": 1.7537777777777777, + "grad_norm": 0.00349752651527524, + "learning_rate": 4.548293157042581e-06, + "loss": 0.5077825784683228, + "step": 124 + }, + { + "epoch": 1.768, + "grad_norm": 0.0037042393814772367, + "learning_rate": 4.466453599588103e-06, + "loss": 0.5107009410858154, + "step": 125 + }, + { + "epoch": 1.7822222222222224, + "grad_norm": 0.003853494767099619, + "learning_rate": 4.384758385029457e-06, + "loss": 0.4728143811225891, + "step": 126 + }, + { + "epoch": 1.7964444444444445, + "grad_norm": 0.0038627111352980137, + "learning_rate": 4.3032296147693225e-06, + "loss": 0.4858134984970093, + "step": 127 + }, + { + "epoch": 1.8106666666666666, + "grad_norm": 0.003657885594293475, + "learning_rate": 4.2218893451814005e-06, + "loss": 0.48357316851615906, + "step": 128 + }, + { + "epoch": 1.8248888888888888, + "grad_norm": 0.00355988135561347, + "learning_rate": 4.140759581643386e-06, + "loss": 0.49349913001060486, + "step": 129 + }, + { + "epoch": 1.8391111111111111, + "grad_norm": 0.004014762118458748, + "learning_rate": 4.059862272583755e-06, + "loss": 0.4969041347503662, + "step": 130 + }, + { + "epoch": 1.8533333333333335, + "grad_norm": 0.003631556173786521, + "learning_rate": 3.979219303543942e-06, + "loss": 0.47194504737854004, + "step": 131 + }, + { + "epoch": 1.8675555555555556, + "grad_norm": 0.003577238181605935, + "learning_rate": 3.898852491257547e-06, + "loss": 0.5020972490310669, + "step": 132 + }, + { + "epoch": 1.8817777777777778, + "grad_norm": 0.003780671861022711, + "learning_rate": 3.818783577748138e-06, + "loss": 0.5002884268760681, + "step": 133 + }, + { + "epoch": 1.896, + "grad_norm": 0.003708280622959137, + "learning_rate": 3.7390342244472886e-06, + "loss": 0.5007095336914062, + "step": 134 + }, + { + "epoch": 1.9102222222222223, + "grad_norm": 0.003498840844258666, + "learning_rate": 3.659626006334395e-06, + "loss": 0.4763813316822052, + "step": 135 + }, + { + "epoch": 1.9244444444444444, + "grad_norm": 0.003659547306597233, + "learning_rate": 3.5805804060998926e-06, + "loss": 0.48733818531036377, + "step": 136 + }, + { + "epoch": 1.9386666666666668, + "grad_norm": 0.003515617921948433, + "learning_rate": 3.501918808333453e-06, + "loss": 0.5118749141693115, + "step": 137 + }, + { + "epoch": 1.952888888888889, + "grad_norm": 0.0036544103641062975, + "learning_rate": 3.4236624937386874e-06, + "loss": 0.5178316831588745, + "step": 138 + }, + { + "epoch": 1.967111111111111, + "grad_norm": 0.003433604259043932, + "learning_rate": 3.3458326333759927e-06, + "loss": 0.4853537678718567, + "step": 139 + }, + { + "epoch": 1.9813333333333332, + "grad_norm": 0.003547502215951681, + "learning_rate": 3.268450282935026e-06, + "loss": 0.49210506677627563, + "step": 140 + }, + { + "epoch": 1.9955555555555555, + "grad_norm": 0.003462104359641671, + "learning_rate": 3.1915363770384223e-06, + "loss": 0.47841814160346985, + "step": 141 + }, + { + "epoch": 2.0, + "grad_norm": 0.005856912117451429, + "learning_rate": 3.1151117235782346e-06, + "loss": 0.5034412145614624, + "step": 142 + } + ], + "logging_steps": 1, + "max_steps": 213, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.648495698116608e+17, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/qwen2.5-1.5b-instruct/checkpoint-142/training_args.bin b/qwen2.5-1.5b-instruct/checkpoint-142/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..4badd3789f92f64e02e2816810bfe346973afac5 --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-142/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39e78c30a5784e935e65cb8fa1a73b2afb467afccc55e7223863b667252fac3b +size 5649 diff --git a/qwen2.5-1.5b-instruct/checkpoint-213/chat_template.jinja b/qwen2.5-1.5b-instruct/checkpoint-213/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-213/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/qwen2.5-1.5b-instruct/checkpoint-213/config.json b/qwen2.5-1.5b-instruct/checkpoint-213/config.json new file mode 100644 index 0000000000000000000000000000000000000000..499065fd744a64fc8ad2884da26e85a6e99d468a --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-213/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 21, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/qwen2.5-1.5b-instruct/checkpoint-213/generation_config.json b/qwen2.5-1.5b-instruct/checkpoint-213/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c5b4e11411fed3e7a5e6557165f888a7b0b1248b --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-213/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.2.0" +} diff --git a/qwen2.5-1.5b-instruct/checkpoint-213/model.safetensors b/qwen2.5-1.5b-instruct/checkpoint-213/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2db9ad9ca66aca5f7b313dd910472c08389e0134 --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-213/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f14867d928257934392dd2a17c62080e0613e998d7c4e10eba82a99bef35153 +size 6174895536 diff --git a/qwen2.5-1.5b-instruct/checkpoint-213/optimizer.pt b/qwen2.5-1.5b-instruct/checkpoint-213/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..1cc22ff88dcc5db318930eb1a66a8c49a7d51c21 --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-213/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47247bfc837aa5754be918243d2bc8ffb68d36278fb0e061bfc4345e106d04b1 +size 12350013801 diff --git a/qwen2.5-1.5b-instruct/checkpoint-213/rng_state_0.pth b/qwen2.5-1.5b-instruct/checkpoint-213/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..870021e3fa5ac35c2f711adf0c93a556ab4842da --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-213/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95e5fc2074c0df31522a514f862c86cb00d71c946a7f15cc9ec0e53a69fb28a7 +size 14917 diff --git a/qwen2.5-1.5b-instruct/checkpoint-213/rng_state_1.pth b/qwen2.5-1.5b-instruct/checkpoint-213/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..21f20da5eb1da017f08aaa88bd19cf24d40e3fbf --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-213/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e7153eae67b6c9232a41bc996a2bf5b83229b8c7230d61911ac0fd40e64154e +size 14917 diff --git a/qwen2.5-1.5b-instruct/checkpoint-213/scheduler.pt b/qwen2.5-1.5b-instruct/checkpoint-213/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..39cf87775c3a71d10e7311e0cab769e112f357ba --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-213/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c38a67b98c17be81d4c6d00b60029cdedd43e9aba35ca28ec28ef946c400f268 +size 1465 diff --git a/qwen2.5-1.5b-instruct/checkpoint-213/tokenizer.json b/qwen2.5-1.5b-instruct/checkpoint-213/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..34510ff0037cd50428af467a17ead5a96140a32c --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-213/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/qwen2.5-1.5b-instruct/checkpoint-213/tokenizer_config.json b/qwen2.5-1.5b-instruct/checkpoint-213/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..145e2c7a68b109ae0253dda29fc2e66cb01b4b85 --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-213/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen2.5-1.5b-instruct/checkpoint-213/trainer_state.json b/qwen2.5-1.5b-instruct/checkpoint-213/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..2f0fd6e8d4d4c96822a52443b8e4b008a99dbca6 --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-213/trainer_state.json @@ -0,0 +1,1557 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 50, + "global_step": 213, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.014222222222222223, + "grad_norm": 0.03586147725582123, + "learning_rate": 0.0, + "loss": 0.9098724722862244, + "step": 1 + }, + { + "epoch": 0.028444444444444446, + "grad_norm": 0.03577122464776039, + "learning_rate": 4.5454545454545457e-07, + "loss": 0.9084522724151611, + "step": 2 + }, + { + "epoch": 0.042666666666666665, + "grad_norm": 0.031653374433517456, + "learning_rate": 9.090909090909091e-07, + "loss": 0.8834755420684814, + "step": 3 + }, + { + "epoch": 0.05688888888888889, + "grad_norm": 0.025831686332821846, + "learning_rate": 1.3636363636363636e-06, + "loss": 0.871230959892273, + "step": 4 + }, + { + "epoch": 0.07111111111111111, + "grad_norm": 0.019876576960086823, + "learning_rate": 1.8181818181818183e-06, + "loss": 0.7917206287384033, + "step": 5 + }, + { + "epoch": 0.08533333333333333, + "grad_norm": 0.018677273765206337, + "learning_rate": 2.2727272727272728e-06, + "loss": 0.8436126708984375, + "step": 6 + }, + { + "epoch": 0.09955555555555555, + "grad_norm": 0.017528358846902847, + "learning_rate": 2.7272727272727272e-06, + "loss": 0.8075190782546997, + "step": 7 + }, + { + "epoch": 0.11377777777777778, + "grad_norm": 0.01399513240903616, + "learning_rate": 3.181818181818182e-06, + "loss": 0.7754716873168945, + "step": 8 + }, + { + "epoch": 0.128, + "grad_norm": 0.012644859962165356, + "learning_rate": 3.6363636363636366e-06, + "loss": 0.7459083199501038, + "step": 9 + }, + { + "epoch": 0.14222222222222222, + "grad_norm": 0.01209743320941925, + "learning_rate": 4.0909090909090915e-06, + "loss": 0.7365868091583252, + "step": 10 + }, + { + "epoch": 0.15644444444444444, + "grad_norm": 0.012133832089602947, + "learning_rate": 4.5454545454545455e-06, + "loss": 0.7119501829147339, + "step": 11 + }, + { + "epoch": 0.17066666666666666, + "grad_norm": 0.010017997585237026, + "learning_rate": 5e-06, + "loss": 0.6901087760925293, + "step": 12 + }, + { + "epoch": 0.18488888888888888, + "grad_norm": 0.009233366698026657, + "learning_rate": 5.4545454545454545e-06, + "loss": 0.6611638069152832, + "step": 13 + }, + { + "epoch": 0.1991111111111111, + "grad_norm": 0.00782010331749916, + "learning_rate": 5.90909090909091e-06, + "loss": 0.6715224981307983, + "step": 14 + }, + { + "epoch": 0.21333333333333335, + "grad_norm": 0.007341764867305756, + "learning_rate": 6.363636363636364e-06, + "loss": 0.6277110576629639, + "step": 15 + }, + { + "epoch": 0.22755555555555557, + "grad_norm": 0.00713470159098506, + "learning_rate": 6.818181818181818e-06, + "loss": 0.6689565181732178, + "step": 16 + }, + { + "epoch": 0.24177777777777779, + "grad_norm": 0.0067572277039289474, + "learning_rate": 7.272727272727273e-06, + "loss": 0.6363875865936279, + "step": 17 + }, + { + "epoch": 0.256, + "grad_norm": 0.006228511221706867, + "learning_rate": 7.727272727272727e-06, + "loss": 0.6257224678993225, + "step": 18 + }, + { + "epoch": 0.2702222222222222, + "grad_norm": 0.006048806011676788, + "learning_rate": 8.181818181818183e-06, + "loss": 0.6355900764465332, + "step": 19 + }, + { + "epoch": 0.28444444444444444, + "grad_norm": 0.005988306365907192, + "learning_rate": 8.636363636363637e-06, + "loss": 0.6338369846343994, + "step": 20 + }, + { + "epoch": 0.2986666666666667, + "grad_norm": 0.005441761109977961, + "learning_rate": 9.090909090909091e-06, + "loss": 0.6133573055267334, + "step": 21 + }, + { + "epoch": 0.3128888888888889, + "grad_norm": 0.005174445919692516, + "learning_rate": 9.545454545454547e-06, + "loss": 0.6042410135269165, + "step": 22 + }, + { + "epoch": 0.32711111111111113, + "grad_norm": 0.005199746694415808, + "learning_rate": 1e-05, + "loss": 0.602219820022583, + "step": 23 + }, + { + "epoch": 0.3413333333333333, + "grad_norm": 0.005485028959810734, + "learning_rate": 9.999323662872998e-06, + "loss": 0.6062123775482178, + "step": 24 + }, + { + "epoch": 0.35555555555555557, + "grad_norm": 0.004896003287285566, + "learning_rate": 9.99729483446475e-06, + "loss": 0.6077945232391357, + "step": 25 + }, + { + "epoch": 0.36977777777777776, + "grad_norm": 0.00495151337236166, + "learning_rate": 9.993914063644053e-06, + "loss": 0.5923135280609131, + "step": 26 + }, + { + "epoch": 0.384, + "grad_norm": 0.004744194447994232, + "learning_rate": 9.989182265027232e-06, + "loss": 0.5860621929168701, + "step": 27 + }, + { + "epoch": 0.3982222222222222, + "grad_norm": 0.004471767693758011, + "learning_rate": 9.98310071873072e-06, + "loss": 0.5823097229003906, + "step": 28 + }, + { + "epoch": 0.41244444444444445, + "grad_norm": 0.004788320977240801, + "learning_rate": 9.975671070024741e-06, + "loss": 0.5872600078582764, + "step": 29 + }, + { + "epoch": 0.4266666666666667, + "grad_norm": 0.0044632842764258385, + "learning_rate": 9.966895328888195e-06, + "loss": 0.6070278286933899, + "step": 30 + }, + { + "epoch": 0.4408888888888889, + "grad_norm": 0.004317905753850937, + "learning_rate": 9.956775869464901e-06, + "loss": 0.5780316591262817, + "step": 31 + }, + { + "epoch": 0.45511111111111113, + "grad_norm": 0.004291852004826069, + "learning_rate": 9.945315429421307e-06, + "loss": 0.5697590112686157, + "step": 32 + }, + { + "epoch": 0.4693333333333333, + "grad_norm": 0.003922830801457167, + "learning_rate": 9.932517109205849e-06, + "loss": 0.5747107267379761, + "step": 33 + }, + { + "epoch": 0.48355555555555557, + "grad_norm": 0.0041889771819114685, + "learning_rate": 9.918384371210178e-06, + "loss": 0.6021177172660828, + "step": 34 + }, + { + "epoch": 0.49777777777777776, + "grad_norm": 0.004147357773035765, + "learning_rate": 9.902921038832456e-06, + "loss": 0.5805741548538208, + "step": 35 + }, + { + "epoch": 0.512, + "grad_norm": 0.0040724254213273525, + "learning_rate": 9.886131295443003e-06, + "loss": 0.5483200550079346, + "step": 36 + }, + { + "epoch": 0.5262222222222223, + "grad_norm": 0.004085644613951445, + "learning_rate": 9.868019683252543e-06, + "loss": 0.557392954826355, + "step": 37 + }, + { + "epoch": 0.5404444444444444, + "grad_norm": 0.004289102274924517, + "learning_rate": 9.848591102083375e-06, + "loss": 0.590857207775116, + "step": 38 + }, + { + "epoch": 0.5546666666666666, + "grad_norm": 0.0040345704182982445, + "learning_rate": 9.82785080804381e-06, + "loss": 0.5750541090965271, + "step": 39 + }, + { + "epoch": 0.5688888888888889, + "grad_norm": 0.004034748766571283, + "learning_rate": 9.805804412106197e-06, + "loss": 0.5949826240539551, + "step": 40 + }, + { + "epoch": 0.5831111111111111, + "grad_norm": 0.003911909181624651, + "learning_rate": 9.782457878588977e-06, + "loss": 0.5623177289962769, + "step": 41 + }, + { + "epoch": 0.5973333333333334, + "grad_norm": 0.0037001846358180046, + "learning_rate": 9.75781752354311e-06, + "loss": 0.5465081930160522, + "step": 42 + }, + { + "epoch": 0.6115555555555555, + "grad_norm": 0.0037095139268785715, + "learning_rate": 9.731890013043367e-06, + "loss": 0.5752100944519043, + "step": 43 + }, + { + "epoch": 0.6257777777777778, + "grad_norm": 0.003655617358162999, + "learning_rate": 9.704682361384941e-06, + "loss": 0.5452020168304443, + "step": 44 + }, + { + "epoch": 0.64, + "grad_norm": 0.003784338477998972, + "learning_rate": 9.676201929185809e-06, + "loss": 0.5616332292556763, + "step": 45 + }, + { + "epoch": 0.6542222222222223, + "grad_norm": 0.003718078136444092, + "learning_rate": 9.646456421395447e-06, + "loss": 0.5558297634124756, + "step": 46 + }, + { + "epoch": 0.6684444444444444, + "grad_norm": 0.00395169947296381, + "learning_rate": 9.615453885210368e-06, + "loss": 0.5600595474243164, + "step": 47 + }, + { + "epoch": 0.6826666666666666, + "grad_norm": 0.003555440343916416, + "learning_rate": 9.583202707897075e-06, + "loss": 0.5372895002365112, + "step": 48 + }, + { + "epoch": 0.6968888888888889, + "grad_norm": 0.0037386869080364704, + "learning_rate": 9.549711614523007e-06, + "loss": 0.5322733521461487, + "step": 49 + }, + { + "epoch": 0.7111111111111111, + "grad_norm": 0.003739988896995783, + "learning_rate": 9.514989665596114e-06, + "loss": 0.5380419492721558, + "step": 50 + }, + { + "epoch": 0.7111111111111111, + "eval_loss": 0.5587145090103149, + "eval_runtime": 8.7676, + "eval_samples_per_second": 22.811, + "eval_steps_per_second": 11.406, + "step": 50 + }, + { + "epoch": 0.7253333333333334, + "grad_norm": 0.0037539806216955185, + "learning_rate": 9.479046254613673e-06, + "loss": 0.5472225546836853, + "step": 51 + }, + { + "epoch": 0.7395555555555555, + "grad_norm": 0.003411181503906846, + "learning_rate": 9.441891105521005e-06, + "loss": 0.5134032964706421, + "step": 52 + }, + { + "epoch": 0.7537777777777778, + "grad_norm": 0.0037538670003414154, + "learning_rate": 9.40353427008083e-06, + "loss": 0.5361013412475586, + "step": 53 + }, + { + "epoch": 0.768, + "grad_norm": 0.0036495670210570097, + "learning_rate": 9.3639861251539e-06, + "loss": 0.5299779772758484, + "step": 54 + }, + { + "epoch": 0.7822222222222223, + "grad_norm": 0.0037342547439038754, + "learning_rate": 9.323257369891702e-06, + "loss": 0.5607945919036865, + "step": 55 + }, + { + "epoch": 0.7964444444444444, + "grad_norm": 0.003572584595531225, + "learning_rate": 9.281359022841966e-06, + "loss": 0.5528460741043091, + "step": 56 + }, + { + "epoch": 0.8106666666666666, + "grad_norm": 0.003770101349800825, + "learning_rate": 9.238302418967757e-06, + "loss": 0.5440288186073303, + "step": 57 + }, + { + "epoch": 0.8248888888888889, + "grad_norm": 0.0036679303739219904, + "learning_rate": 9.194099206580981e-06, + "loss": 0.5564252138137817, + "step": 58 + }, + { + "epoch": 0.8391111111111111, + "grad_norm": 0.003834612900391221, + "learning_rate": 9.14876134419111e-06, + "loss": 0.5473330020904541, + "step": 59 + }, + { + "epoch": 0.8533333333333334, + "grad_norm": 0.003683249931782484, + "learning_rate": 9.102301097269974e-06, + "loss": 0.5463743209838867, + "step": 60 + }, + { + "epoch": 0.8675555555555555, + "grad_norm": 0.0036777963396161795, + "learning_rate": 9.05473103493355e-06, + "loss": 0.524878740310669, + "step": 61 + }, + { + "epoch": 0.8817777777777778, + "grad_norm": 0.0038230482023209333, + "learning_rate": 9.006064026541549e-06, + "loss": 0.5563522577285767, + "step": 62 + }, + { + "epoch": 0.896, + "grad_norm": 0.003645201213657856, + "learning_rate": 8.956313238215824e-06, + "loss": 0.5059154033660889, + "step": 63 + }, + { + "epoch": 0.9102222222222223, + "grad_norm": 0.0035886459518224, + "learning_rate": 8.905492129278478e-06, + "loss": 0.5312881469726562, + "step": 64 + }, + { + "epoch": 0.9244444444444444, + "grad_norm": 0.003721872577443719, + "learning_rate": 8.85361444861063e-06, + "loss": 0.5358959436416626, + "step": 65 + }, + { + "epoch": 0.9386666666666666, + "grad_norm": 0.003683999879285693, + "learning_rate": 8.800694230932885e-06, + "loss": 0.5442053079605103, + "step": 66 + }, + { + "epoch": 0.9528888888888889, + "grad_norm": 0.0036851842887699604, + "learning_rate": 8.74674579300843e-06, + "loss": 0.5263723731040955, + "step": 67 + }, + { + "epoch": 0.9671111111111111, + "grad_norm": 0.003754258854314685, + "learning_rate": 8.691783729769874e-06, + "loss": 0.5200538039207458, + "step": 68 + }, + { + "epoch": 0.9813333333333333, + "grad_norm": 0.0038373274728655815, + "learning_rate": 8.635822910370793e-06, + "loss": 0.5452108383178711, + "step": 69 + }, + { + "epoch": 0.9955555555555555, + "grad_norm": 0.0039480882696807384, + "learning_rate": 8.578878474163115e-06, + "loss": 0.5066030621528625, + "step": 70 + }, + { + "epoch": 1.0, + "grad_norm": 0.005897017661482096, + "learning_rate": 8.520965826601394e-06, + "loss": 0.5380499958992004, + "step": 71 + }, + { + "epoch": 1.0142222222222221, + "grad_norm": 0.0037728557363152504, + "learning_rate": 8.462100635075097e-06, + "loss": 0.5328541398048401, + "step": 72 + }, + { + "epoch": 1.0284444444444445, + "grad_norm": 0.003953218925744295, + "learning_rate": 8.40229882467003e-06, + "loss": 0.5096005201339722, + "step": 73 + }, + { + "epoch": 1.0426666666666666, + "grad_norm": 0.0036587677896022797, + "learning_rate": 8.341576573860049e-06, + "loss": 0.4889724850654602, + "step": 74 + }, + { + "epoch": 1.056888888888889, + "grad_norm": 0.0038338187150657177, + "learning_rate": 8.279950310130218e-06, + "loss": 0.5157173871994019, + "step": 75 + }, + { + "epoch": 1.0711111111111111, + "grad_norm": 0.003540106350556016, + "learning_rate": 8.2174367055326e-06, + "loss": 0.49691659212112427, + "step": 76 + }, + { + "epoch": 1.0853333333333333, + "grad_norm": 0.0037315969821065664, + "learning_rate": 8.154052672175888e-06, + "loss": 0.4873769283294678, + "step": 77 + }, + { + "epoch": 1.0995555555555556, + "grad_norm": 0.003963674418628216, + "learning_rate": 8.08981535765009e-06, + "loss": 0.5266215801239014, + "step": 78 + }, + { + "epoch": 1.1137777777777778, + "grad_norm": 0.0037685995921492577, + "learning_rate": 8.024742140387506e-06, + "loss": 0.5039150714874268, + "step": 79 + }, + { + "epoch": 1.1280000000000001, + "grad_norm": 0.003613364417105913, + "learning_rate": 7.95885062496126e-06, + "loss": 0.4896891117095947, + "step": 80 + }, + { + "epoch": 1.1422222222222222, + "grad_norm": 0.0037347134202718735, + "learning_rate": 7.892158637322647e-06, + "loss": 0.45997822284698486, + "step": 81 + }, + { + "epoch": 1.1564444444444444, + "grad_norm": 0.0037277438677847385, + "learning_rate": 7.824684219978591e-06, + "loss": 0.4917220175266266, + "step": 82 + }, + { + "epoch": 1.1706666666666667, + "grad_norm": 0.0034888957161456347, + "learning_rate": 7.756445627110523e-06, + "loss": 0.5024806261062622, + "step": 83 + }, + { + "epoch": 1.1848888888888889, + "grad_norm": 0.004098390229046345, + "learning_rate": 7.68746131963598e-06, + "loss": 0.5110447406768799, + "step": 84 + }, + { + "epoch": 1.199111111111111, + "grad_norm": 0.0036922753788530827, + "learning_rate": 7.6177499602143e-06, + "loss": 0.5031975507736206, + "step": 85 + }, + { + "epoch": 1.2133333333333334, + "grad_norm": 0.0035749904345721006, + "learning_rate": 7.547330408197695e-06, + "loss": 0.4899296164512634, + "step": 86 + }, + { + "epoch": 1.2275555555555555, + "grad_norm": 0.003764625173062086, + "learning_rate": 7.476221714529167e-06, + "loss": 0.5040560960769653, + "step": 87 + }, + { + "epoch": 1.2417777777777779, + "grad_norm": 0.003831087611615658, + "learning_rate": 7.404443116588548e-06, + "loss": 0.5307536125183105, + "step": 88 + }, + { + "epoch": 1.256, + "grad_norm": 0.003740105777978897, + "learning_rate": 7.332014032988123e-06, + "loss": 0.5025057792663574, + "step": 89 + }, + { + "epoch": 1.2702222222222221, + "grad_norm": 0.0038286929484456778, + "learning_rate": 7.2589540583192165e-06, + "loss": 0.5043497085571289, + "step": 90 + }, + { + "epoch": 1.2844444444444445, + "grad_norm": 0.003593199420720339, + "learning_rate": 7.185282957851175e-06, + "loss": 0.4883122444152832, + "step": 91 + }, + { + "epoch": 1.2986666666666666, + "grad_norm": 0.003721701679751277, + "learning_rate": 7.111020662184174e-06, + "loss": 0.5055882930755615, + "step": 92 + }, + { + "epoch": 1.3128888888888888, + "grad_norm": 0.0036744684912264347, + "learning_rate": 7.036187261857289e-06, + "loss": 0.4962804615497589, + "step": 93 + }, + { + "epoch": 1.3271111111111111, + "grad_norm": 0.004013169556856155, + "learning_rate": 6.960803001913315e-06, + "loss": 0.524259090423584, + "step": 94 + }, + { + "epoch": 1.3413333333333333, + "grad_norm": 0.003746299771592021, + "learning_rate": 6.884888276421766e-06, + "loss": 0.5073269009590149, + "step": 95 + }, + { + "epoch": 1.3555555555555556, + "grad_norm": 0.0036679194308817387, + "learning_rate": 6.8084636229615786e-06, + "loss": 0.4886914789676666, + "step": 96 + }, + { + "epoch": 1.3697777777777778, + "grad_norm": 0.00375540042296052, + "learning_rate": 6.731549717064975e-06, + "loss": 0.4920746088027954, + "step": 97 + }, + { + "epoch": 1.384, + "grad_norm": 0.003901587100699544, + "learning_rate": 6.654167366624009e-06, + "loss": 0.503207266330719, + "step": 98 + }, + { + "epoch": 1.3982222222222223, + "grad_norm": 0.003911704756319523, + "learning_rate": 6.576337506261314e-06, + "loss": 0.5012996196746826, + "step": 99 + }, + { + "epoch": 1.4124444444444444, + "grad_norm": 0.0035876361653208733, + "learning_rate": 6.498081191666549e-06, + "loss": 0.5059630274772644, + "step": 100 + }, + { + "epoch": 1.4124444444444444, + "eval_loss": 0.5291631817817688, + "eval_runtime": 10.3556, + "eval_samples_per_second": 19.313, + "eval_steps_per_second": 9.657, + "step": 100 + }, + { + "epoch": 1.4266666666666667, + "grad_norm": 0.003729900810867548, + "learning_rate": 6.419419593900109e-06, + "loss": 0.4896056056022644, + "step": 101 + }, + { + "epoch": 1.4408888888888889, + "grad_norm": 0.003648703685030341, + "learning_rate": 6.340373993665607e-06, + "loss": 0.48181772232055664, + "step": 102 + }, + { + "epoch": 1.455111111111111, + "grad_norm": 0.0035914399195462465, + "learning_rate": 6.2609657755527135e-06, + "loss": 0.4775124192237854, + "step": 103 + }, + { + "epoch": 1.4693333333333334, + "grad_norm": 0.0035880866926163435, + "learning_rate": 6.1812164222518626e-06, + "loss": 0.5167958736419678, + "step": 104 + }, + { + "epoch": 1.4835555555555555, + "grad_norm": 0.003788268193602562, + "learning_rate": 6.101147508742456e-06, + "loss": 0.4946171045303345, + "step": 105 + }, + { + "epoch": 1.4977777777777779, + "grad_norm": 0.0036002870183438063, + "learning_rate": 6.020780696456059e-06, + "loss": 0.4849076569080353, + "step": 106 + }, + { + "epoch": 1.512, + "grad_norm": 0.0036388293374329805, + "learning_rate": 5.940137727416247e-06, + "loss": 0.4958712160587311, + "step": 107 + }, + { + "epoch": 1.5262222222222221, + "grad_norm": 0.0034927036613225937, + "learning_rate": 5.859240418356614e-06, + "loss": 0.4822971820831299, + "step": 108 + }, + { + "epoch": 1.5404444444444443, + "grad_norm": 0.0038439796771854162, + "learning_rate": 5.778110654818602e-06, + "loss": 0.5280419588088989, + "step": 109 + }, + { + "epoch": 1.5546666666666666, + "grad_norm": 0.003715698141604662, + "learning_rate": 5.696770385230679e-06, + "loss": 0.4952104687690735, + "step": 110 + }, + { + "epoch": 1.568888888888889, + "grad_norm": 0.0036385883577167988, + "learning_rate": 5.615241614970546e-06, + "loss": 0.4887908101081848, + "step": 111 + }, + { + "epoch": 1.5831111111111111, + "grad_norm": 0.0036841081455349922, + "learning_rate": 5.533546400411899e-06, + "loss": 0.4928241968154907, + "step": 112 + }, + { + "epoch": 1.5973333333333333, + "grad_norm": 0.0036721359938383102, + "learning_rate": 5.4517068429574215e-06, + "loss": 0.5109080076217651, + "step": 113 + }, + { + "epoch": 1.6115555555555554, + "grad_norm": 0.0037413975223898888, + "learning_rate": 5.3697450830595775e-06, + "loss": 0.48911499977111816, + "step": 114 + }, + { + "epoch": 1.6257777777777778, + "grad_norm": 0.003966668155044317, + "learning_rate": 5.287683294230855e-06, + "loss": 0.49537116289138794, + "step": 115 + }, + { + "epoch": 1.6400000000000001, + "grad_norm": 0.0036018763203173876, + "learning_rate": 5.20554367704505e-06, + "loss": 0.4613257348537445, + "step": 116 + }, + { + "epoch": 1.6542222222222223, + "grad_norm": 0.0038206754252314568, + "learning_rate": 5.123348453131242e-06, + "loss": 0.5043697357177734, + "step": 117 + }, + { + "epoch": 1.6684444444444444, + "grad_norm": 0.0037831810768693686, + "learning_rate": 5.041119859162068e-06, + "loss": 0.48968735337257385, + "step": 118 + }, + { + "epoch": 1.6826666666666665, + "grad_norm": 0.0038370522670447826, + "learning_rate": 4.958880140837934e-06, + "loss": 0.49605250358581543, + "step": 119 + }, + { + "epoch": 1.696888888888889, + "grad_norm": 0.003586502280086279, + "learning_rate": 4.876651546868759e-06, + "loss": 0.4839916229248047, + "step": 120 + }, + { + "epoch": 1.7111111111111112, + "grad_norm": 0.003694413462653756, + "learning_rate": 4.794456322954953e-06, + "loss": 0.5173561573028564, + "step": 121 + }, + { + "epoch": 1.7253333333333334, + "grad_norm": 0.003933500964194536, + "learning_rate": 4.7123167057691446e-06, + "loss": 0.5177811980247498, + "step": 122 + }, + { + "epoch": 1.7395555555555555, + "grad_norm": 0.0036068507470190525, + "learning_rate": 4.630254916940424e-06, + "loss": 0.5164836049079895, + "step": 123 + }, + { + "epoch": 1.7537777777777777, + "grad_norm": 0.00349752651527524, + "learning_rate": 4.548293157042581e-06, + "loss": 0.5077825784683228, + "step": 124 + }, + { + "epoch": 1.768, + "grad_norm": 0.0037042393814772367, + "learning_rate": 4.466453599588103e-06, + "loss": 0.5107009410858154, + "step": 125 + }, + { + "epoch": 1.7822222222222224, + "grad_norm": 0.003853494767099619, + "learning_rate": 4.384758385029457e-06, + "loss": 0.4728143811225891, + "step": 126 + }, + { + "epoch": 1.7964444444444445, + "grad_norm": 0.0038627111352980137, + "learning_rate": 4.3032296147693225e-06, + "loss": 0.4858134984970093, + "step": 127 + }, + { + "epoch": 1.8106666666666666, + "grad_norm": 0.003657885594293475, + "learning_rate": 4.2218893451814005e-06, + "loss": 0.48357316851615906, + "step": 128 + }, + { + "epoch": 1.8248888888888888, + "grad_norm": 0.00355988135561347, + "learning_rate": 4.140759581643386e-06, + "loss": 0.49349913001060486, + "step": 129 + }, + { + "epoch": 1.8391111111111111, + "grad_norm": 0.004014762118458748, + "learning_rate": 4.059862272583755e-06, + "loss": 0.4969041347503662, + "step": 130 + }, + { + "epoch": 1.8533333333333335, + "grad_norm": 0.003631556173786521, + "learning_rate": 3.979219303543942e-06, + "loss": 0.47194504737854004, + "step": 131 + }, + { + "epoch": 1.8675555555555556, + "grad_norm": 0.003577238181605935, + "learning_rate": 3.898852491257547e-06, + "loss": 0.5020972490310669, + "step": 132 + }, + { + "epoch": 1.8817777777777778, + "grad_norm": 0.003780671861022711, + "learning_rate": 3.818783577748138e-06, + "loss": 0.5002884268760681, + "step": 133 + }, + { + "epoch": 1.896, + "grad_norm": 0.003708280622959137, + "learning_rate": 3.7390342244472886e-06, + "loss": 0.5007095336914062, + "step": 134 + }, + { + "epoch": 1.9102222222222223, + "grad_norm": 0.003498840844258666, + "learning_rate": 3.659626006334395e-06, + "loss": 0.4763813316822052, + "step": 135 + }, + { + "epoch": 1.9244444444444444, + "grad_norm": 0.003659547306597233, + "learning_rate": 3.5805804060998926e-06, + "loss": 0.48733818531036377, + "step": 136 + }, + { + "epoch": 1.9386666666666668, + "grad_norm": 0.003515617921948433, + "learning_rate": 3.501918808333453e-06, + "loss": 0.5118749141693115, + "step": 137 + }, + { + "epoch": 1.952888888888889, + "grad_norm": 0.0036544103641062975, + "learning_rate": 3.4236624937386874e-06, + "loss": 0.5178316831588745, + "step": 138 + }, + { + "epoch": 1.967111111111111, + "grad_norm": 0.003433604259043932, + "learning_rate": 3.3458326333759927e-06, + "loss": 0.4853537678718567, + "step": 139 + }, + { + "epoch": 1.9813333333333332, + "grad_norm": 0.003547502215951681, + "learning_rate": 3.268450282935026e-06, + "loss": 0.49210506677627563, + "step": 140 + }, + { + "epoch": 1.9955555555555555, + "grad_norm": 0.003462104359641671, + "learning_rate": 3.1915363770384223e-06, + "loss": 0.47841814160346985, + "step": 141 + }, + { + "epoch": 2.0, + "grad_norm": 0.005856912117451429, + "learning_rate": 3.1151117235782346e-06, + "loss": 0.5034412145614624, + "step": 142 + }, + { + "epoch": 2.014222222222222, + "grad_norm": 0.003923188429325819, + "learning_rate": 3.0391969980866874e-06, + "loss": 0.4864930808544159, + "step": 143 + }, + { + "epoch": 2.0284444444444443, + "grad_norm": 0.003596341237425804, + "learning_rate": 2.963812738142713e-06, + "loss": 0.4642598032951355, + "step": 144 + }, + { + "epoch": 2.042666666666667, + "grad_norm": 0.0038887078408151865, + "learning_rate": 2.8889793378158284e-06, + "loss": 0.48541027307510376, + "step": 145 + }, + { + "epoch": 2.056888888888889, + "grad_norm": 0.0034043709747493267, + "learning_rate": 2.814717042148827e-06, + "loss": 0.47221142053604126, + "step": 146 + }, + { + "epoch": 2.071111111111111, + "grad_norm": 0.0034274954814463854, + "learning_rate": 2.7410459416807856e-06, + "loss": 0.46124035120010376, + "step": 147 + }, + { + "epoch": 2.0853333333333333, + "grad_norm": 0.00352494022808969, + "learning_rate": 2.6679859670118785e-06, + "loss": 0.4869857430458069, + "step": 148 + }, + { + "epoch": 2.0995555555555554, + "grad_norm": 0.0035326213110238314, + "learning_rate": 2.5955568834114523e-06, + "loss": 0.45158863067626953, + "step": 149 + }, + { + "epoch": 2.113777777777778, + "grad_norm": 0.003645913675427437, + "learning_rate": 2.523778285470835e-06, + "loss": 0.4742748439311981, + "step": 150 + }, + { + "epoch": 2.113777777777778, + "eval_loss": 0.5196705460548401, + "eval_runtime": 15.8074, + "eval_samples_per_second": 12.652, + "eval_steps_per_second": 6.326, + "step": 150 + }, + { + "epoch": 2.128, + "grad_norm": 0.0036555698607116938, + "learning_rate": 2.452669591802307e-06, + "loss": 0.4526304602622986, + "step": 151 + }, + { + "epoch": 2.1422222222222222, + "grad_norm": 0.003359478199854493, + "learning_rate": 2.3822500397857016e-06, + "loss": 0.4651503562927246, + "step": 152 + }, + { + "epoch": 2.1564444444444444, + "grad_norm": 0.003462468506768346, + "learning_rate": 2.3125386803640188e-06, + "loss": 0.48414769768714905, + "step": 153 + }, + { + "epoch": 2.1706666666666665, + "grad_norm": 0.003318715374916792, + "learning_rate": 2.243554372889479e-06, + "loss": 0.462248295545578, + "step": 154 + }, + { + "epoch": 2.1848888888888887, + "grad_norm": 0.0034426096826791763, + "learning_rate": 2.1753157800214107e-06, + "loss": 0.4559134244918823, + "step": 155 + }, + { + "epoch": 2.1991111111111112, + "grad_norm": 0.0035529518499970436, + "learning_rate": 2.1078413626773547e-06, + "loss": 0.48357266187667847, + "step": 156 + }, + { + "epoch": 2.2133333333333334, + "grad_norm": 0.0034895483404397964, + "learning_rate": 2.0411493750387423e-06, + "loss": 0.4366765022277832, + "step": 157 + }, + { + "epoch": 2.2275555555555555, + "grad_norm": 0.003682621056213975, + "learning_rate": 1.9752578596124955e-06, + "loss": 0.4868413209915161, + "step": 158 + }, + { + "epoch": 2.2417777777777776, + "grad_norm": 0.0035529278684407473, + "learning_rate": 1.9101846423499113e-06, + "loss": 0.46049410104751587, + "step": 159 + }, + { + "epoch": 2.2560000000000002, + "grad_norm": 0.003581352299079299, + "learning_rate": 1.8459473278241125e-06, + "loss": 0.4605860412120819, + "step": 160 + }, + { + "epoch": 2.2702222222222224, + "grad_norm": 0.0033597166184335947, + "learning_rate": 1.7825632944674016e-06, + "loss": 0.46191370487213135, + "step": 161 + }, + { + "epoch": 2.2844444444444445, + "grad_norm": 0.003529531881213188, + "learning_rate": 1.7200496898697832e-06, + "loss": 0.4780347943305969, + "step": 162 + }, + { + "epoch": 2.2986666666666666, + "grad_norm": 0.0034702750854194164, + "learning_rate": 1.6584234261399535e-06, + "loss": 0.4648103713989258, + "step": 163 + }, + { + "epoch": 2.3128888888888888, + "grad_norm": 0.0036891205236315727, + "learning_rate": 1.5977011753299726e-06, + "loss": 0.46155521273612976, + "step": 164 + }, + { + "epoch": 2.327111111111111, + "grad_norm": 0.003741780063137412, + "learning_rate": 1.5378993649249053e-06, + "loss": 0.49238717555999756, + "step": 165 + }, + { + "epoch": 2.3413333333333335, + "grad_norm": 0.003572724526748061, + "learning_rate": 1.4790341733986085e-06, + "loss": 0.46214908361434937, + "step": 166 + }, + { + "epoch": 2.3555555555555556, + "grad_norm": 0.003393909428268671, + "learning_rate": 1.4211215258368866e-06, + "loss": 0.44751304388046265, + "step": 167 + }, + { + "epoch": 2.3697777777777778, + "grad_norm": 0.0035024855751544237, + "learning_rate": 1.3641770896292083e-06, + "loss": 0.4523189067840576, + "step": 168 + }, + { + "epoch": 2.384, + "grad_norm": 0.0034958638716489077, + "learning_rate": 1.3082162702301276e-06, + "loss": 0.46704721450805664, + "step": 169 + }, + { + "epoch": 2.398222222222222, + "grad_norm": 0.0035048348363488913, + "learning_rate": 1.2532542069915722e-06, + "loss": 0.47263163328170776, + "step": 170 + }, + { + "epoch": 2.4124444444444446, + "grad_norm": 0.0034445980563759804, + "learning_rate": 1.1993057690671174e-06, + "loss": 0.4592151641845703, + "step": 171 + }, + { + "epoch": 2.4266666666666667, + "grad_norm": 0.0034275518264621496, + "learning_rate": 1.1463855513893695e-06, + "loss": 0.4800827205181122, + "step": 172 + }, + { + "epoch": 2.440888888888889, + "grad_norm": 0.0034792288206517696, + "learning_rate": 1.0945078707215224e-06, + "loss": 0.4688478112220764, + "step": 173 + }, + { + "epoch": 2.455111111111111, + "grad_norm": 0.0035636303946375847, + "learning_rate": 1.0436867617841768e-06, + "loss": 0.46817073225975037, + "step": 174 + }, + { + "epoch": 2.469333333333333, + "grad_norm": 0.0033977399580180645, + "learning_rate": 9.939359734584552e-07, + "loss": 0.4763898253440857, + "step": 175 + }, + { + "epoch": 2.4835555555555557, + "grad_norm": 0.003528163768351078, + "learning_rate": 9.452689650664515e-07, + "loss": 0.480793833732605, + "step": 176 + }, + { + "epoch": 2.497777777777778, + "grad_norm": 0.00341871683485806, + "learning_rate": 8.976989027300265e-07, + "loss": 0.4694904088973999, + "step": 177 + }, + { + "epoch": 2.512, + "grad_norm": 0.003544109407812357, + "learning_rate": 8.512386558088919e-07, + "loss": 0.4877663254737854, + "step": 178 + }, + { + "epoch": 2.526222222222222, + "grad_norm": 0.0034784283488988876, + "learning_rate": 8.059007934190194e-07, + "loss": 0.46579453349113464, + "step": 179 + }, + { + "epoch": 2.5404444444444443, + "grad_norm": 0.003506515873596072, + "learning_rate": 7.61697581032243e-07, + "loss": 0.4954630434513092, + "step": 180 + }, + { + "epoch": 2.554666666666667, + "grad_norm": 0.003402002854272723, + "learning_rate": 7.186409771580355e-07, + "loss": 0.48210158944129944, + "step": 181 + }, + { + "epoch": 2.568888888888889, + "grad_norm": 0.003400753950700164, + "learning_rate": 6.76742630108298e-07, + "loss": 0.4673936069011688, + "step": 182 + }, + { + "epoch": 2.583111111111111, + "grad_norm": 0.0034171557053923607, + "learning_rate": 6.360138748461015e-07, + "loss": 0.46786054968833923, + "step": 183 + }, + { + "epoch": 2.5973333333333333, + "grad_norm": 0.003441787790507078, + "learning_rate": 5.964657299191712e-07, + "loss": 0.45768898725509644, + "step": 184 + }, + { + "epoch": 2.6115555555555554, + "grad_norm": 0.0034633874893188477, + "learning_rate": 5.581088944789953e-07, + "loss": 0.468585342168808, + "step": 185 + }, + { + "epoch": 2.6257777777777775, + "grad_norm": 0.0034696508664637804, + "learning_rate": 5.20953745386329e-07, + "loss": 0.4843568205833435, + "step": 186 + }, + { + "epoch": 2.64, + "grad_norm": 0.003448233939707279, + "learning_rate": 4.850103344038853e-07, + "loss": 0.4747229218482971, + "step": 187 + }, + { + "epoch": 2.6542222222222223, + "grad_norm": 0.00340772676281631, + "learning_rate": 4.502883854769935e-07, + "loss": 0.44664108753204346, + "step": 188 + }, + { + "epoch": 2.6684444444444444, + "grad_norm": 0.0034893041010946035, + "learning_rate": 4.167972921029262e-07, + "loss": 0.47903192043304443, + "step": 189 + }, + { + "epoch": 2.6826666666666665, + "grad_norm": 0.003642126452177763, + "learning_rate": 3.8454611478963235e-07, + "loss": 0.4844057559967041, + "step": 190 + }, + { + "epoch": 2.696888888888889, + "grad_norm": 0.0034686627332121134, + "learning_rate": 3.535435786045538e-07, + "loss": 0.4564388394355774, + "step": 191 + }, + { + "epoch": 2.7111111111111112, + "grad_norm": 0.003412955440580845, + "learning_rate": 3.237980708141919e-07, + "loss": 0.47008293867111206, + "step": 192 + }, + { + "epoch": 2.7253333333333334, + "grad_norm": 0.0035507141146808863, + "learning_rate": 2.9531763861505967e-07, + "loss": 0.479957640171051, + "step": 193 + }, + { + "epoch": 2.7395555555555555, + "grad_norm": 0.0035486763808876276, + "learning_rate": 2.681099869566328e-07, + "loss": 0.44965675473213196, + "step": 194 + }, + { + "epoch": 2.7537777777777777, + "grad_norm": 0.0034812854137271643, + "learning_rate": 2.4218247645689306e-07, + "loss": 0.4747314453125, + "step": 195 + }, + { + "epoch": 2.768, + "grad_norm": 0.00335957994684577, + "learning_rate": 2.175421214110235e-07, + "loss": 0.45799821615219116, + "step": 196 + }, + { + "epoch": 2.7822222222222224, + "grad_norm": 0.0035609498154371977, + "learning_rate": 1.941955878938029e-07, + "loss": 0.47467684745788574, + "step": 197 + }, + { + "epoch": 2.7964444444444445, + "grad_norm": 0.003390205791220069, + "learning_rate": 1.721491919561913e-07, + "loss": 0.4817090332508087, + "step": 198 + }, + { + "epoch": 2.8106666666666666, + "grad_norm": 0.003445240668952465, + "learning_rate": 1.514088979166256e-07, + "loss": 0.4649164080619812, + "step": 199 + }, + { + "epoch": 2.824888888888889, + "grad_norm": 0.003372759558260441, + "learning_rate": 1.3198031674745814e-07, + "loss": 0.4765157699584961, + "step": 200 + }, + { + "epoch": 2.824888888888889, + "eval_loss": 0.5171427726745605, + "eval_runtime": 9.8598, + "eval_samples_per_second": 20.284, + "eval_steps_per_second": 10.142, + "step": 200 + }, + { + "epoch": 2.8391111111111114, + "grad_norm": 0.0034065141808241606, + "learning_rate": 1.138687045569975e-07, + "loss": 0.46831822395324707, + "step": 201 + }, + { + "epoch": 2.8533333333333335, + "grad_norm": 0.0034053055569529533, + "learning_rate": 9.70789611675449e-08, + "loss": 0.47398146986961365, + "step": 202 + }, + { + "epoch": 2.8675555555555556, + "grad_norm": 0.0034186639823019505, + "learning_rate": 8.161562878982399e-08, + "loss": 0.45744937658309937, + "step": 203 + }, + { + "epoch": 2.8817777777777778, + "grad_norm": 0.0034667483996599913, + "learning_rate": 6.74828907941516e-08, + "loss": 0.48529934883117676, + "step": 204 + }, + { + "epoch": 2.896, + "grad_norm": 0.003455272177234292, + "learning_rate": 5.468457057869358e-08, + "loss": 0.4858051836490631, + "step": 205 + }, + { + "epoch": 2.910222222222222, + "grad_norm": 0.0033815570641309023, + "learning_rate": 4.322413053509944e-08, + "loss": 0.4590516984462738, + "step": 206 + }, + { + "epoch": 2.924444444444444, + "grad_norm": 0.0034011146053671837, + "learning_rate": 3.31046711118066e-08, + "loss": 0.45788976550102234, + "step": 207 + }, + { + "epoch": 2.9386666666666668, + "grad_norm": 0.003420215332880616, + "learning_rate": 2.4328929975260262e-08, + "loss": 0.4702548086643219, + "step": 208 + }, + { + "epoch": 2.952888888888889, + "grad_norm": 0.0033962675370275974, + "learning_rate": 1.6899281269279756e-08, + "loss": 0.4560105800628662, + "step": 209 + }, + { + "epoch": 2.967111111111111, + "grad_norm": 0.003477597376331687, + "learning_rate": 1.0817734972768946e-08, + "loss": 0.46400371193885803, + "step": 210 + }, + { + "epoch": 2.981333333333333, + "grad_norm": 0.0036108316853642464, + "learning_rate": 6.085936355947897e-09, + "loss": 0.48368915915489197, + "step": 211 + }, + { + "epoch": 2.9955555555555557, + "grad_norm": 0.003335504326969385, + "learning_rate": 2.7051655352494654e-09, + "loss": 0.45437300205230713, + "step": 212 + }, + { + "epoch": 3.0, + "grad_norm": 0.00642085587605834, + "learning_rate": 6.763371270035457e-10, + "loss": 0.5508654713630676, + "step": 213 + } + ], + "logging_steps": 1, + "max_steps": 213, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.4727435500270387e+17, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/qwen2.5-1.5b-instruct/checkpoint-213/training_args.bin b/qwen2.5-1.5b-instruct/checkpoint-213/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..4badd3789f92f64e02e2816810bfe346973afac5 --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-213/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39e78c30a5784e935e65cb8fa1a73b2afb467afccc55e7223863b667252fac3b +size 5649 diff --git a/qwen2.5-1.5b-instruct/checkpoint-71/chat_template.jinja b/qwen2.5-1.5b-instruct/checkpoint-71/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-71/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/qwen2.5-1.5b-instruct/checkpoint-71/config.json b/qwen2.5-1.5b-instruct/checkpoint-71/config.json new file mode 100644 index 0000000000000000000000000000000000000000..499065fd744a64fc8ad2884da26e85a6e99d468a --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-71/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 21, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/qwen2.5-1.5b-instruct/checkpoint-71/generation_config.json b/qwen2.5-1.5b-instruct/checkpoint-71/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c5b4e11411fed3e7a5e6557165f888a7b0b1248b --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-71/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.2.0" +} diff --git a/qwen2.5-1.5b-instruct/checkpoint-71/model.safetensors b/qwen2.5-1.5b-instruct/checkpoint-71/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b9fd8c015c05807c95b58fd30bf8659b7f56c44f --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-71/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1f0b1bbce5ea376a4ef5e6eb42573baa753fc0c17d93c09cfe0dcbcb8667dee +size 6174895536 diff --git a/qwen2.5-1.5b-instruct/checkpoint-71/optimizer.pt b/qwen2.5-1.5b-instruct/checkpoint-71/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f6437f64b3af512bf8834c34597b1f3728800ea --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-71/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23bb7c09eb4777a07e001a06cb196cc13c55aa7113596085a8060ccc4768e6d9 +size 12350013801 diff --git a/qwen2.5-1.5b-instruct/checkpoint-71/rng_state_0.pth b/qwen2.5-1.5b-instruct/checkpoint-71/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..870021e3fa5ac35c2f711adf0c93a556ab4842da --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-71/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95e5fc2074c0df31522a514f862c86cb00d71c946a7f15cc9ec0e53a69fb28a7 +size 14917 diff --git a/qwen2.5-1.5b-instruct/checkpoint-71/rng_state_1.pth b/qwen2.5-1.5b-instruct/checkpoint-71/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..21f20da5eb1da017f08aaa88bd19cf24d40e3fbf --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-71/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e7153eae67b6c9232a41bc996a2bf5b83229b8c7230d61911ac0fd40e64154e +size 14917 diff --git a/qwen2.5-1.5b-instruct/checkpoint-71/scheduler.pt b/qwen2.5-1.5b-instruct/checkpoint-71/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3be1d29bde40f225713ff5084d3d9d69858fcfd --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-71/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c96c28f12671e2f5db493417daefb3ff0b1bc36602843e0d7ef6df3649c27b3 +size 1465 diff --git a/qwen2.5-1.5b-instruct/checkpoint-71/tokenizer.json b/qwen2.5-1.5b-instruct/checkpoint-71/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..34510ff0037cd50428af467a17ead5a96140a32c --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-71/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/qwen2.5-1.5b-instruct/checkpoint-71/tokenizer_config.json b/qwen2.5-1.5b-instruct/checkpoint-71/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..145e2c7a68b109ae0253dda29fc2e66cb01b4b85 --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-71/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen2.5-1.5b-instruct/checkpoint-71/trainer_state.json b/qwen2.5-1.5b-instruct/checkpoint-71/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..7c70eb116e93c5bbb08d90845d3679a19bcdb2cd --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-71/trainer_state.json @@ -0,0 +1,539 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 50, + "global_step": 71, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.014222222222222223, + "grad_norm": 0.03586147725582123, + "learning_rate": 0.0, + "loss": 0.9098724722862244, + "step": 1 + }, + { + "epoch": 0.028444444444444446, + "grad_norm": 0.03577122464776039, + "learning_rate": 4.5454545454545457e-07, + "loss": 0.9084522724151611, + "step": 2 + }, + { + "epoch": 0.042666666666666665, + "grad_norm": 0.031653374433517456, + "learning_rate": 9.090909090909091e-07, + "loss": 0.8834755420684814, + "step": 3 + }, + { + "epoch": 0.05688888888888889, + "grad_norm": 0.025831686332821846, + "learning_rate": 1.3636363636363636e-06, + "loss": 0.871230959892273, + "step": 4 + }, + { + "epoch": 0.07111111111111111, + "grad_norm": 0.019876576960086823, + "learning_rate": 1.8181818181818183e-06, + "loss": 0.7917206287384033, + "step": 5 + }, + { + "epoch": 0.08533333333333333, + "grad_norm": 0.018677273765206337, + "learning_rate": 2.2727272727272728e-06, + "loss": 0.8436126708984375, + "step": 6 + }, + { + "epoch": 0.09955555555555555, + "grad_norm": 0.017528358846902847, + "learning_rate": 2.7272727272727272e-06, + "loss": 0.8075190782546997, + "step": 7 + }, + { + "epoch": 0.11377777777777778, + "grad_norm": 0.01399513240903616, + "learning_rate": 3.181818181818182e-06, + "loss": 0.7754716873168945, + "step": 8 + }, + { + "epoch": 0.128, + "grad_norm": 0.012644859962165356, + "learning_rate": 3.6363636363636366e-06, + "loss": 0.7459083199501038, + "step": 9 + }, + { + "epoch": 0.14222222222222222, + "grad_norm": 0.01209743320941925, + "learning_rate": 4.0909090909090915e-06, + "loss": 0.7365868091583252, + "step": 10 + }, + { + "epoch": 0.15644444444444444, + "grad_norm": 0.012133832089602947, + "learning_rate": 4.5454545454545455e-06, + "loss": 0.7119501829147339, + "step": 11 + }, + { + "epoch": 0.17066666666666666, + "grad_norm": 0.010017997585237026, + "learning_rate": 5e-06, + "loss": 0.6901087760925293, + "step": 12 + }, + { + "epoch": 0.18488888888888888, + "grad_norm": 0.009233366698026657, + "learning_rate": 5.4545454545454545e-06, + "loss": 0.6611638069152832, + "step": 13 + }, + { + "epoch": 0.1991111111111111, + "grad_norm": 0.00782010331749916, + "learning_rate": 5.90909090909091e-06, + "loss": 0.6715224981307983, + "step": 14 + }, + { + "epoch": 0.21333333333333335, + "grad_norm": 0.007341764867305756, + "learning_rate": 6.363636363636364e-06, + "loss": 0.6277110576629639, + "step": 15 + }, + { + "epoch": 0.22755555555555557, + "grad_norm": 0.00713470159098506, + "learning_rate": 6.818181818181818e-06, + "loss": 0.6689565181732178, + "step": 16 + }, + { + "epoch": 0.24177777777777779, + "grad_norm": 0.0067572277039289474, + "learning_rate": 7.272727272727273e-06, + "loss": 0.6363875865936279, + "step": 17 + }, + { + "epoch": 0.256, + "grad_norm": 0.006228511221706867, + "learning_rate": 7.727272727272727e-06, + "loss": 0.6257224678993225, + "step": 18 + }, + { + "epoch": 0.2702222222222222, + "grad_norm": 0.006048806011676788, + "learning_rate": 8.181818181818183e-06, + "loss": 0.6355900764465332, + "step": 19 + }, + { + "epoch": 0.28444444444444444, + "grad_norm": 0.005988306365907192, + "learning_rate": 8.636363636363637e-06, + "loss": 0.6338369846343994, + "step": 20 + }, + { + "epoch": 0.2986666666666667, + "grad_norm": 0.005441761109977961, + "learning_rate": 9.090909090909091e-06, + "loss": 0.6133573055267334, + "step": 21 + }, + { + "epoch": 0.3128888888888889, + "grad_norm": 0.005174445919692516, + "learning_rate": 9.545454545454547e-06, + "loss": 0.6042410135269165, + "step": 22 + }, + { + "epoch": 0.32711111111111113, + "grad_norm": 0.005199746694415808, + "learning_rate": 1e-05, + "loss": 0.602219820022583, + "step": 23 + }, + { + "epoch": 0.3413333333333333, + "grad_norm": 0.005485028959810734, + "learning_rate": 9.999323662872998e-06, + "loss": 0.6062123775482178, + "step": 24 + }, + { + "epoch": 0.35555555555555557, + "grad_norm": 0.004896003287285566, + "learning_rate": 9.99729483446475e-06, + "loss": 0.6077945232391357, + "step": 25 + }, + { + "epoch": 0.36977777777777776, + "grad_norm": 0.00495151337236166, + "learning_rate": 9.993914063644053e-06, + "loss": 0.5923135280609131, + "step": 26 + }, + { + "epoch": 0.384, + "grad_norm": 0.004744194447994232, + "learning_rate": 9.989182265027232e-06, + "loss": 0.5860621929168701, + "step": 27 + }, + { + "epoch": 0.3982222222222222, + "grad_norm": 0.004471767693758011, + "learning_rate": 9.98310071873072e-06, + "loss": 0.5823097229003906, + "step": 28 + }, + { + "epoch": 0.41244444444444445, + "grad_norm": 0.004788320977240801, + "learning_rate": 9.975671070024741e-06, + "loss": 0.5872600078582764, + "step": 29 + }, + { + "epoch": 0.4266666666666667, + "grad_norm": 0.0044632842764258385, + "learning_rate": 9.966895328888195e-06, + "loss": 0.6070278286933899, + "step": 30 + }, + { + "epoch": 0.4408888888888889, + "grad_norm": 0.004317905753850937, + "learning_rate": 9.956775869464901e-06, + "loss": 0.5780316591262817, + "step": 31 + }, + { + "epoch": 0.45511111111111113, + "grad_norm": 0.004291852004826069, + "learning_rate": 9.945315429421307e-06, + "loss": 0.5697590112686157, + "step": 32 + }, + { + "epoch": 0.4693333333333333, + "grad_norm": 0.003922830801457167, + "learning_rate": 9.932517109205849e-06, + "loss": 0.5747107267379761, + "step": 33 + }, + { + "epoch": 0.48355555555555557, + "grad_norm": 0.0041889771819114685, + "learning_rate": 9.918384371210178e-06, + "loss": 0.6021177172660828, + "step": 34 + }, + { + "epoch": 0.49777777777777776, + "grad_norm": 0.004147357773035765, + "learning_rate": 9.902921038832456e-06, + "loss": 0.5805741548538208, + "step": 35 + }, + { + "epoch": 0.512, + "grad_norm": 0.0040724254213273525, + "learning_rate": 9.886131295443003e-06, + "loss": 0.5483200550079346, + "step": 36 + }, + { + "epoch": 0.5262222222222223, + "grad_norm": 0.004085644613951445, + "learning_rate": 9.868019683252543e-06, + "loss": 0.557392954826355, + "step": 37 + }, + { + "epoch": 0.5404444444444444, + "grad_norm": 0.004289102274924517, + "learning_rate": 9.848591102083375e-06, + "loss": 0.590857207775116, + "step": 38 + }, + { + "epoch": 0.5546666666666666, + "grad_norm": 0.0040345704182982445, + "learning_rate": 9.82785080804381e-06, + "loss": 0.5750541090965271, + "step": 39 + }, + { + "epoch": 0.5688888888888889, + "grad_norm": 0.004034748766571283, + "learning_rate": 9.805804412106197e-06, + "loss": 0.5949826240539551, + "step": 40 + }, + { + "epoch": 0.5831111111111111, + "grad_norm": 0.003911909181624651, + "learning_rate": 9.782457878588977e-06, + "loss": 0.5623177289962769, + "step": 41 + }, + { + "epoch": 0.5973333333333334, + "grad_norm": 0.0037001846358180046, + "learning_rate": 9.75781752354311e-06, + "loss": 0.5465081930160522, + "step": 42 + }, + { + "epoch": 0.6115555555555555, + "grad_norm": 0.0037095139268785715, + "learning_rate": 9.731890013043367e-06, + "loss": 0.5752100944519043, + "step": 43 + }, + { + "epoch": 0.6257777777777778, + "grad_norm": 0.003655617358162999, + "learning_rate": 9.704682361384941e-06, + "loss": 0.5452020168304443, + "step": 44 + }, + { + "epoch": 0.64, + "grad_norm": 0.003784338477998972, + "learning_rate": 9.676201929185809e-06, + "loss": 0.5616332292556763, + "step": 45 + }, + { + "epoch": 0.6542222222222223, + "grad_norm": 0.003718078136444092, + "learning_rate": 9.646456421395447e-06, + "loss": 0.5558297634124756, + "step": 46 + }, + { + "epoch": 0.6684444444444444, + "grad_norm": 0.00395169947296381, + "learning_rate": 9.615453885210368e-06, + "loss": 0.5600595474243164, + "step": 47 + }, + { + "epoch": 0.6826666666666666, + "grad_norm": 0.003555440343916416, + "learning_rate": 9.583202707897075e-06, + "loss": 0.5372895002365112, + "step": 48 + }, + { + "epoch": 0.6968888888888889, + "grad_norm": 0.0037386869080364704, + "learning_rate": 9.549711614523007e-06, + "loss": 0.5322733521461487, + "step": 49 + }, + { + "epoch": 0.7111111111111111, + "grad_norm": 0.003739988896995783, + "learning_rate": 9.514989665596114e-06, + "loss": 0.5380419492721558, + "step": 50 + }, + { + "epoch": 0.7111111111111111, + "eval_loss": 0.5587145090103149, + "eval_runtime": 8.7676, + "eval_samples_per_second": 22.811, + "eval_steps_per_second": 11.406, + "step": 50 + }, + { + "epoch": 0.7253333333333334, + "grad_norm": 0.0037539806216955185, + "learning_rate": 9.479046254613673e-06, + "loss": 0.5472225546836853, + "step": 51 + }, + { + "epoch": 0.7395555555555555, + "grad_norm": 0.003411181503906846, + "learning_rate": 9.441891105521005e-06, + "loss": 0.5134032964706421, + "step": 52 + }, + { + "epoch": 0.7537777777777778, + "grad_norm": 0.0037538670003414154, + "learning_rate": 9.40353427008083e-06, + "loss": 0.5361013412475586, + "step": 53 + }, + { + "epoch": 0.768, + "grad_norm": 0.0036495670210570097, + "learning_rate": 9.3639861251539e-06, + "loss": 0.5299779772758484, + "step": 54 + }, + { + "epoch": 0.7822222222222223, + "grad_norm": 0.0037342547439038754, + "learning_rate": 9.323257369891702e-06, + "loss": 0.5607945919036865, + "step": 55 + }, + { + "epoch": 0.7964444444444444, + "grad_norm": 0.003572584595531225, + "learning_rate": 9.281359022841966e-06, + "loss": 0.5528460741043091, + "step": 56 + }, + { + "epoch": 0.8106666666666666, + "grad_norm": 0.003770101349800825, + "learning_rate": 9.238302418967757e-06, + "loss": 0.5440288186073303, + "step": 57 + }, + { + "epoch": 0.8248888888888889, + "grad_norm": 0.0036679303739219904, + "learning_rate": 9.194099206580981e-06, + "loss": 0.5564252138137817, + "step": 58 + }, + { + "epoch": 0.8391111111111111, + "grad_norm": 0.003834612900391221, + "learning_rate": 9.14876134419111e-06, + "loss": 0.5473330020904541, + "step": 59 + }, + { + "epoch": 0.8533333333333334, + "grad_norm": 0.003683249931782484, + "learning_rate": 9.102301097269974e-06, + "loss": 0.5463743209838867, + "step": 60 + }, + { + "epoch": 0.8675555555555555, + "grad_norm": 0.0036777963396161795, + "learning_rate": 9.05473103493355e-06, + "loss": 0.524878740310669, + "step": 61 + }, + { + "epoch": 0.8817777777777778, + "grad_norm": 0.0038230482023209333, + "learning_rate": 9.006064026541549e-06, + "loss": 0.5563522577285767, + "step": 62 + }, + { + "epoch": 0.896, + "grad_norm": 0.003645201213657856, + "learning_rate": 8.956313238215824e-06, + "loss": 0.5059154033660889, + "step": 63 + }, + { + "epoch": 0.9102222222222223, + "grad_norm": 0.0035886459518224, + "learning_rate": 8.905492129278478e-06, + "loss": 0.5312881469726562, + "step": 64 + }, + { + "epoch": 0.9244444444444444, + "grad_norm": 0.003721872577443719, + "learning_rate": 8.85361444861063e-06, + "loss": 0.5358959436416626, + "step": 65 + }, + { + "epoch": 0.9386666666666666, + "grad_norm": 0.003683999879285693, + "learning_rate": 8.800694230932885e-06, + "loss": 0.5442053079605103, + "step": 66 + }, + { + "epoch": 0.9528888888888889, + "grad_norm": 0.0036851842887699604, + "learning_rate": 8.74674579300843e-06, + "loss": 0.5263723731040955, + "step": 67 + }, + { + "epoch": 0.9671111111111111, + "grad_norm": 0.003754258854314685, + "learning_rate": 8.691783729769874e-06, + "loss": 0.5200538039207458, + "step": 68 + }, + { + "epoch": 0.9813333333333333, + "grad_norm": 0.0038373274728655815, + "learning_rate": 8.635822910370793e-06, + "loss": 0.5452108383178711, + "step": 69 + }, + { + "epoch": 0.9955555555555555, + "grad_norm": 0.0039480882696807384, + "learning_rate": 8.578878474163115e-06, + "loss": 0.5066030621528625, + "step": 70 + }, + { + "epoch": 1.0, + "grad_norm": 0.005897017661482096, + "learning_rate": 8.520965826601394e-06, + "loss": 0.5380499958992004, + "step": 71 + } + ], + "logging_steps": 1, + "max_steps": 213, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8.242478482194432e+16, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/qwen2.5-1.5b-instruct/checkpoint-71/training_args.bin b/qwen2.5-1.5b-instruct/checkpoint-71/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..4badd3789f92f64e02e2816810bfe346973afac5 --- /dev/null +++ b/qwen2.5-1.5b-instruct/checkpoint-71/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39e78c30a5784e935e65cb8fa1a73b2afb467afccc55e7223863b667252fac3b +size 5649 diff --git a/qwen2.5-1.5b-instruct/config.json b/qwen2.5-1.5b-instruct/config.json new file mode 100644 index 0000000000000000000000000000000000000000..499065fd744a64fc8ad2884da26e85a6e99d468a --- /dev/null +++ b/qwen2.5-1.5b-instruct/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 21, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/qwen2.5-1.5b-instruct/eval_results.json b/qwen2.5-1.5b-instruct/eval_results.json new file mode 100644 index 0000000000000000000000000000000000000000..4067c423d5a9a71932113934e3e062e9533b4f8a --- /dev/null +++ b/qwen2.5-1.5b-instruct/eval_results.json @@ -0,0 +1,7 @@ +{ + "epoch": 3.0, + "eval_loss": 0.5171241164207458, + "eval_runtime": 8.31, + "eval_samples_per_second": 24.067, + "eval_steps_per_second": 12.034 +} \ No newline at end of file diff --git a/qwen2.5-1.5b-instruct/generation_config.json b/qwen2.5-1.5b-instruct/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c5b4e11411fed3e7a5e6557165f888a7b0b1248b --- /dev/null +++ b/qwen2.5-1.5b-instruct/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.2.0" +} diff --git a/qwen2.5-1.5b-instruct/model.safetensors b/qwen2.5-1.5b-instruct/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2db9ad9ca66aca5f7b313dd910472c08389e0134 --- /dev/null +++ b/qwen2.5-1.5b-instruct/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f14867d928257934392dd2a17c62080e0613e998d7c4e10eba82a99bef35153 +size 6174895536 diff --git a/qwen2.5-1.5b-instruct/tokenizer.json b/qwen2.5-1.5b-instruct/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..34510ff0037cd50428af467a17ead5a96140a32c --- /dev/null +++ b/qwen2.5-1.5b-instruct/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/qwen2.5-1.5b-instruct/tokenizer_config.json b/qwen2.5-1.5b-instruct/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..145e2c7a68b109ae0253dda29fc2e66cb01b4b85 --- /dev/null +++ b/qwen2.5-1.5b-instruct/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen2.5-1.5b-instruct/train_results.json b/qwen2.5-1.5b-instruct/train_results.json new file mode 100644 index 0000000000000000000000000000000000000000..bfefc0b6438d0cca7c40a998db820a51cb64cf88 --- /dev/null +++ b/qwen2.5-1.5b-instruct/train_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 3.0, + "total_flos": 2.4727435500270387e+17, + "train_loss": 0.5265021298972654, + "train_runtime": 6756.1203, + "train_samples_per_second": 7.993, + "train_steps_per_second": 0.032 +} \ No newline at end of file diff --git a/qwen2.5-1.5b-instruct/trainer_log.jsonl b/qwen2.5-1.5b-instruct/trainer_log.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b651cc09d895b45f3113a97a4d0291a99d9df8ea --- /dev/null +++ b/qwen2.5-1.5b-instruct/trainer_log.jsonl @@ -0,0 +1,218 @@ +{"current_steps": 1, "total_steps": 213, "loss": 0.9098724722862244, "lr": 0.0, "epoch": 0.014222222222222223, "percentage": 0.47, "elapsed_time": "0:00:35", "remaining_time": "2:06:27"} +{"current_steps": 2, "total_steps": 213, "loss": 0.9084522724151611, "lr": 4.5454545454545457e-07, "epoch": 0.028444444444444446, "percentage": 0.94, "elapsed_time": "0:01:09", "remaining_time": "2:02:58"} +{"current_steps": 3, "total_steps": 213, "loss": 0.8834755420684814, "lr": 9.090909090909091e-07, "epoch": 0.042666666666666665, "percentage": 1.41, "elapsed_time": "0:01:44", "remaining_time": "2:02:28"} +{"current_steps": 4, "total_steps": 213, "loss": 0.871230959892273, "lr": 1.3636363636363636e-06, "epoch": 0.05688888888888889, "percentage": 1.88, "elapsed_time": "0:02:18", "remaining_time": "2:00:49"} +{"current_steps": 5, "total_steps": 213, "loss": 0.7917206287384033, "lr": 1.8181818181818183e-06, "epoch": 0.07111111111111111, "percentage": 2.35, "elapsed_time": "0:02:53", "remaining_time": "2:00:16"} +{"current_steps": 6, "total_steps": 213, "loss": 0.8436126708984375, "lr": 2.2727272727272728e-06, "epoch": 0.08533333333333333, "percentage": 2.82, "elapsed_time": "0:03:27", "remaining_time": "1:59:28"} +{"current_steps": 7, "total_steps": 213, "loss": 0.8075190782546997, "lr": 2.7272727272727272e-06, "epoch": 0.09955555555555555, "percentage": 3.29, "elapsed_time": "0:04:01", "remaining_time": "1:58:19"} +{"current_steps": 8, "total_steps": 213, "loss": 0.7754716873168945, "lr": 3.181818181818182e-06, "epoch": 0.11377777777777778, "percentage": 3.76, "elapsed_time": "0:04:35", "remaining_time": "1:57:42"} +{"current_steps": 9, "total_steps": 213, "loss": 0.7459083199501038, "lr": 3.6363636363636366e-06, "epoch": 0.128, "percentage": 4.23, "elapsed_time": "0:05:09", "remaining_time": "1:57:01"} +{"current_steps": 10, "total_steps": 213, "loss": 0.7365868091583252, "lr": 4.0909090909090915e-06, "epoch": 0.14222222222222222, "percentage": 4.69, "elapsed_time": "0:05:45", "remaining_time": "1:56:44"} +{"current_steps": 11, "total_steps": 213, "loss": 0.7119501829147339, "lr": 4.5454545454545455e-06, "epoch": 0.15644444444444444, "percentage": 5.16, "elapsed_time": "0:06:26", "remaining_time": "1:58:20"} +{"current_steps": 12, "total_steps": 213, "loss": 0.6901087760925293, "lr": 5e-06, "epoch": 0.17066666666666666, "percentage": 5.63, "elapsed_time": "0:07:08", "remaining_time": "1:59:41"} +{"current_steps": 13, "total_steps": 213, "loss": 0.6611638069152832, "lr": 5.4545454545454545e-06, "epoch": 0.18488888888888888, "percentage": 6.1, "elapsed_time": "0:07:50", "remaining_time": "2:00:40"} +{"current_steps": 14, "total_steps": 213, "loss": 0.6715224981307983, "lr": 5.90909090909091e-06, "epoch": 0.1991111111111111, "percentage": 6.57, "elapsed_time": "0:08:33", "remaining_time": "2:01:35"} +{"current_steps": 15, "total_steps": 213, "loss": 0.6277110576629639, "lr": 6.363636363636364e-06, "epoch": 0.21333333333333335, "percentage": 7.04, "elapsed_time": "0:09:14", "remaining_time": "2:02:01"} +{"current_steps": 16, "total_steps": 213, "loss": 0.6689565181732178, "lr": 6.818181818181818e-06, "epoch": 0.22755555555555557, "percentage": 7.51, "elapsed_time": "0:09:56", "remaining_time": "2:02:24"} +{"current_steps": 17, "total_steps": 213, "loss": 0.6363875865936279, "lr": 7.272727272727273e-06, "epoch": 0.24177777777777779, "percentage": 7.98, "elapsed_time": "0:10:38", "remaining_time": "2:02:36"} +{"current_steps": 18, "total_steps": 213, "loss": 0.6257224678993225, "lr": 7.727272727272727e-06, "epoch": 0.256, "percentage": 8.45, "elapsed_time": "0:11:19", "remaining_time": "2:02:38"} +{"current_steps": 19, "total_steps": 213, "loss": 0.6355900764465332, "lr": 8.181818181818183e-06, "epoch": 0.2702222222222222, "percentage": 8.92, "elapsed_time": "0:12:01", "remaining_time": "2:02:46"} +{"current_steps": 20, "total_steps": 213, "loss": 0.6338369846343994, "lr": 8.636363636363637e-06, "epoch": 0.28444444444444444, "percentage": 9.39, "elapsed_time": "0:12:43", "remaining_time": "2:02:44"} +{"current_steps": 21, "total_steps": 213, "loss": 0.6133573055267334, "lr": 9.090909090909091e-06, "epoch": 0.2986666666666667, "percentage": 9.86, "elapsed_time": "0:13:25", "remaining_time": "2:02:43"} +{"current_steps": 22, "total_steps": 213, "loss": 0.6042410135269165, "lr": 9.545454545454547e-06, "epoch": 0.3128888888888889, "percentage": 10.33, "elapsed_time": "0:14:07", "remaining_time": "2:02:36"} +{"current_steps": 23, "total_steps": 213, "loss": 0.602219820022583, "lr": 1e-05, "epoch": 0.32711111111111113, "percentage": 10.8, "elapsed_time": "0:14:49", "remaining_time": "2:02:28"} +{"current_steps": 24, "total_steps": 213, "loss": 0.6062123775482178, "lr": 9.999323662872998e-06, "epoch": 0.3413333333333333, "percentage": 11.27, "elapsed_time": "0:15:30", "remaining_time": "2:02:11"} +{"current_steps": 25, "total_steps": 213, "loss": 0.6077945232391357, "lr": 9.99729483446475e-06, "epoch": 0.35555555555555557, "percentage": 11.74, "elapsed_time": "0:16:13", "remaining_time": "2:01:57"} +{"current_steps": 26, "total_steps": 213, "loss": 0.5923135280609131, "lr": 9.993914063644053e-06, "epoch": 0.36977777777777776, "percentage": 12.21, "elapsed_time": "0:16:54", "remaining_time": "2:01:36"} +{"current_steps": 27, "total_steps": 213, "loss": 0.5860621929168701, "lr": 9.989182265027232e-06, "epoch": 0.384, "percentage": 12.68, "elapsed_time": "0:17:33", "remaining_time": "2:00:57"} +{"current_steps": 28, "total_steps": 213, "loss": 0.5823097229003906, "lr": 9.98310071873072e-06, "epoch": 0.3982222222222222, "percentage": 13.15, "elapsed_time": "0:18:03", "remaining_time": "1:59:16"} +{"current_steps": 29, "total_steps": 213, "loss": 0.5872600078582764, "lr": 9.975671070024741e-06, "epoch": 0.41244444444444445, "percentage": 13.62, "elapsed_time": "0:18:32", "remaining_time": "1:57:36"} +{"current_steps": 30, "total_steps": 213, "loss": 0.6070278286933899, "lr": 9.966895328888195e-06, "epoch": 0.4266666666666667, "percentage": 14.08, "elapsed_time": "0:19:01", "remaining_time": "1:56:06"} +{"current_steps": 31, "total_steps": 213, "loss": 0.5780316591262817, "lr": 9.956775869464901e-06, "epoch": 0.4408888888888889, "percentage": 14.55, "elapsed_time": "0:19:31", "remaining_time": "1:54:39"} +{"current_steps": 32, "total_steps": 213, "loss": 0.5697590112686157, "lr": 9.945315429421307e-06, "epoch": 0.45511111111111113, "percentage": 15.02, "elapsed_time": "0:20:01", "remaining_time": "1:53:15"} +{"current_steps": 33, "total_steps": 213, "loss": 0.5747107267379761, "lr": 9.932517109205849e-06, "epoch": 0.4693333333333333, "percentage": 15.49, "elapsed_time": "0:20:30", "remaining_time": "1:51:52"} +{"current_steps": 34, "total_steps": 213, "loss": 0.6021177172660828, "lr": 9.918384371210178e-06, "epoch": 0.48355555555555557, "percentage": 15.96, "elapsed_time": "0:20:59", "remaining_time": "1:50:30"} +{"current_steps": 35, "total_steps": 213, "loss": 0.5805741548538208, "lr": 9.902921038832456e-06, "epoch": 0.49777777777777776, "percentage": 16.43, "elapsed_time": "0:21:28", "remaining_time": "1:49:12"} +{"current_steps": 36, "total_steps": 213, "loss": 0.5483200550079346, "lr": 9.886131295443003e-06, "epoch": 0.512, "percentage": 16.9, "elapsed_time": "0:21:57", "remaining_time": "1:47:58"} +{"current_steps": 37, "total_steps": 213, "loss": 0.557392954826355, "lr": 9.868019683252543e-06, "epoch": 0.5262222222222223, "percentage": 17.37, "elapsed_time": "0:22:26", "remaining_time": "1:46:47"} +{"current_steps": 38, "total_steps": 213, "loss": 0.590857207775116, "lr": 9.848591102083375e-06, "epoch": 0.5404444444444444, "percentage": 17.84, "elapsed_time": "0:22:55", "remaining_time": "1:45:36"} +{"current_steps": 39, "total_steps": 213, "loss": 0.5750541090965271, "lr": 9.82785080804381e-06, "epoch": 0.5546666666666666, "percentage": 18.31, "elapsed_time": "0:23:25", "remaining_time": "1:44:29"} +{"current_steps": 40, "total_steps": 213, "loss": 0.5949826240539551, "lr": 9.805804412106197e-06, "epoch": 0.5688888888888889, "percentage": 18.78, "elapsed_time": "0:23:53", "remaining_time": "1:43:21"} +{"current_steps": 41, "total_steps": 213, "loss": 0.5623177289962769, "lr": 9.782457878588977e-06, "epoch": 0.5831111111111111, "percentage": 19.25, "elapsed_time": "0:24:23", "remaining_time": "1:42:20"} +{"current_steps": 42, "total_steps": 213, "loss": 0.5465081930160522, "lr": 9.75781752354311e-06, "epoch": 0.5973333333333334, "percentage": 19.72, "elapsed_time": "0:24:54", "remaining_time": "1:41:22"} +{"current_steps": 43, "total_steps": 213, "loss": 0.5752100944519043, "lr": 9.731890013043367e-06, "epoch": 0.6115555555555555, "percentage": 20.19, "elapsed_time": "0:25:23", "remaining_time": "1:40:22"} +{"current_steps": 44, "total_steps": 213, "loss": 0.5452020168304443, "lr": 9.704682361384941e-06, "epoch": 0.6257777777777778, "percentage": 20.66, "elapsed_time": "0:25:54", "remaining_time": "1:39:31"} +{"current_steps": 45, "total_steps": 213, "loss": 0.5616332292556763, "lr": 9.676201929185809e-06, "epoch": 0.64, "percentage": 21.13, "elapsed_time": "0:26:23", "remaining_time": "1:38:30"} +{"current_steps": 46, "total_steps": 213, "loss": 0.5558297634124756, "lr": 9.646456421395447e-06, "epoch": 0.6542222222222223, "percentage": 21.6, "elapsed_time": "0:26:53", "remaining_time": "1:37:38"} +{"current_steps": 47, "total_steps": 213, "loss": 0.5600595474243164, "lr": 9.615453885210368e-06, "epoch": 0.6684444444444444, "percentage": 22.07, "elapsed_time": "0:27:23", "remaining_time": "1:36:43"} +{"current_steps": 48, "total_steps": 213, "loss": 0.5372895002365112, "lr": 9.583202707897075e-06, "epoch": 0.6826666666666666, "percentage": 22.54, "elapsed_time": "0:27:53", "remaining_time": "1:35:53"} +{"current_steps": 49, "total_steps": 213, "loss": 0.5322733521461487, "lr": 9.549711614523007e-06, "epoch": 0.6968888888888889, "percentage": 23.0, "elapsed_time": "0:28:21", "remaining_time": "1:34:55"} +{"current_steps": 50, "total_steps": 213, "loss": 0.5380419492721558, "lr": 9.514989665596114e-06, "epoch": 0.7111111111111111, "percentage": 23.47, "elapsed_time": "0:28:52", "remaining_time": "1:34:06"} +{"current_steps": 50, "total_steps": 213, "eval_loss": 0.5587145090103149, "epoch": 0.7111111111111111, "percentage": 23.47, "elapsed_time": "0:29:01", "remaining_time": "1:34:37"} +{"current_steps": 51, "total_steps": 213, "loss": 0.5472225546836853, "lr": 9.479046254613673e-06, "epoch": 0.7253333333333334, "percentage": 23.94, "elapsed_time": "0:29:32", "remaining_time": "1:33:48"} +{"current_steps": 52, "total_steps": 213, "loss": 0.5134032964706421, "lr": 9.441891105521005e-06, "epoch": 0.7395555555555555, "percentage": 24.41, "elapsed_time": "0:30:03", "remaining_time": "1:33:03"} +{"current_steps": 53, "total_steps": 213, "loss": 0.5361013412475586, "lr": 9.40353427008083e-06, "epoch": 0.7537777777777778, "percentage": 24.88, "elapsed_time": "0:30:33", "remaining_time": "1:32:15"} +{"current_steps": 54, "total_steps": 213, "loss": 0.5299779772758484, "lr": 9.3639861251539e-06, "epoch": 0.768, "percentage": 25.35, "elapsed_time": "0:31:04", "remaining_time": "1:31:30"} +{"current_steps": 55, "total_steps": 213, "loss": 0.5607945919036865, "lr": 9.323257369891702e-06, "epoch": 0.7822222222222223, "percentage": 25.82, "elapsed_time": "0:31:33", "remaining_time": "1:30:40"} +{"current_steps": 56, "total_steps": 213, "loss": 0.5528460741043091, "lr": 9.281359022841966e-06, "epoch": 0.7964444444444444, "percentage": 26.29, "elapsed_time": "0:32:03", "remaining_time": "1:29:51"} +{"current_steps": 57, "total_steps": 213, "loss": 0.5440288186073303, "lr": 9.238302418967757e-06, "epoch": 0.8106666666666666, "percentage": 26.76, "elapsed_time": "0:32:32", "remaining_time": "1:29:03"} +{"current_steps": 58, "total_steps": 213, "loss": 0.5564252138137817, "lr": 9.194099206580981e-06, "epoch": 0.8248888888888889, "percentage": 27.23, "elapsed_time": "0:33:02", "remaining_time": "1:28:17"} +{"current_steps": 59, "total_steps": 213, "loss": 0.5473330020904541, "lr": 9.14876134419111e-06, "epoch": 0.8391111111111111, "percentage": 27.7, "elapsed_time": "0:33:31", "remaining_time": "1:27:30"} +{"current_steps": 60, "total_steps": 213, "loss": 0.5463743209838867, "lr": 9.102301097269974e-06, "epoch": 0.8533333333333334, "percentage": 28.17, "elapsed_time": "0:34:01", "remaining_time": "1:26:46"} +{"current_steps": 61, "total_steps": 213, "loss": 0.524878740310669, "lr": 9.05473103493355e-06, "epoch": 0.8675555555555555, "percentage": 28.64, "elapsed_time": "0:34:31", "remaining_time": "1:26:01"} +{"current_steps": 62, "total_steps": 213, "loss": 0.5563522577285767, "lr": 9.006064026541549e-06, "epoch": 0.8817777777777778, "percentage": 29.11, "elapsed_time": "0:35:00", "remaining_time": "1:25:14"} +{"current_steps": 63, "total_steps": 213, "loss": 0.5059154033660889, "lr": 8.956313238215824e-06, "epoch": 0.896, "percentage": 29.58, "elapsed_time": "0:35:32", "remaining_time": "1:24:37"} +{"current_steps": 64, "total_steps": 213, "loss": 0.5312881469726562, "lr": 8.905492129278478e-06, "epoch": 0.9102222222222223, "percentage": 30.05, "elapsed_time": "0:36:01", "remaining_time": "1:23:52"} +{"current_steps": 65, "total_steps": 213, "loss": 0.5358959436416626, "lr": 8.85361444861063e-06, "epoch": 0.9244444444444444, "percentage": 30.52, "elapsed_time": "0:36:32", "remaining_time": "1:23:12"} +{"current_steps": 66, "total_steps": 213, "loss": 0.5442053079605103, "lr": 8.800694230932885e-06, "epoch": 0.9386666666666666, "percentage": 30.99, "elapsed_time": "0:37:02", "remaining_time": "1:22:30"} +{"current_steps": 67, "total_steps": 213, "loss": 0.5263723731040955, "lr": 8.74674579300843e-06, "epoch": 0.9528888888888889, "percentage": 31.46, "elapsed_time": "0:37:33", "remaining_time": "1:21:50"} +{"current_steps": 68, "total_steps": 213, "loss": 0.5200538039207458, "lr": 8.691783729769874e-06, "epoch": 0.9671111111111111, "percentage": 31.92, "elapsed_time": "0:38:03", "remaining_time": "1:21:10"} +{"current_steps": 69, "total_steps": 213, "loss": 0.5452108383178711, "lr": 8.635822910370793e-06, "epoch": 0.9813333333333333, "percentage": 32.39, "elapsed_time": "0:38:34", "remaining_time": "1:20:29"} +{"current_steps": 70, "total_steps": 213, "loss": 0.5066030621528625, "lr": 8.578878474163115e-06, "epoch": 0.9955555555555555, "percentage": 32.86, "elapsed_time": "0:39:03", "remaining_time": "1:19:48"} +{"current_steps": 71, "total_steps": 213, "loss": 0.5380499958992004, "lr": 8.520965826601394e-06, "epoch": 1.0, "percentage": 33.33, "elapsed_time": "0:39:13", "remaining_time": "1:18:27"} +{"current_steps": 72, "total_steps": 213, "loss": 0.5328541398048401, "lr": 8.462100635075097e-06, "epoch": 1.0142222222222221, "percentage": 33.8, "elapsed_time": "0:40:43", "remaining_time": "1:19:46"} +{"current_steps": 73, "total_steps": 213, "loss": 0.5096005201339722, "lr": 8.40229882467003e-06, "epoch": 1.0284444444444445, "percentage": 34.27, "elapsed_time": "0:41:13", "remaining_time": "1:19:04"} +{"current_steps": 74, "total_steps": 213, "loss": 0.4889724850654602, "lr": 8.341576573860049e-06, "epoch": 1.0426666666666666, "percentage": 34.74, "elapsed_time": "0:41:45", "remaining_time": "1:18:26"} +{"current_steps": 75, "total_steps": 213, "loss": 0.5157173871994019, "lr": 8.279950310130218e-06, "epoch": 1.056888888888889, "percentage": 35.21, "elapsed_time": "0:42:15", "remaining_time": "1:17:45"} +{"current_steps": 76, "total_steps": 213, "loss": 0.49691659212112427, "lr": 8.2174367055326e-06, "epoch": 1.0711111111111111, "percentage": 35.68, "elapsed_time": "0:42:44", "remaining_time": "1:17:03"} +{"current_steps": 77, "total_steps": 213, "loss": 0.4873769283294678, "lr": 8.154052672175888e-06, "epoch": 1.0853333333333333, "percentage": 36.15, "elapsed_time": "0:43:14", "remaining_time": "1:16:21"} +{"current_steps": 78, "total_steps": 213, "loss": 0.5266215801239014, "lr": 8.08981535765009e-06, "epoch": 1.0995555555555556, "percentage": 36.62, "elapsed_time": "0:43:44", "remaining_time": "1:15:42"} +{"current_steps": 79, "total_steps": 213, "loss": 0.5039150714874268, "lr": 8.024742140387506e-06, "epoch": 1.1137777777777778, "percentage": 37.09, "elapsed_time": "0:44:13", "remaining_time": "1:15:01"} +{"current_steps": 80, "total_steps": 213, "loss": 0.4896891117095947, "lr": 7.95885062496126e-06, "epoch": 1.1280000000000001, "percentage": 37.56, "elapsed_time": "0:44:43", "remaining_time": "1:14:21"} +{"current_steps": 81, "total_steps": 213, "loss": 0.45997822284698486, "lr": 7.892158637322647e-06, "epoch": 1.1422222222222222, "percentage": 38.03, "elapsed_time": "0:45:12", "remaining_time": "1:13:40"} +{"current_steps": 82, "total_steps": 213, "loss": 0.4917220175266266, "lr": 7.824684219978591e-06, "epoch": 1.1564444444444444, "percentage": 38.5, "elapsed_time": "0:45:42", "remaining_time": "1:13:01"} +{"current_steps": 83, "total_steps": 213, "loss": 0.5024806261062622, "lr": 7.756445627110523e-06, "epoch": 1.1706666666666667, "percentage": 38.97, "elapsed_time": "0:46:13", "remaining_time": "1:12:23"} +{"current_steps": 84, "total_steps": 213, "loss": 0.5110447406768799, "lr": 7.68746131963598e-06, "epoch": 1.1848888888888889, "percentage": 39.44, "elapsed_time": "0:46:42", "remaining_time": "1:11:43"} +{"current_steps": 85, "total_steps": 213, "loss": 0.5031975507736206, "lr": 7.6177499602143e-06, "epoch": 1.199111111111111, "percentage": 39.91, "elapsed_time": "0:47:11", "remaining_time": "1:11:04"} +{"current_steps": 86, "total_steps": 213, "loss": 0.4899296164512634, "lr": 7.547330408197695e-06, "epoch": 1.2133333333333334, "percentage": 40.38, "elapsed_time": "0:47:42", "remaining_time": "1:10:27"} +{"current_steps": 87, "total_steps": 213, "loss": 0.5040560960769653, "lr": 7.476221714529167e-06, "epoch": 1.2275555555555555, "percentage": 40.85, "elapsed_time": "0:48:12", "remaining_time": "1:09:49"} +{"current_steps": 88, "total_steps": 213, "loss": 0.5307536125183105, "lr": 7.404443116588548e-06, "epoch": 1.2417777777777779, "percentage": 41.31, "elapsed_time": "0:48:41", "remaining_time": "1:09:09"} +{"current_steps": 89, "total_steps": 213, "loss": 0.5025057792663574, "lr": 7.332014032988123e-06, "epoch": 1.256, "percentage": 41.78, "elapsed_time": "0:49:10", "remaining_time": "1:08:31"} +{"current_steps": 90, "total_steps": 213, "loss": 0.5043497085571289, "lr": 7.2589540583192165e-06, "epoch": 1.2702222222222221, "percentage": 42.25, "elapsed_time": "0:49:40", "remaining_time": "1:07:53"} +{"current_steps": 91, "total_steps": 213, "loss": 0.4883122444152832, "lr": 7.185282957851175e-06, "epoch": 1.2844444444444445, "percentage": 42.72, "elapsed_time": "0:50:10", "remaining_time": "1:07:16"} +{"current_steps": 92, "total_steps": 213, "loss": 0.5055882930755615, "lr": 7.111020662184174e-06, "epoch": 1.2986666666666666, "percentage": 43.19, "elapsed_time": "0:50:41", "remaining_time": "1:06:40"} +{"current_steps": 93, "total_steps": 213, "loss": 0.4962804615497589, "lr": 7.036187261857289e-06, "epoch": 1.3128888888888888, "percentage": 43.66, "elapsed_time": "0:51:10", "remaining_time": "1:06:02"} +{"current_steps": 94, "total_steps": 213, "loss": 0.524259090423584, "lr": 6.960803001913315e-06, "epoch": 1.3271111111111111, "percentage": 44.13, "elapsed_time": "0:51:39", "remaining_time": "1:05:23"} +{"current_steps": 95, "total_steps": 213, "loss": 0.5073269009590149, "lr": 6.884888276421766e-06, "epoch": 1.3413333333333333, "percentage": 44.6, "elapsed_time": "0:52:08", "remaining_time": "1:04:46"} +{"current_steps": 96, "total_steps": 213, "loss": 0.4886914789676666, "lr": 6.8084636229615786e-06, "epoch": 1.3555555555555556, "percentage": 45.07, "elapsed_time": "0:52:38", "remaining_time": "1:04:09"} +{"current_steps": 97, "total_steps": 213, "loss": 0.4920746088027954, "lr": 6.731549717064975e-06, "epoch": 1.3697777777777778, "percentage": 45.54, "elapsed_time": "0:53:08", "remaining_time": "1:03:33"} +{"current_steps": 98, "total_steps": 213, "loss": 0.503207266330719, "lr": 6.654167366624009e-06, "epoch": 1.384, "percentage": 46.01, "elapsed_time": "0:53:36", "remaining_time": "1:02:54"} +{"current_steps": 99, "total_steps": 213, "loss": 0.5012996196746826, "lr": 6.576337506261314e-06, "epoch": 1.3982222222222223, "percentage": 46.48, "elapsed_time": "0:54:06", "remaining_time": "1:02:18"} +{"current_steps": 100, "total_steps": 213, "loss": 0.5059630274772644, "lr": 6.498081191666549e-06, "epoch": 1.4124444444444444, "percentage": 46.95, "elapsed_time": "0:54:37", "remaining_time": "1:01:43"} +{"current_steps": 100, "total_steps": 213, "eval_loss": 0.5291631817817688, "epoch": 1.4124444444444444, "percentage": 46.95, "elapsed_time": "0:54:47", "remaining_time": "1:01:54"} +{"current_steps": 101, "total_steps": 213, "loss": 0.4896056056022644, "lr": 6.419419593900109e-06, "epoch": 1.4266666666666667, "percentage": 47.42, "elapsed_time": "0:55:16", "remaining_time": "1:01:17"} +{"current_steps": 102, "total_steps": 213, "loss": 0.48181772232055664, "lr": 6.340373993665607e-06, "epoch": 1.4408888888888889, "percentage": 47.89, "elapsed_time": "0:55:45", "remaining_time": "1:00:41"} +{"current_steps": 103, "total_steps": 213, "loss": 0.4775124192237854, "lr": 6.2609657755527135e-06, "epoch": 1.455111111111111, "percentage": 48.36, "elapsed_time": "0:56:15", "remaining_time": "1:00:04"} +{"current_steps": 104, "total_steps": 213, "loss": 0.5167958736419678, "lr": 6.1812164222518626e-06, "epoch": 1.4693333333333334, "percentage": 48.83, "elapsed_time": "0:56:44", "remaining_time": "0:59:28"} +{"current_steps": 105, "total_steps": 213, "loss": 0.4946171045303345, "lr": 6.101147508742456e-06, "epoch": 1.4835555555555555, "percentage": 49.3, "elapsed_time": "0:57:14", "remaining_time": "0:58:52"} +{"current_steps": 106, "total_steps": 213, "loss": 0.4849076569080353, "lr": 6.020780696456059e-06, "epoch": 1.4977777777777779, "percentage": 49.77, "elapsed_time": "0:57:44", "remaining_time": "0:58:17"} +{"current_steps": 107, "total_steps": 213, "loss": 0.4958712160587311, "lr": 5.940137727416247e-06, "epoch": 1.512, "percentage": 50.23, "elapsed_time": "0:58:15", "remaining_time": "0:57:42"} +{"current_steps": 108, "total_steps": 213, "loss": 0.4822971820831299, "lr": 5.859240418356614e-06, "epoch": 1.5262222222222221, "percentage": 50.7, "elapsed_time": "0:58:45", "remaining_time": "0:57:07"} +{"current_steps": 109, "total_steps": 213, "loss": 0.5280419588088989, "lr": 5.778110654818602e-06, "epoch": 1.5404444444444443, "percentage": 51.17, "elapsed_time": "0:59:14", "remaining_time": "0:56:31"} +{"current_steps": 110, "total_steps": 213, "loss": 0.4952104687690735, "lr": 5.696770385230679e-06, "epoch": 1.5546666666666666, "percentage": 51.64, "elapsed_time": "0:59:44", "remaining_time": "0:55:55"} +{"current_steps": 111, "total_steps": 213, "loss": 0.4887908101081848, "lr": 5.615241614970546e-06, "epoch": 1.568888888888889, "percentage": 52.11, "elapsed_time": "1:00:13", "remaining_time": "0:55:20"} +{"current_steps": 112, "total_steps": 213, "loss": 0.4928241968154907, "lr": 5.533546400411899e-06, "epoch": 1.5831111111111111, "percentage": 52.58, "elapsed_time": "1:00:42", "remaining_time": "0:54:44"} +{"current_steps": 113, "total_steps": 213, "loss": 0.5109080076217651, "lr": 5.4517068429574215e-06, "epoch": 1.5973333333333333, "percentage": 53.05, "elapsed_time": "1:01:12", "remaining_time": "0:54:10"} +{"current_steps": 114, "total_steps": 213, "loss": 0.48911499977111816, "lr": 5.3697450830595775e-06, "epoch": 1.6115555555555554, "percentage": 53.52, "elapsed_time": "1:01:43", "remaining_time": "0:53:36"} +{"current_steps": 115, "total_steps": 213, "loss": 0.49537116289138794, "lr": 5.287683294230855e-06, "epoch": 1.6257777777777778, "percentage": 53.99, "elapsed_time": "1:02:11", "remaining_time": "0:53:00"} +{"current_steps": 116, "total_steps": 213, "loss": 0.4613257348537445, "lr": 5.20554367704505e-06, "epoch": 1.6400000000000001, "percentage": 54.46, "elapsed_time": "1:02:42", "remaining_time": "0:52:26"} +{"current_steps": 117, "total_steps": 213, "loss": 0.5043697357177734, "lr": 5.123348453131242e-06, "epoch": 1.6542222222222223, "percentage": 54.93, "elapsed_time": "1:03:12", "remaining_time": "0:51:51"} +{"current_steps": 118, "total_steps": 213, "loss": 0.48968735337257385, "lr": 5.041119859162068e-06, "epoch": 1.6684444444444444, "percentage": 55.4, "elapsed_time": "1:03:42", "remaining_time": "0:51:17"} +{"current_steps": 119, "total_steps": 213, "loss": 0.49605250358581543, "lr": 4.958880140837934e-06, "epoch": 1.6826666666666665, "percentage": 55.87, "elapsed_time": "1:04:11", "remaining_time": "0:50:42"} +{"current_steps": 120, "total_steps": 213, "loss": 0.4839916229248047, "lr": 4.876651546868759e-06, "epoch": 1.696888888888889, "percentage": 56.34, "elapsed_time": "1:04:41", "remaining_time": "0:50:07"} +{"current_steps": 121, "total_steps": 213, "loss": 0.5173561573028564, "lr": 4.794456322954953e-06, "epoch": 1.7111111111111112, "percentage": 56.81, "elapsed_time": "1:05:11", "remaining_time": "0:49:34"} +{"current_steps": 122, "total_steps": 213, "loss": 0.5177811980247498, "lr": 4.7123167057691446e-06, "epoch": 1.7253333333333334, "percentage": 57.28, "elapsed_time": "1:05:41", "remaining_time": "0:48:59"} +{"current_steps": 123, "total_steps": 213, "loss": 0.5164836049079895, "lr": 4.630254916940424e-06, "epoch": 1.7395555555555555, "percentage": 57.75, "elapsed_time": "1:06:12", "remaining_time": "0:48:26"} +{"current_steps": 124, "total_steps": 213, "loss": 0.5077825784683228, "lr": 4.548293157042581e-06, "epoch": 1.7537777777777777, "percentage": 58.22, "elapsed_time": "1:06:44", "remaining_time": "0:47:53"} +{"current_steps": 125, "total_steps": 213, "loss": 0.5107009410858154, "lr": 4.466453599588103e-06, "epoch": 1.768, "percentage": 58.69, "elapsed_time": "1:07:14", "remaining_time": "0:47:20"} +{"current_steps": 126, "total_steps": 213, "loss": 0.4728143811225891, "lr": 4.384758385029457e-06, "epoch": 1.7822222222222224, "percentage": 59.15, "elapsed_time": "1:07:44", "remaining_time": "0:46:46"} +{"current_steps": 127, "total_steps": 213, "loss": 0.4858134984970093, "lr": 4.3032296147693225e-06, "epoch": 1.7964444444444445, "percentage": 59.62, "elapsed_time": "1:08:13", "remaining_time": "0:46:12"} +{"current_steps": 128, "total_steps": 213, "loss": 0.48357316851615906, "lr": 4.2218893451814005e-06, "epoch": 1.8106666666666666, "percentage": 60.09, "elapsed_time": "1:08:43", "remaining_time": "0:45:38"} +{"current_steps": 129, "total_steps": 213, "loss": 0.49349913001060486, "lr": 4.140759581643386e-06, "epoch": 1.8248888888888888, "percentage": 60.56, "elapsed_time": "1:09:14", "remaining_time": "0:45:05"} +{"current_steps": 130, "total_steps": 213, "loss": 0.4969041347503662, "lr": 4.059862272583755e-06, "epoch": 1.8391111111111111, "percentage": 61.03, "elapsed_time": "1:09:44", "remaining_time": "0:44:31"} +{"current_steps": 131, "total_steps": 213, "loss": 0.47194504737854004, "lr": 3.979219303543942e-06, "epoch": 1.8533333333333335, "percentage": 61.5, "elapsed_time": "1:10:15", "remaining_time": "0:43:58"} +{"current_steps": 132, "total_steps": 213, "loss": 0.5020972490310669, "lr": 3.898852491257547e-06, "epoch": 1.8675555555555556, "percentage": 61.97, "elapsed_time": "1:10:45", "remaining_time": "0:43:25"} +{"current_steps": 133, "total_steps": 213, "loss": 0.5002884268760681, "lr": 3.818783577748138e-06, "epoch": 1.8817777777777778, "percentage": 62.44, "elapsed_time": "1:11:15", "remaining_time": "0:42:51"} +{"current_steps": 134, "total_steps": 213, "loss": 0.5007095336914062, "lr": 3.7390342244472886e-06, "epoch": 1.896, "percentage": 62.91, "elapsed_time": "1:11:44", "remaining_time": "0:42:17"} +{"current_steps": 135, "total_steps": 213, "loss": 0.4763813316822052, "lr": 3.659626006334395e-06, "epoch": 1.9102222222222223, "percentage": 63.38, "elapsed_time": "1:12:13", "remaining_time": "0:41:43"} +{"current_steps": 136, "total_steps": 213, "loss": 0.48733818531036377, "lr": 3.5805804060998926e-06, "epoch": 1.9244444444444444, "percentage": 63.85, "elapsed_time": "1:12:41", "remaining_time": "0:41:09"} +{"current_steps": 137, "total_steps": 213, "loss": 0.5118749141693115, "lr": 3.501918808333453e-06, "epoch": 1.9386666666666668, "percentage": 64.32, "elapsed_time": "1:13:11", "remaining_time": "0:40:36"} +{"current_steps": 138, "total_steps": 213, "loss": 0.5178316831588745, "lr": 3.4236624937386874e-06, "epoch": 1.952888888888889, "percentage": 64.79, "elapsed_time": "1:13:40", "remaining_time": "0:40:02"} +{"current_steps": 139, "total_steps": 213, "loss": 0.4853537678718567, "lr": 3.3458326333759927e-06, "epoch": 1.967111111111111, "percentage": 65.26, "elapsed_time": "1:14:11", "remaining_time": "0:39:29"} +{"current_steps": 140, "total_steps": 213, "loss": 0.49210506677627563, "lr": 3.268450282935026e-06, "epoch": 1.9813333333333332, "percentage": 65.73, "elapsed_time": "1:14:40", "remaining_time": "0:38:56"} +{"current_steps": 141, "total_steps": 213, "loss": 0.47841814160346985, "lr": 3.1915363770384223e-06, "epoch": 1.9955555555555555, "percentage": 66.2, "elapsed_time": "1:15:11", "remaining_time": "0:38:23"} +{"current_steps": 142, "total_steps": 213, "loss": 0.5034412145614624, "lr": 3.1151117235782346e-06, "epoch": 2.0, "percentage": 66.67, "elapsed_time": "1:15:21", "remaining_time": "0:37:40"} +{"current_steps": 143, "total_steps": 213, "loss": 0.4864930808544159, "lr": 3.0391969980866874e-06, "epoch": 2.014222222222222, "percentage": 67.14, "elapsed_time": "1:16:43", "remaining_time": "0:37:33"} +{"current_steps": 144, "total_steps": 213, "loss": 0.4642598032951355, "lr": 2.963812738142713e-06, "epoch": 2.0284444444444443, "percentage": 67.61, "elapsed_time": "1:17:14", "remaining_time": "0:37:00"} +{"current_steps": 145, "total_steps": 213, "loss": 0.48541027307510376, "lr": 2.8889793378158284e-06, "epoch": 2.042666666666667, "percentage": 68.08, "elapsed_time": "1:17:43", "remaining_time": "0:36:27"} +{"current_steps": 146, "total_steps": 213, "loss": 0.47221142053604126, "lr": 2.814717042148827e-06, "epoch": 2.056888888888889, "percentage": 68.54, "elapsed_time": "1:18:13", "remaining_time": "0:35:54"} +{"current_steps": 147, "total_steps": 213, "loss": 0.46124035120010376, "lr": 2.7410459416807856e-06, "epoch": 2.071111111111111, "percentage": 69.01, "elapsed_time": "1:18:44", "remaining_time": "0:35:21"} +{"current_steps": 148, "total_steps": 213, "loss": 0.4869857430458069, "lr": 2.6679859670118785e-06, "epoch": 2.0853333333333333, "percentage": 69.48, "elapsed_time": "1:19:13", "remaining_time": "0:34:47"} +{"current_steps": 149, "total_steps": 213, "loss": 0.45158863067626953, "lr": 2.5955568834114523e-06, "epoch": 2.0995555555555554, "percentage": 69.95, "elapsed_time": "1:19:43", "remaining_time": "0:34:14"} +{"current_steps": 150, "total_steps": 213, "loss": 0.4742748439311981, "lr": 2.523778285470835e-06, "epoch": 2.113777777777778, "percentage": 70.42, "elapsed_time": "1:20:14", "remaining_time": "0:33:42"} +{"current_steps": 150, "total_steps": 213, "eval_loss": 0.5196705460548401, "epoch": 2.113777777777778, "percentage": 70.42, "elapsed_time": "1:20:30", "remaining_time": "0:33:48"} +{"current_steps": 151, "total_steps": 213, "loss": 0.4526304602622986, "lr": 2.452669591802307e-06, "epoch": 2.128, "percentage": 70.89, "elapsed_time": "1:20:58", "remaining_time": "0:33:14"} +{"current_steps": 152, "total_steps": 213, "loss": 0.4651503562927246, "lr": 2.3822500397857016e-06, "epoch": 2.1422222222222222, "percentage": 71.36, "elapsed_time": "1:21:28", "remaining_time": "0:32:41"} +{"current_steps": 153, "total_steps": 213, "loss": 0.48414769768714905, "lr": 2.3125386803640188e-06, "epoch": 2.1564444444444444, "percentage": 71.83, "elapsed_time": "1:21:58", "remaining_time": "0:32:08"} +{"current_steps": 154, "total_steps": 213, "loss": 0.462248295545578, "lr": 2.243554372889479e-06, "epoch": 2.1706666666666665, "percentage": 72.3, "elapsed_time": "1:22:28", "remaining_time": "0:31:35"} +{"current_steps": 155, "total_steps": 213, "loss": 0.4559134244918823, "lr": 2.1753157800214107e-06, "epoch": 2.1848888888888887, "percentage": 72.77, "elapsed_time": "1:22:58", "remaining_time": "0:31:03"} +{"current_steps": 156, "total_steps": 213, "loss": 0.48357266187667847, "lr": 2.1078413626773547e-06, "epoch": 2.1991111111111112, "percentage": 73.24, "elapsed_time": "1:23:28", "remaining_time": "0:30:30"} +{"current_steps": 157, "total_steps": 213, "loss": 0.4366765022277832, "lr": 2.0411493750387423e-06, "epoch": 2.2133333333333334, "percentage": 73.71, "elapsed_time": "1:23:57", "remaining_time": "0:29:56"} +{"current_steps": 158, "total_steps": 213, "loss": 0.4868413209915161, "lr": 1.9752578596124955e-06, "epoch": 2.2275555555555555, "percentage": 74.18, "elapsed_time": "1:24:26", "remaining_time": "0:29:23"} +{"current_steps": 159, "total_steps": 213, "loss": 0.46049410104751587, "lr": 1.9101846423499113e-06, "epoch": 2.2417777777777776, "percentage": 74.65, "elapsed_time": "1:24:55", "remaining_time": "0:28:50"} +{"current_steps": 160, "total_steps": 213, "loss": 0.4605860412120819, "lr": 1.8459473278241125e-06, "epoch": 2.2560000000000002, "percentage": 75.12, "elapsed_time": "1:25:25", "remaining_time": "0:28:17"} +{"current_steps": 161, "total_steps": 213, "loss": 0.46191370487213135, "lr": 1.7825632944674016e-06, "epoch": 2.2702222222222224, "percentage": 75.59, "elapsed_time": "1:25:56", "remaining_time": "0:27:45"} +{"current_steps": 162, "total_steps": 213, "loss": 0.4780347943305969, "lr": 1.7200496898697832e-06, "epoch": 2.2844444444444445, "percentage": 76.06, "elapsed_time": "1:26:25", "remaining_time": "0:27:12"} +{"current_steps": 163, "total_steps": 213, "loss": 0.4648103713989258, "lr": 1.6584234261399535e-06, "epoch": 2.2986666666666666, "percentage": 76.53, "elapsed_time": "1:26:56", "remaining_time": "0:26:40"} +{"current_steps": 164, "total_steps": 213, "loss": 0.46155521273612976, "lr": 1.5977011753299726e-06, "epoch": 2.3128888888888888, "percentage": 77.0, "elapsed_time": "1:27:24", "remaining_time": "0:26:07"} +{"current_steps": 165, "total_steps": 213, "loss": 0.49238717555999756, "lr": 1.5378993649249053e-06, "epoch": 2.327111111111111, "percentage": 77.46, "elapsed_time": "1:27:52", "remaining_time": "0:25:33"} +{"current_steps": 166, "total_steps": 213, "loss": 0.46214908361434937, "lr": 1.4790341733986085e-06, "epoch": 2.3413333333333335, "percentage": 77.93, "elapsed_time": "1:28:22", "remaining_time": "0:25:01"} +{"current_steps": 167, "total_steps": 213, "loss": 0.44751304388046265, "lr": 1.4211215258368866e-06, "epoch": 2.3555555555555556, "percentage": 78.4, "elapsed_time": "1:28:53", "remaining_time": "0:24:29"} +{"current_steps": 168, "total_steps": 213, "loss": 0.4523189067840576, "lr": 1.3641770896292083e-06, "epoch": 2.3697777777777778, "percentage": 78.87, "elapsed_time": "1:29:24", "remaining_time": "0:23:56"} +{"current_steps": 169, "total_steps": 213, "loss": 0.46704721450805664, "lr": 1.3082162702301276e-06, "epoch": 2.384, "percentage": 79.34, "elapsed_time": "1:29:54", "remaining_time": "0:23:24"} +{"current_steps": 170, "total_steps": 213, "loss": 0.47263163328170776, "lr": 1.2532542069915722e-06, "epoch": 2.398222222222222, "percentage": 79.81, "elapsed_time": "1:30:23", "remaining_time": "0:22:51"} +{"current_steps": 171, "total_steps": 213, "loss": 0.4592151641845703, "lr": 1.1993057690671174e-06, "epoch": 2.4124444444444446, "percentage": 80.28, "elapsed_time": "1:30:53", "remaining_time": "0:22:19"} +{"current_steps": 172, "total_steps": 213, "loss": 0.4800827205181122, "lr": 1.1463855513893695e-06, "epoch": 2.4266666666666667, "percentage": 80.75, "elapsed_time": "1:31:24", "remaining_time": "0:21:47"} +{"current_steps": 173, "total_steps": 213, "loss": 0.4688478112220764, "lr": 1.0945078707215224e-06, "epoch": 2.440888888888889, "percentage": 81.22, "elapsed_time": "1:31:53", "remaining_time": "0:21:14"} +{"current_steps": 174, "total_steps": 213, "loss": 0.46817073225975037, "lr": 1.0436867617841768e-06, "epoch": 2.455111111111111, "percentage": 81.69, "elapsed_time": "1:32:21", "remaining_time": "0:20:42"} +{"current_steps": 175, "total_steps": 213, "loss": 0.4763898253440857, "lr": 9.939359734584552e-07, "epoch": 2.469333333333333, "percentage": 82.16, "elapsed_time": "1:32:51", "remaining_time": "0:20:09"} +{"current_steps": 176, "total_steps": 213, "loss": 0.480793833732605, "lr": 9.452689650664515e-07, "epoch": 2.4835555555555557, "percentage": 82.63, "elapsed_time": "1:33:21", "remaining_time": "0:19:37"} +{"current_steps": 177, "total_steps": 213, "loss": 0.4694904088973999, "lr": 8.976989027300265e-07, "epoch": 2.497777777777778, "percentage": 83.1, "elapsed_time": "1:33:51", "remaining_time": "0:19:05"} +{"current_steps": 178, "total_steps": 213, "loss": 0.4877663254737854, "lr": 8.512386558088919e-07, "epoch": 2.512, "percentage": 83.57, "elapsed_time": "1:34:20", "remaining_time": "0:18:33"} +{"current_steps": 179, "total_steps": 213, "loss": 0.46579453349113464, "lr": 8.059007934190194e-07, "epoch": 2.526222222222222, "percentage": 84.04, "elapsed_time": "1:34:51", "remaining_time": "0:18:00"} +{"current_steps": 180, "total_steps": 213, "loss": 0.4954630434513092, "lr": 7.61697581032243e-07, "epoch": 2.5404444444444443, "percentage": 84.51, "elapsed_time": "1:35:20", "remaining_time": "0:17:28"} +{"current_steps": 181, "total_steps": 213, "loss": 0.48210158944129944, "lr": 7.186409771580355e-07, "epoch": 2.554666666666667, "percentage": 84.98, "elapsed_time": "1:35:51", "remaining_time": "0:16:56"} +{"current_steps": 182, "total_steps": 213, "loss": 0.4673936069011688, "lr": 6.76742630108298e-07, "epoch": 2.568888888888889, "percentage": 85.45, "elapsed_time": "1:36:21", "remaining_time": "0:16:24"} +{"current_steps": 183, "total_steps": 213, "loss": 0.46786054968833923, "lr": 6.360138748461015e-07, "epoch": 2.583111111111111, "percentage": 85.92, "elapsed_time": "1:36:52", "remaining_time": "0:15:52"} +{"current_steps": 184, "total_steps": 213, "loss": 0.45768898725509644, "lr": 5.964657299191712e-07, "epoch": 2.5973333333333333, "percentage": 86.38, "elapsed_time": "1:37:22", "remaining_time": "0:15:20"} +{"current_steps": 185, "total_steps": 213, "loss": 0.468585342168808, "lr": 5.581088944789953e-07, "epoch": 2.6115555555555554, "percentage": 86.85, "elapsed_time": "1:37:53", "remaining_time": "0:14:48"} +{"current_steps": 186, "total_steps": 213, "loss": 0.4843568205833435, "lr": 5.20953745386329e-07, "epoch": 2.6257777777777775, "percentage": 87.32, "elapsed_time": "1:38:23", "remaining_time": "0:14:16"} +{"current_steps": 187, "total_steps": 213, "loss": 0.4747229218482971, "lr": 4.850103344038853e-07, "epoch": 2.64, "percentage": 87.79, "elapsed_time": "1:38:52", "remaining_time": "0:13:44"} +{"current_steps": 188, "total_steps": 213, "loss": 0.44664108753204346, "lr": 4.502883854769935e-07, "epoch": 2.6542222222222223, "percentage": 88.26, "elapsed_time": "1:39:22", "remaining_time": "0:13:12"} +{"current_steps": 189, "total_steps": 213, "loss": 0.47903192043304443, "lr": 4.167972921029262e-07, "epoch": 2.6684444444444444, "percentage": 88.73, "elapsed_time": "1:39:52", "remaining_time": "0:12:40"} +{"current_steps": 190, "total_steps": 213, "loss": 0.4844057559967041, "lr": 3.8454611478963235e-07, "epoch": 2.6826666666666665, "percentage": 89.2, "elapsed_time": "1:40:21", "remaining_time": "0:12:08"} +{"current_steps": 191, "total_steps": 213, "loss": 0.4564388394355774, "lr": 3.535435786045538e-07, "epoch": 2.696888888888889, "percentage": 89.67, "elapsed_time": "1:40:50", "remaining_time": "0:11:36"} +{"current_steps": 192, "total_steps": 213, "loss": 0.47008293867111206, "lr": 3.237980708141919e-07, "epoch": 2.7111111111111112, "percentage": 90.14, "elapsed_time": "1:41:20", "remaining_time": "0:11:05"} +{"current_steps": 193, "total_steps": 213, "loss": 0.479957640171051, "lr": 2.9531763861505967e-07, "epoch": 2.7253333333333334, "percentage": 90.61, "elapsed_time": "1:41:49", "remaining_time": "0:10:33"} +{"current_steps": 194, "total_steps": 213, "loss": 0.44965675473213196, "lr": 2.681099869566328e-07, "epoch": 2.7395555555555555, "percentage": 91.08, "elapsed_time": "1:42:18", "remaining_time": "0:10:01"} +{"current_steps": 195, "total_steps": 213, "loss": 0.4747314453125, "lr": 2.4218247645689306e-07, "epoch": 2.7537777777777777, "percentage": 91.55, "elapsed_time": "1:42:48", "remaining_time": "0:09:29"} +{"current_steps": 196, "total_steps": 213, "loss": 0.45799821615219116, "lr": 2.175421214110235e-07, "epoch": 2.768, "percentage": 92.02, "elapsed_time": "1:43:18", "remaining_time": "0:08:57"} +{"current_steps": 197, "total_steps": 213, "loss": 0.47467684745788574, "lr": 1.941955878938029e-07, "epoch": 2.7822222222222224, "percentage": 92.49, "elapsed_time": "1:43:48", "remaining_time": "0:08:25"} +{"current_steps": 198, "total_steps": 213, "loss": 0.4817090332508087, "lr": 1.721491919561913e-07, "epoch": 2.7964444444444445, "percentage": 92.96, "elapsed_time": "1:44:17", "remaining_time": "0:07:54"} +{"current_steps": 199, "total_steps": 213, "loss": 0.4649164080619812, "lr": 1.514088979166256e-07, "epoch": 2.8106666666666666, "percentage": 93.43, "elapsed_time": "1:44:47", "remaining_time": "0:07:22"} +{"current_steps": 200, "total_steps": 213, "loss": 0.4765157699584961, "lr": 1.3198031674745814e-07, "epoch": 2.824888888888889, "percentage": 93.9, "elapsed_time": "1:45:17", "remaining_time": "0:06:50"} +{"current_steps": 200, "total_steps": 213, "eval_loss": 0.5171427726745605, "epoch": 2.824888888888889, "percentage": 93.9, "elapsed_time": "1:45:27", "remaining_time": "0:06:51"} +{"current_steps": 201, "total_steps": 213, "loss": 0.46831822395324707, "lr": 1.138687045569975e-07, "epoch": 2.8391111111111114, "percentage": 94.37, "elapsed_time": "1:45:58", "remaining_time": "0:06:19"} +{"current_steps": 202, "total_steps": 213, "loss": 0.47398146986961365, "lr": 9.70789611675449e-08, "epoch": 2.8533333333333335, "percentage": 94.84, "elapsed_time": "1:46:28", "remaining_time": "0:05:47"} +{"current_steps": 203, "total_steps": 213, "loss": 0.45744937658309937, "lr": 8.161562878982399e-08, "epoch": 2.8675555555555556, "percentage": 95.31, "elapsed_time": "1:46:57", "remaining_time": "0:05:16"} +{"current_steps": 204, "total_steps": 213, "loss": 0.48529934883117676, "lr": 6.74828907941516e-08, "epoch": 2.8817777777777778, "percentage": 95.77, "elapsed_time": "1:47:26", "remaining_time": "0:04:44"} +{"current_steps": 205, "total_steps": 213, "loss": 0.4858051836490631, "lr": 5.468457057869358e-08, "epoch": 2.896, "percentage": 96.24, "elapsed_time": "1:47:56", "remaining_time": "0:04:12"} +{"current_steps": 206, "total_steps": 213, "loss": 0.4590516984462738, "lr": 4.322413053509944e-08, "epoch": 2.910222222222222, "percentage": 96.71, "elapsed_time": "1:48:28", "remaining_time": "0:03:41"} +{"current_steps": 207, "total_steps": 213, "loss": 0.45788976550102234, "lr": 3.31046711118066e-08, "epoch": 2.924444444444444, "percentage": 97.18, "elapsed_time": "1:48:57", "remaining_time": "0:03:09"} +{"current_steps": 208, "total_steps": 213, "loss": 0.4702548086643219, "lr": 2.4328929975260262e-08, "epoch": 2.9386666666666668, "percentage": 97.65, "elapsed_time": "1:49:27", "remaining_time": "0:02:37"} +{"current_steps": 209, "total_steps": 213, "loss": 0.4560105800628662, "lr": 1.6899281269279756e-08, "epoch": 2.952888888888889, "percentage": 98.12, "elapsed_time": "1:49:56", "remaining_time": "0:02:06"} +{"current_steps": 210, "total_steps": 213, "loss": 0.46400371193885803, "lr": 1.0817734972768946e-08, "epoch": 2.967111111111111, "percentage": 98.59, "elapsed_time": "1:50:25", "remaining_time": "0:01:34"} +{"current_steps": 211, "total_steps": 213, "loss": 0.48368915915489197, "lr": 6.085936355947897e-09, "epoch": 2.981333333333333, "percentage": 99.06, "elapsed_time": "1:50:55", "remaining_time": "0:01:03"} +{"current_steps": 212, "total_steps": 213, "loss": 0.45437300205230713, "lr": 2.7051655352494654e-09, "epoch": 2.9955555555555557, "percentage": 99.53, "elapsed_time": "1:51:24", "remaining_time": "0:00:31"} +{"current_steps": 213, "total_steps": 213, "loss": 0.5508654713630676, "lr": 6.763371270035457e-10, "epoch": 3.0, "percentage": 100.0, "elapsed_time": "1:51:34", "remaining_time": "0:00:00"} +{"current_steps": 213, "total_steps": 213, "epoch": 3.0, "percentage": 100.0, "elapsed_time": "1:52:32", "remaining_time": "0:00:00"} diff --git a/qwen2.5-1.5b-instruct/trainer_state.json b/qwen2.5-1.5b-instruct/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..cf8c5e592e055ae522f9aff5a5a9b5ec5a0a9a6a --- /dev/null +++ b/qwen2.5-1.5b-instruct/trainer_state.json @@ -0,0 +1,1566 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 50, + "global_step": 213, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.014222222222222223, + "grad_norm": 0.03586147725582123, + "learning_rate": 0.0, + "loss": 0.9098724722862244, + "step": 1 + }, + { + "epoch": 0.028444444444444446, + "grad_norm": 0.03577122464776039, + "learning_rate": 4.5454545454545457e-07, + "loss": 0.9084522724151611, + "step": 2 + }, + { + "epoch": 0.042666666666666665, + "grad_norm": 0.031653374433517456, + "learning_rate": 9.090909090909091e-07, + "loss": 0.8834755420684814, + "step": 3 + }, + { + "epoch": 0.05688888888888889, + "grad_norm": 0.025831686332821846, + "learning_rate": 1.3636363636363636e-06, + "loss": 0.871230959892273, + "step": 4 + }, + { + "epoch": 0.07111111111111111, + "grad_norm": 0.019876576960086823, + "learning_rate": 1.8181818181818183e-06, + "loss": 0.7917206287384033, + "step": 5 + }, + { + "epoch": 0.08533333333333333, + "grad_norm": 0.018677273765206337, + "learning_rate": 2.2727272727272728e-06, + "loss": 0.8436126708984375, + "step": 6 + }, + { + "epoch": 0.09955555555555555, + "grad_norm": 0.017528358846902847, + "learning_rate": 2.7272727272727272e-06, + "loss": 0.8075190782546997, + "step": 7 + }, + { + "epoch": 0.11377777777777778, + "grad_norm": 0.01399513240903616, + "learning_rate": 3.181818181818182e-06, + "loss": 0.7754716873168945, + "step": 8 + }, + { + "epoch": 0.128, + "grad_norm": 0.012644859962165356, + "learning_rate": 3.6363636363636366e-06, + "loss": 0.7459083199501038, + "step": 9 + }, + { + "epoch": 0.14222222222222222, + "grad_norm": 0.01209743320941925, + "learning_rate": 4.0909090909090915e-06, + "loss": 0.7365868091583252, + "step": 10 + }, + { + "epoch": 0.15644444444444444, + "grad_norm": 0.012133832089602947, + "learning_rate": 4.5454545454545455e-06, + "loss": 0.7119501829147339, + "step": 11 + }, + { + "epoch": 0.17066666666666666, + "grad_norm": 0.010017997585237026, + "learning_rate": 5e-06, + "loss": 0.6901087760925293, + "step": 12 + }, + { + "epoch": 0.18488888888888888, + "grad_norm": 0.009233366698026657, + "learning_rate": 5.4545454545454545e-06, + "loss": 0.6611638069152832, + "step": 13 + }, + { + "epoch": 0.1991111111111111, + "grad_norm": 0.00782010331749916, + "learning_rate": 5.90909090909091e-06, + "loss": 0.6715224981307983, + "step": 14 + }, + { + "epoch": 0.21333333333333335, + "grad_norm": 0.007341764867305756, + "learning_rate": 6.363636363636364e-06, + "loss": 0.6277110576629639, + "step": 15 + }, + { + "epoch": 0.22755555555555557, + "grad_norm": 0.00713470159098506, + "learning_rate": 6.818181818181818e-06, + "loss": 0.6689565181732178, + "step": 16 + }, + { + "epoch": 0.24177777777777779, + "grad_norm": 0.0067572277039289474, + "learning_rate": 7.272727272727273e-06, + "loss": 0.6363875865936279, + "step": 17 + }, + { + "epoch": 0.256, + "grad_norm": 0.006228511221706867, + "learning_rate": 7.727272727272727e-06, + "loss": 0.6257224678993225, + "step": 18 + }, + { + "epoch": 0.2702222222222222, + "grad_norm": 0.006048806011676788, + "learning_rate": 8.181818181818183e-06, + "loss": 0.6355900764465332, + "step": 19 + }, + { + "epoch": 0.28444444444444444, + "grad_norm": 0.005988306365907192, + "learning_rate": 8.636363636363637e-06, + "loss": 0.6338369846343994, + "step": 20 + }, + { + "epoch": 0.2986666666666667, + "grad_norm": 0.005441761109977961, + "learning_rate": 9.090909090909091e-06, + "loss": 0.6133573055267334, + "step": 21 + }, + { + "epoch": 0.3128888888888889, + "grad_norm": 0.005174445919692516, + "learning_rate": 9.545454545454547e-06, + "loss": 0.6042410135269165, + "step": 22 + }, + { + "epoch": 0.32711111111111113, + "grad_norm": 0.005199746694415808, + "learning_rate": 1e-05, + "loss": 0.602219820022583, + "step": 23 + }, + { + "epoch": 0.3413333333333333, + "grad_norm": 0.005485028959810734, + "learning_rate": 9.999323662872998e-06, + "loss": 0.6062123775482178, + "step": 24 + }, + { + "epoch": 0.35555555555555557, + "grad_norm": 0.004896003287285566, + "learning_rate": 9.99729483446475e-06, + "loss": 0.6077945232391357, + "step": 25 + }, + { + "epoch": 0.36977777777777776, + "grad_norm": 0.00495151337236166, + "learning_rate": 9.993914063644053e-06, + "loss": 0.5923135280609131, + "step": 26 + }, + { + "epoch": 0.384, + "grad_norm": 0.004744194447994232, + "learning_rate": 9.989182265027232e-06, + "loss": 0.5860621929168701, + "step": 27 + }, + { + "epoch": 0.3982222222222222, + "grad_norm": 0.004471767693758011, + "learning_rate": 9.98310071873072e-06, + "loss": 0.5823097229003906, + "step": 28 + }, + { + "epoch": 0.41244444444444445, + "grad_norm": 0.004788320977240801, + "learning_rate": 9.975671070024741e-06, + "loss": 0.5872600078582764, + "step": 29 + }, + { + "epoch": 0.4266666666666667, + "grad_norm": 0.0044632842764258385, + "learning_rate": 9.966895328888195e-06, + "loss": 0.6070278286933899, + "step": 30 + }, + { + "epoch": 0.4408888888888889, + "grad_norm": 0.004317905753850937, + "learning_rate": 9.956775869464901e-06, + "loss": 0.5780316591262817, + "step": 31 + }, + { + "epoch": 0.45511111111111113, + "grad_norm": 0.004291852004826069, + "learning_rate": 9.945315429421307e-06, + "loss": 0.5697590112686157, + "step": 32 + }, + { + "epoch": 0.4693333333333333, + "grad_norm": 0.003922830801457167, + "learning_rate": 9.932517109205849e-06, + "loss": 0.5747107267379761, + "step": 33 + }, + { + "epoch": 0.48355555555555557, + "grad_norm": 0.0041889771819114685, + "learning_rate": 9.918384371210178e-06, + "loss": 0.6021177172660828, + "step": 34 + }, + { + "epoch": 0.49777777777777776, + "grad_norm": 0.004147357773035765, + "learning_rate": 9.902921038832456e-06, + "loss": 0.5805741548538208, + "step": 35 + }, + { + "epoch": 0.512, + "grad_norm": 0.0040724254213273525, + "learning_rate": 9.886131295443003e-06, + "loss": 0.5483200550079346, + "step": 36 + }, + { + "epoch": 0.5262222222222223, + "grad_norm": 0.004085644613951445, + "learning_rate": 9.868019683252543e-06, + "loss": 0.557392954826355, + "step": 37 + }, + { + "epoch": 0.5404444444444444, + "grad_norm": 0.004289102274924517, + "learning_rate": 9.848591102083375e-06, + "loss": 0.590857207775116, + "step": 38 + }, + { + "epoch": 0.5546666666666666, + "grad_norm": 0.0040345704182982445, + "learning_rate": 9.82785080804381e-06, + "loss": 0.5750541090965271, + "step": 39 + }, + { + "epoch": 0.5688888888888889, + "grad_norm": 0.004034748766571283, + "learning_rate": 9.805804412106197e-06, + "loss": 0.5949826240539551, + "step": 40 + }, + { + "epoch": 0.5831111111111111, + "grad_norm": 0.003911909181624651, + "learning_rate": 9.782457878588977e-06, + "loss": 0.5623177289962769, + "step": 41 + }, + { + "epoch": 0.5973333333333334, + "grad_norm": 0.0037001846358180046, + "learning_rate": 9.75781752354311e-06, + "loss": 0.5465081930160522, + "step": 42 + }, + { + "epoch": 0.6115555555555555, + "grad_norm": 0.0037095139268785715, + "learning_rate": 9.731890013043367e-06, + "loss": 0.5752100944519043, + "step": 43 + }, + { + "epoch": 0.6257777777777778, + "grad_norm": 0.003655617358162999, + "learning_rate": 9.704682361384941e-06, + "loss": 0.5452020168304443, + "step": 44 + }, + { + "epoch": 0.64, + "grad_norm": 0.003784338477998972, + "learning_rate": 9.676201929185809e-06, + "loss": 0.5616332292556763, + "step": 45 + }, + { + "epoch": 0.6542222222222223, + "grad_norm": 0.003718078136444092, + "learning_rate": 9.646456421395447e-06, + "loss": 0.5558297634124756, + "step": 46 + }, + { + "epoch": 0.6684444444444444, + "grad_norm": 0.00395169947296381, + "learning_rate": 9.615453885210368e-06, + "loss": 0.5600595474243164, + "step": 47 + }, + { + "epoch": 0.6826666666666666, + "grad_norm": 0.003555440343916416, + "learning_rate": 9.583202707897075e-06, + "loss": 0.5372895002365112, + "step": 48 + }, + { + "epoch": 0.6968888888888889, + "grad_norm": 0.0037386869080364704, + "learning_rate": 9.549711614523007e-06, + "loss": 0.5322733521461487, + "step": 49 + }, + { + "epoch": 0.7111111111111111, + "grad_norm": 0.003739988896995783, + "learning_rate": 9.514989665596114e-06, + "loss": 0.5380419492721558, + "step": 50 + }, + { + "epoch": 0.7111111111111111, + "eval_loss": 0.5587145090103149, + "eval_runtime": 8.7676, + "eval_samples_per_second": 22.811, + "eval_steps_per_second": 11.406, + "step": 50 + }, + { + "epoch": 0.7253333333333334, + "grad_norm": 0.0037539806216955185, + "learning_rate": 9.479046254613673e-06, + "loss": 0.5472225546836853, + "step": 51 + }, + { + "epoch": 0.7395555555555555, + "grad_norm": 0.003411181503906846, + "learning_rate": 9.441891105521005e-06, + "loss": 0.5134032964706421, + "step": 52 + }, + { + "epoch": 0.7537777777777778, + "grad_norm": 0.0037538670003414154, + "learning_rate": 9.40353427008083e-06, + "loss": 0.5361013412475586, + "step": 53 + }, + { + "epoch": 0.768, + "grad_norm": 0.0036495670210570097, + "learning_rate": 9.3639861251539e-06, + "loss": 0.5299779772758484, + "step": 54 + }, + { + "epoch": 0.7822222222222223, + "grad_norm": 0.0037342547439038754, + "learning_rate": 9.323257369891702e-06, + "loss": 0.5607945919036865, + "step": 55 + }, + { + "epoch": 0.7964444444444444, + "grad_norm": 0.003572584595531225, + "learning_rate": 9.281359022841966e-06, + "loss": 0.5528460741043091, + "step": 56 + }, + { + "epoch": 0.8106666666666666, + "grad_norm": 0.003770101349800825, + "learning_rate": 9.238302418967757e-06, + "loss": 0.5440288186073303, + "step": 57 + }, + { + "epoch": 0.8248888888888889, + "grad_norm": 0.0036679303739219904, + "learning_rate": 9.194099206580981e-06, + "loss": 0.5564252138137817, + "step": 58 + }, + { + "epoch": 0.8391111111111111, + "grad_norm": 0.003834612900391221, + "learning_rate": 9.14876134419111e-06, + "loss": 0.5473330020904541, + "step": 59 + }, + { + "epoch": 0.8533333333333334, + "grad_norm": 0.003683249931782484, + "learning_rate": 9.102301097269974e-06, + "loss": 0.5463743209838867, + "step": 60 + }, + { + "epoch": 0.8675555555555555, + "grad_norm": 0.0036777963396161795, + "learning_rate": 9.05473103493355e-06, + "loss": 0.524878740310669, + "step": 61 + }, + { + "epoch": 0.8817777777777778, + "grad_norm": 0.0038230482023209333, + "learning_rate": 9.006064026541549e-06, + "loss": 0.5563522577285767, + "step": 62 + }, + { + "epoch": 0.896, + "grad_norm": 0.003645201213657856, + "learning_rate": 8.956313238215824e-06, + "loss": 0.5059154033660889, + "step": 63 + }, + { + "epoch": 0.9102222222222223, + "grad_norm": 0.0035886459518224, + "learning_rate": 8.905492129278478e-06, + "loss": 0.5312881469726562, + "step": 64 + }, + { + "epoch": 0.9244444444444444, + "grad_norm": 0.003721872577443719, + "learning_rate": 8.85361444861063e-06, + "loss": 0.5358959436416626, + "step": 65 + }, + { + "epoch": 0.9386666666666666, + "grad_norm": 0.003683999879285693, + "learning_rate": 8.800694230932885e-06, + "loss": 0.5442053079605103, + "step": 66 + }, + { + "epoch": 0.9528888888888889, + "grad_norm": 0.0036851842887699604, + "learning_rate": 8.74674579300843e-06, + "loss": 0.5263723731040955, + "step": 67 + }, + { + "epoch": 0.9671111111111111, + "grad_norm": 0.003754258854314685, + "learning_rate": 8.691783729769874e-06, + "loss": 0.5200538039207458, + "step": 68 + }, + { + "epoch": 0.9813333333333333, + "grad_norm": 0.0038373274728655815, + "learning_rate": 8.635822910370793e-06, + "loss": 0.5452108383178711, + "step": 69 + }, + { + "epoch": 0.9955555555555555, + "grad_norm": 0.0039480882696807384, + "learning_rate": 8.578878474163115e-06, + "loss": 0.5066030621528625, + "step": 70 + }, + { + "epoch": 1.0, + "grad_norm": 0.005897017661482096, + "learning_rate": 8.520965826601394e-06, + "loss": 0.5380499958992004, + "step": 71 + }, + { + "epoch": 1.0142222222222221, + "grad_norm": 0.0037728557363152504, + "learning_rate": 8.462100635075097e-06, + "loss": 0.5328541398048401, + "step": 72 + }, + { + "epoch": 1.0284444444444445, + "grad_norm": 0.003953218925744295, + "learning_rate": 8.40229882467003e-06, + "loss": 0.5096005201339722, + "step": 73 + }, + { + "epoch": 1.0426666666666666, + "grad_norm": 0.0036587677896022797, + "learning_rate": 8.341576573860049e-06, + "loss": 0.4889724850654602, + "step": 74 + }, + { + "epoch": 1.056888888888889, + "grad_norm": 0.0038338187150657177, + "learning_rate": 8.279950310130218e-06, + "loss": 0.5157173871994019, + "step": 75 + }, + { + "epoch": 1.0711111111111111, + "grad_norm": 0.003540106350556016, + "learning_rate": 8.2174367055326e-06, + "loss": 0.49691659212112427, + "step": 76 + }, + { + "epoch": 1.0853333333333333, + "grad_norm": 0.0037315969821065664, + "learning_rate": 8.154052672175888e-06, + "loss": 0.4873769283294678, + "step": 77 + }, + { + "epoch": 1.0995555555555556, + "grad_norm": 0.003963674418628216, + "learning_rate": 8.08981535765009e-06, + "loss": 0.5266215801239014, + "step": 78 + }, + { + "epoch": 1.1137777777777778, + "grad_norm": 0.0037685995921492577, + "learning_rate": 8.024742140387506e-06, + "loss": 0.5039150714874268, + "step": 79 + }, + { + "epoch": 1.1280000000000001, + "grad_norm": 0.003613364417105913, + "learning_rate": 7.95885062496126e-06, + "loss": 0.4896891117095947, + "step": 80 + }, + { + "epoch": 1.1422222222222222, + "grad_norm": 0.0037347134202718735, + "learning_rate": 7.892158637322647e-06, + "loss": 0.45997822284698486, + "step": 81 + }, + { + "epoch": 1.1564444444444444, + "grad_norm": 0.0037277438677847385, + "learning_rate": 7.824684219978591e-06, + "loss": 0.4917220175266266, + "step": 82 + }, + { + "epoch": 1.1706666666666667, + "grad_norm": 0.0034888957161456347, + "learning_rate": 7.756445627110523e-06, + "loss": 0.5024806261062622, + "step": 83 + }, + { + "epoch": 1.1848888888888889, + "grad_norm": 0.004098390229046345, + "learning_rate": 7.68746131963598e-06, + "loss": 0.5110447406768799, + "step": 84 + }, + { + "epoch": 1.199111111111111, + "grad_norm": 0.0036922753788530827, + "learning_rate": 7.6177499602143e-06, + "loss": 0.5031975507736206, + "step": 85 + }, + { + "epoch": 1.2133333333333334, + "grad_norm": 0.0035749904345721006, + "learning_rate": 7.547330408197695e-06, + "loss": 0.4899296164512634, + "step": 86 + }, + { + "epoch": 1.2275555555555555, + "grad_norm": 0.003764625173062086, + "learning_rate": 7.476221714529167e-06, + "loss": 0.5040560960769653, + "step": 87 + }, + { + "epoch": 1.2417777777777779, + "grad_norm": 0.003831087611615658, + "learning_rate": 7.404443116588548e-06, + "loss": 0.5307536125183105, + "step": 88 + }, + { + "epoch": 1.256, + "grad_norm": 0.003740105777978897, + "learning_rate": 7.332014032988123e-06, + "loss": 0.5025057792663574, + "step": 89 + }, + { + "epoch": 1.2702222222222221, + "grad_norm": 0.0038286929484456778, + "learning_rate": 7.2589540583192165e-06, + "loss": 0.5043497085571289, + "step": 90 + }, + { + "epoch": 1.2844444444444445, + "grad_norm": 0.003593199420720339, + "learning_rate": 7.185282957851175e-06, + "loss": 0.4883122444152832, + "step": 91 + }, + { + "epoch": 1.2986666666666666, + "grad_norm": 0.003721701679751277, + "learning_rate": 7.111020662184174e-06, + "loss": 0.5055882930755615, + "step": 92 + }, + { + "epoch": 1.3128888888888888, + "grad_norm": 0.0036744684912264347, + "learning_rate": 7.036187261857289e-06, + "loss": 0.4962804615497589, + "step": 93 + }, + { + "epoch": 1.3271111111111111, + "grad_norm": 0.004013169556856155, + "learning_rate": 6.960803001913315e-06, + "loss": 0.524259090423584, + "step": 94 + }, + { + "epoch": 1.3413333333333333, + "grad_norm": 0.003746299771592021, + "learning_rate": 6.884888276421766e-06, + "loss": 0.5073269009590149, + "step": 95 + }, + { + "epoch": 1.3555555555555556, + "grad_norm": 0.0036679194308817387, + "learning_rate": 6.8084636229615786e-06, + "loss": 0.4886914789676666, + "step": 96 + }, + { + "epoch": 1.3697777777777778, + "grad_norm": 0.00375540042296052, + "learning_rate": 6.731549717064975e-06, + "loss": 0.4920746088027954, + "step": 97 + }, + { + "epoch": 1.384, + "grad_norm": 0.003901587100699544, + "learning_rate": 6.654167366624009e-06, + "loss": 0.503207266330719, + "step": 98 + }, + { + "epoch": 1.3982222222222223, + "grad_norm": 0.003911704756319523, + "learning_rate": 6.576337506261314e-06, + "loss": 0.5012996196746826, + "step": 99 + }, + { + "epoch": 1.4124444444444444, + "grad_norm": 0.0035876361653208733, + "learning_rate": 6.498081191666549e-06, + "loss": 0.5059630274772644, + "step": 100 + }, + { + "epoch": 1.4124444444444444, + "eval_loss": 0.5291631817817688, + "eval_runtime": 10.3556, + "eval_samples_per_second": 19.313, + "eval_steps_per_second": 9.657, + "step": 100 + }, + { + "epoch": 1.4266666666666667, + "grad_norm": 0.003729900810867548, + "learning_rate": 6.419419593900109e-06, + "loss": 0.4896056056022644, + "step": 101 + }, + { + "epoch": 1.4408888888888889, + "grad_norm": 0.003648703685030341, + "learning_rate": 6.340373993665607e-06, + "loss": 0.48181772232055664, + "step": 102 + }, + { + "epoch": 1.455111111111111, + "grad_norm": 0.0035914399195462465, + "learning_rate": 6.2609657755527135e-06, + "loss": 0.4775124192237854, + "step": 103 + }, + { + "epoch": 1.4693333333333334, + "grad_norm": 0.0035880866926163435, + "learning_rate": 6.1812164222518626e-06, + "loss": 0.5167958736419678, + "step": 104 + }, + { + "epoch": 1.4835555555555555, + "grad_norm": 0.003788268193602562, + "learning_rate": 6.101147508742456e-06, + "loss": 0.4946171045303345, + "step": 105 + }, + { + "epoch": 1.4977777777777779, + "grad_norm": 0.0036002870183438063, + "learning_rate": 6.020780696456059e-06, + "loss": 0.4849076569080353, + "step": 106 + }, + { + "epoch": 1.512, + "grad_norm": 0.0036388293374329805, + "learning_rate": 5.940137727416247e-06, + "loss": 0.4958712160587311, + "step": 107 + }, + { + "epoch": 1.5262222222222221, + "grad_norm": 0.0034927036613225937, + "learning_rate": 5.859240418356614e-06, + "loss": 0.4822971820831299, + "step": 108 + }, + { + "epoch": 1.5404444444444443, + "grad_norm": 0.0038439796771854162, + "learning_rate": 5.778110654818602e-06, + "loss": 0.5280419588088989, + "step": 109 + }, + { + "epoch": 1.5546666666666666, + "grad_norm": 0.003715698141604662, + "learning_rate": 5.696770385230679e-06, + "loss": 0.4952104687690735, + "step": 110 + }, + { + "epoch": 1.568888888888889, + "grad_norm": 0.0036385883577167988, + "learning_rate": 5.615241614970546e-06, + "loss": 0.4887908101081848, + "step": 111 + }, + { + "epoch": 1.5831111111111111, + "grad_norm": 0.0036841081455349922, + "learning_rate": 5.533546400411899e-06, + "loss": 0.4928241968154907, + "step": 112 + }, + { + "epoch": 1.5973333333333333, + "grad_norm": 0.0036721359938383102, + "learning_rate": 5.4517068429574215e-06, + "loss": 0.5109080076217651, + "step": 113 + }, + { + "epoch": 1.6115555555555554, + "grad_norm": 0.0037413975223898888, + "learning_rate": 5.3697450830595775e-06, + "loss": 0.48911499977111816, + "step": 114 + }, + { + "epoch": 1.6257777777777778, + "grad_norm": 0.003966668155044317, + "learning_rate": 5.287683294230855e-06, + "loss": 0.49537116289138794, + "step": 115 + }, + { + "epoch": 1.6400000000000001, + "grad_norm": 0.0036018763203173876, + "learning_rate": 5.20554367704505e-06, + "loss": 0.4613257348537445, + "step": 116 + }, + { + "epoch": 1.6542222222222223, + "grad_norm": 0.0038206754252314568, + "learning_rate": 5.123348453131242e-06, + "loss": 0.5043697357177734, + "step": 117 + }, + { + "epoch": 1.6684444444444444, + "grad_norm": 0.0037831810768693686, + "learning_rate": 5.041119859162068e-06, + "loss": 0.48968735337257385, + "step": 118 + }, + { + "epoch": 1.6826666666666665, + "grad_norm": 0.0038370522670447826, + "learning_rate": 4.958880140837934e-06, + "loss": 0.49605250358581543, + "step": 119 + }, + { + "epoch": 1.696888888888889, + "grad_norm": 0.003586502280086279, + "learning_rate": 4.876651546868759e-06, + "loss": 0.4839916229248047, + "step": 120 + }, + { + "epoch": 1.7111111111111112, + "grad_norm": 0.003694413462653756, + "learning_rate": 4.794456322954953e-06, + "loss": 0.5173561573028564, + "step": 121 + }, + { + "epoch": 1.7253333333333334, + "grad_norm": 0.003933500964194536, + "learning_rate": 4.7123167057691446e-06, + "loss": 0.5177811980247498, + "step": 122 + }, + { + "epoch": 1.7395555555555555, + "grad_norm": 0.0036068507470190525, + "learning_rate": 4.630254916940424e-06, + "loss": 0.5164836049079895, + "step": 123 + }, + { + "epoch": 1.7537777777777777, + "grad_norm": 0.00349752651527524, + "learning_rate": 4.548293157042581e-06, + "loss": 0.5077825784683228, + "step": 124 + }, + { + "epoch": 1.768, + "grad_norm": 0.0037042393814772367, + "learning_rate": 4.466453599588103e-06, + "loss": 0.5107009410858154, + "step": 125 + }, + { + "epoch": 1.7822222222222224, + "grad_norm": 0.003853494767099619, + "learning_rate": 4.384758385029457e-06, + "loss": 0.4728143811225891, + "step": 126 + }, + { + "epoch": 1.7964444444444445, + "grad_norm": 0.0038627111352980137, + "learning_rate": 4.3032296147693225e-06, + "loss": 0.4858134984970093, + "step": 127 + }, + { + "epoch": 1.8106666666666666, + "grad_norm": 0.003657885594293475, + "learning_rate": 4.2218893451814005e-06, + "loss": 0.48357316851615906, + "step": 128 + }, + { + "epoch": 1.8248888888888888, + "grad_norm": 0.00355988135561347, + "learning_rate": 4.140759581643386e-06, + "loss": 0.49349913001060486, + "step": 129 + }, + { + "epoch": 1.8391111111111111, + "grad_norm": 0.004014762118458748, + "learning_rate": 4.059862272583755e-06, + "loss": 0.4969041347503662, + "step": 130 + }, + { + "epoch": 1.8533333333333335, + "grad_norm": 0.003631556173786521, + "learning_rate": 3.979219303543942e-06, + "loss": 0.47194504737854004, + "step": 131 + }, + { + "epoch": 1.8675555555555556, + "grad_norm": 0.003577238181605935, + "learning_rate": 3.898852491257547e-06, + "loss": 0.5020972490310669, + "step": 132 + }, + { + "epoch": 1.8817777777777778, + "grad_norm": 0.003780671861022711, + "learning_rate": 3.818783577748138e-06, + "loss": 0.5002884268760681, + "step": 133 + }, + { + "epoch": 1.896, + "grad_norm": 0.003708280622959137, + "learning_rate": 3.7390342244472886e-06, + "loss": 0.5007095336914062, + "step": 134 + }, + { + "epoch": 1.9102222222222223, + "grad_norm": 0.003498840844258666, + "learning_rate": 3.659626006334395e-06, + "loss": 0.4763813316822052, + "step": 135 + }, + { + "epoch": 1.9244444444444444, + "grad_norm": 0.003659547306597233, + "learning_rate": 3.5805804060998926e-06, + "loss": 0.48733818531036377, + "step": 136 + }, + { + "epoch": 1.9386666666666668, + "grad_norm": 0.003515617921948433, + "learning_rate": 3.501918808333453e-06, + "loss": 0.5118749141693115, + "step": 137 + }, + { + "epoch": 1.952888888888889, + "grad_norm": 0.0036544103641062975, + "learning_rate": 3.4236624937386874e-06, + "loss": 0.5178316831588745, + "step": 138 + }, + { + "epoch": 1.967111111111111, + "grad_norm": 0.003433604259043932, + "learning_rate": 3.3458326333759927e-06, + "loss": 0.4853537678718567, + "step": 139 + }, + { + "epoch": 1.9813333333333332, + "grad_norm": 0.003547502215951681, + "learning_rate": 3.268450282935026e-06, + "loss": 0.49210506677627563, + "step": 140 + }, + { + "epoch": 1.9955555555555555, + "grad_norm": 0.003462104359641671, + "learning_rate": 3.1915363770384223e-06, + "loss": 0.47841814160346985, + "step": 141 + }, + { + "epoch": 2.0, + "grad_norm": 0.005856912117451429, + "learning_rate": 3.1151117235782346e-06, + "loss": 0.5034412145614624, + "step": 142 + }, + { + "epoch": 2.014222222222222, + "grad_norm": 0.003923188429325819, + "learning_rate": 3.0391969980866874e-06, + "loss": 0.4864930808544159, + "step": 143 + }, + { + "epoch": 2.0284444444444443, + "grad_norm": 0.003596341237425804, + "learning_rate": 2.963812738142713e-06, + "loss": 0.4642598032951355, + "step": 144 + }, + { + "epoch": 2.042666666666667, + "grad_norm": 0.0038887078408151865, + "learning_rate": 2.8889793378158284e-06, + "loss": 0.48541027307510376, + "step": 145 + }, + { + "epoch": 2.056888888888889, + "grad_norm": 0.0034043709747493267, + "learning_rate": 2.814717042148827e-06, + "loss": 0.47221142053604126, + "step": 146 + }, + { + "epoch": 2.071111111111111, + "grad_norm": 0.0034274954814463854, + "learning_rate": 2.7410459416807856e-06, + "loss": 0.46124035120010376, + "step": 147 + }, + { + "epoch": 2.0853333333333333, + "grad_norm": 0.00352494022808969, + "learning_rate": 2.6679859670118785e-06, + "loss": 0.4869857430458069, + "step": 148 + }, + { + "epoch": 2.0995555555555554, + "grad_norm": 0.0035326213110238314, + "learning_rate": 2.5955568834114523e-06, + "loss": 0.45158863067626953, + "step": 149 + }, + { + "epoch": 2.113777777777778, + "grad_norm": 0.003645913675427437, + "learning_rate": 2.523778285470835e-06, + "loss": 0.4742748439311981, + "step": 150 + }, + { + "epoch": 2.113777777777778, + "eval_loss": 0.5196705460548401, + "eval_runtime": 15.8074, + "eval_samples_per_second": 12.652, + "eval_steps_per_second": 6.326, + "step": 150 + }, + { + "epoch": 2.128, + "grad_norm": 0.0036555698607116938, + "learning_rate": 2.452669591802307e-06, + "loss": 0.4526304602622986, + "step": 151 + }, + { + "epoch": 2.1422222222222222, + "grad_norm": 0.003359478199854493, + "learning_rate": 2.3822500397857016e-06, + "loss": 0.4651503562927246, + "step": 152 + }, + { + "epoch": 2.1564444444444444, + "grad_norm": 0.003462468506768346, + "learning_rate": 2.3125386803640188e-06, + "loss": 0.48414769768714905, + "step": 153 + }, + { + "epoch": 2.1706666666666665, + "grad_norm": 0.003318715374916792, + "learning_rate": 2.243554372889479e-06, + "loss": 0.462248295545578, + "step": 154 + }, + { + "epoch": 2.1848888888888887, + "grad_norm": 0.0034426096826791763, + "learning_rate": 2.1753157800214107e-06, + "loss": 0.4559134244918823, + "step": 155 + }, + { + "epoch": 2.1991111111111112, + "grad_norm": 0.0035529518499970436, + "learning_rate": 2.1078413626773547e-06, + "loss": 0.48357266187667847, + "step": 156 + }, + { + "epoch": 2.2133333333333334, + "grad_norm": 0.0034895483404397964, + "learning_rate": 2.0411493750387423e-06, + "loss": 0.4366765022277832, + "step": 157 + }, + { + "epoch": 2.2275555555555555, + "grad_norm": 0.003682621056213975, + "learning_rate": 1.9752578596124955e-06, + "loss": 0.4868413209915161, + "step": 158 + }, + { + "epoch": 2.2417777777777776, + "grad_norm": 0.0035529278684407473, + "learning_rate": 1.9101846423499113e-06, + "loss": 0.46049410104751587, + "step": 159 + }, + { + "epoch": 2.2560000000000002, + "grad_norm": 0.003581352299079299, + "learning_rate": 1.8459473278241125e-06, + "loss": 0.4605860412120819, + "step": 160 + }, + { + "epoch": 2.2702222222222224, + "grad_norm": 0.0033597166184335947, + "learning_rate": 1.7825632944674016e-06, + "loss": 0.46191370487213135, + "step": 161 + }, + { + "epoch": 2.2844444444444445, + "grad_norm": 0.003529531881213188, + "learning_rate": 1.7200496898697832e-06, + "loss": 0.4780347943305969, + "step": 162 + }, + { + "epoch": 2.2986666666666666, + "grad_norm": 0.0034702750854194164, + "learning_rate": 1.6584234261399535e-06, + "loss": 0.4648103713989258, + "step": 163 + }, + { + "epoch": 2.3128888888888888, + "grad_norm": 0.0036891205236315727, + "learning_rate": 1.5977011753299726e-06, + "loss": 0.46155521273612976, + "step": 164 + }, + { + "epoch": 2.327111111111111, + "grad_norm": 0.003741780063137412, + "learning_rate": 1.5378993649249053e-06, + "loss": 0.49238717555999756, + "step": 165 + }, + { + "epoch": 2.3413333333333335, + "grad_norm": 0.003572724526748061, + "learning_rate": 1.4790341733986085e-06, + "loss": 0.46214908361434937, + "step": 166 + }, + { + "epoch": 2.3555555555555556, + "grad_norm": 0.003393909428268671, + "learning_rate": 1.4211215258368866e-06, + "loss": 0.44751304388046265, + "step": 167 + }, + { + "epoch": 2.3697777777777778, + "grad_norm": 0.0035024855751544237, + "learning_rate": 1.3641770896292083e-06, + "loss": 0.4523189067840576, + "step": 168 + }, + { + "epoch": 2.384, + "grad_norm": 0.0034958638716489077, + "learning_rate": 1.3082162702301276e-06, + "loss": 0.46704721450805664, + "step": 169 + }, + { + "epoch": 2.398222222222222, + "grad_norm": 0.0035048348363488913, + "learning_rate": 1.2532542069915722e-06, + "loss": 0.47263163328170776, + "step": 170 + }, + { + "epoch": 2.4124444444444446, + "grad_norm": 0.0034445980563759804, + "learning_rate": 1.1993057690671174e-06, + "loss": 0.4592151641845703, + "step": 171 + }, + { + "epoch": 2.4266666666666667, + "grad_norm": 0.0034275518264621496, + "learning_rate": 1.1463855513893695e-06, + "loss": 0.4800827205181122, + "step": 172 + }, + { + "epoch": 2.440888888888889, + "grad_norm": 0.0034792288206517696, + "learning_rate": 1.0945078707215224e-06, + "loss": 0.4688478112220764, + "step": 173 + }, + { + "epoch": 2.455111111111111, + "grad_norm": 0.0035636303946375847, + "learning_rate": 1.0436867617841768e-06, + "loss": 0.46817073225975037, + "step": 174 + }, + { + "epoch": 2.469333333333333, + "grad_norm": 0.0033977399580180645, + "learning_rate": 9.939359734584552e-07, + "loss": 0.4763898253440857, + "step": 175 + }, + { + "epoch": 2.4835555555555557, + "grad_norm": 0.003528163768351078, + "learning_rate": 9.452689650664515e-07, + "loss": 0.480793833732605, + "step": 176 + }, + { + "epoch": 2.497777777777778, + "grad_norm": 0.00341871683485806, + "learning_rate": 8.976989027300265e-07, + "loss": 0.4694904088973999, + "step": 177 + }, + { + "epoch": 2.512, + "grad_norm": 0.003544109407812357, + "learning_rate": 8.512386558088919e-07, + "loss": 0.4877663254737854, + "step": 178 + }, + { + "epoch": 2.526222222222222, + "grad_norm": 0.0034784283488988876, + "learning_rate": 8.059007934190194e-07, + "loss": 0.46579453349113464, + "step": 179 + }, + { + "epoch": 2.5404444444444443, + "grad_norm": 0.003506515873596072, + "learning_rate": 7.61697581032243e-07, + "loss": 0.4954630434513092, + "step": 180 + }, + { + "epoch": 2.554666666666667, + "grad_norm": 0.003402002854272723, + "learning_rate": 7.186409771580355e-07, + "loss": 0.48210158944129944, + "step": 181 + }, + { + "epoch": 2.568888888888889, + "grad_norm": 0.003400753950700164, + "learning_rate": 6.76742630108298e-07, + "loss": 0.4673936069011688, + "step": 182 + }, + { + "epoch": 2.583111111111111, + "grad_norm": 0.0034171557053923607, + "learning_rate": 6.360138748461015e-07, + "loss": 0.46786054968833923, + "step": 183 + }, + { + "epoch": 2.5973333333333333, + "grad_norm": 0.003441787790507078, + "learning_rate": 5.964657299191712e-07, + "loss": 0.45768898725509644, + "step": 184 + }, + { + "epoch": 2.6115555555555554, + "grad_norm": 0.0034633874893188477, + "learning_rate": 5.581088944789953e-07, + "loss": 0.468585342168808, + "step": 185 + }, + { + "epoch": 2.6257777777777775, + "grad_norm": 0.0034696508664637804, + "learning_rate": 5.20953745386329e-07, + "loss": 0.4843568205833435, + "step": 186 + }, + { + "epoch": 2.64, + "grad_norm": 0.003448233939707279, + "learning_rate": 4.850103344038853e-07, + "loss": 0.4747229218482971, + "step": 187 + }, + { + "epoch": 2.6542222222222223, + "grad_norm": 0.00340772676281631, + "learning_rate": 4.502883854769935e-07, + "loss": 0.44664108753204346, + "step": 188 + }, + { + "epoch": 2.6684444444444444, + "grad_norm": 0.0034893041010946035, + "learning_rate": 4.167972921029262e-07, + "loss": 0.47903192043304443, + "step": 189 + }, + { + "epoch": 2.6826666666666665, + "grad_norm": 0.003642126452177763, + "learning_rate": 3.8454611478963235e-07, + "loss": 0.4844057559967041, + "step": 190 + }, + { + "epoch": 2.696888888888889, + "grad_norm": 0.0034686627332121134, + "learning_rate": 3.535435786045538e-07, + "loss": 0.4564388394355774, + "step": 191 + }, + { + "epoch": 2.7111111111111112, + "grad_norm": 0.003412955440580845, + "learning_rate": 3.237980708141919e-07, + "loss": 0.47008293867111206, + "step": 192 + }, + { + "epoch": 2.7253333333333334, + "grad_norm": 0.0035507141146808863, + "learning_rate": 2.9531763861505967e-07, + "loss": 0.479957640171051, + "step": 193 + }, + { + "epoch": 2.7395555555555555, + "grad_norm": 0.0035486763808876276, + "learning_rate": 2.681099869566328e-07, + "loss": 0.44965675473213196, + "step": 194 + }, + { + "epoch": 2.7537777777777777, + "grad_norm": 0.0034812854137271643, + "learning_rate": 2.4218247645689306e-07, + "loss": 0.4747314453125, + "step": 195 + }, + { + "epoch": 2.768, + "grad_norm": 0.00335957994684577, + "learning_rate": 2.175421214110235e-07, + "loss": 0.45799821615219116, + "step": 196 + }, + { + "epoch": 2.7822222222222224, + "grad_norm": 0.0035609498154371977, + "learning_rate": 1.941955878938029e-07, + "loss": 0.47467684745788574, + "step": 197 + }, + { + "epoch": 2.7964444444444445, + "grad_norm": 0.003390205791220069, + "learning_rate": 1.721491919561913e-07, + "loss": 0.4817090332508087, + "step": 198 + }, + { + "epoch": 2.8106666666666666, + "grad_norm": 0.003445240668952465, + "learning_rate": 1.514088979166256e-07, + "loss": 0.4649164080619812, + "step": 199 + }, + { + "epoch": 2.824888888888889, + "grad_norm": 0.003372759558260441, + "learning_rate": 1.3198031674745814e-07, + "loss": 0.4765157699584961, + "step": 200 + }, + { + "epoch": 2.824888888888889, + "eval_loss": 0.5171427726745605, + "eval_runtime": 9.8598, + "eval_samples_per_second": 20.284, + "eval_steps_per_second": 10.142, + "step": 200 + }, + { + "epoch": 2.8391111111111114, + "grad_norm": 0.0034065141808241606, + "learning_rate": 1.138687045569975e-07, + "loss": 0.46831822395324707, + "step": 201 + }, + { + "epoch": 2.8533333333333335, + "grad_norm": 0.0034053055569529533, + "learning_rate": 9.70789611675449e-08, + "loss": 0.47398146986961365, + "step": 202 + }, + { + "epoch": 2.8675555555555556, + "grad_norm": 0.0034186639823019505, + "learning_rate": 8.161562878982399e-08, + "loss": 0.45744937658309937, + "step": 203 + }, + { + "epoch": 2.8817777777777778, + "grad_norm": 0.0034667483996599913, + "learning_rate": 6.74828907941516e-08, + "loss": 0.48529934883117676, + "step": 204 + }, + { + "epoch": 2.896, + "grad_norm": 0.003455272177234292, + "learning_rate": 5.468457057869358e-08, + "loss": 0.4858051836490631, + "step": 205 + }, + { + "epoch": 2.910222222222222, + "grad_norm": 0.0033815570641309023, + "learning_rate": 4.322413053509944e-08, + "loss": 0.4590516984462738, + "step": 206 + }, + { + "epoch": 2.924444444444444, + "grad_norm": 0.0034011146053671837, + "learning_rate": 3.31046711118066e-08, + "loss": 0.45788976550102234, + "step": 207 + }, + { + "epoch": 2.9386666666666668, + "grad_norm": 0.003420215332880616, + "learning_rate": 2.4328929975260262e-08, + "loss": 0.4702548086643219, + "step": 208 + }, + { + "epoch": 2.952888888888889, + "grad_norm": 0.0033962675370275974, + "learning_rate": 1.6899281269279756e-08, + "loss": 0.4560105800628662, + "step": 209 + }, + { + "epoch": 2.967111111111111, + "grad_norm": 0.003477597376331687, + "learning_rate": 1.0817734972768946e-08, + "loss": 0.46400371193885803, + "step": 210 + }, + { + "epoch": 2.981333333333333, + "grad_norm": 0.0036108316853642464, + "learning_rate": 6.085936355947897e-09, + "loss": 0.48368915915489197, + "step": 211 + }, + { + "epoch": 2.9955555555555557, + "grad_norm": 0.003335504326969385, + "learning_rate": 2.7051655352494654e-09, + "loss": 0.45437300205230713, + "step": 212 + }, + { + "epoch": 3.0, + "grad_norm": 0.00642085587605834, + "learning_rate": 6.763371270035457e-10, + "loss": 0.5508654713630676, + "step": 213 + }, + { + "epoch": 3.0, + "step": 213, + "total_flos": 2.4727435500270387e+17, + "train_loss": 0.5265021298972654, + "train_runtime": 6756.1203, + "train_samples_per_second": 7.993, + "train_steps_per_second": 0.032 + } + ], + "logging_steps": 1, + "max_steps": 213, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.4727435500270387e+17, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/qwen2.5-1.5b-instruct/training_args.bin b/qwen2.5-1.5b-instruct/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..4badd3789f92f64e02e2816810bfe346973afac5 --- /dev/null +++ b/qwen2.5-1.5b-instruct/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39e78c30a5784e935e65cb8fa1a73b2afb467afccc55e7223863b667252fac3b +size 5649 diff --git a/qwen2.5-1.5b-instruct/training_eval_loss.png b/qwen2.5-1.5b-instruct/training_eval_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..ebb84744b3b7dca5721634934e447264ead07001 Binary files /dev/null and b/qwen2.5-1.5b-instruct/training_eval_loss.png differ diff --git a/qwen2.5-1.5b-instruct/training_loss.png b/qwen2.5-1.5b-instruct/training_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..9a78b4378190bdcfe134b7d68d734efeb26255dc Binary files /dev/null and b/qwen2.5-1.5b-instruct/training_loss.png differ diff --git a/qwen2.5-3b-instruct/README.md b/qwen2.5-3b-instruct/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1ecf35d45f3e759a796bbe2f9d7cca15796c379e --- /dev/null +++ b/qwen2.5-3b-instruct/README.md @@ -0,0 +1,69 @@ +--- +library_name: transformers +license: other +base_model: Qwen/Qwen2.5-3B-Instruct +tags: +- llama-factory +- full +- generated_from_trainer +model-index: +- name: qwen2.5-3b-instruct + results: [] +--- + + + +# qwen2.5-3b-instruct + +This model is a fine-tuned version of [Qwen/Qwen2.5-3B-Instruct](https://huggingface.co/Qwen/Qwen2.5-3B-Instruct) on the codealpaca_train_20k_trace_generation_valid dataset. +It achieves the following results on the evaluation set: +- Loss: 0.4637 + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 1e-05 +- train_batch_size: 1 +- eval_batch_size: 1 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 2 +- gradient_accumulation_steps: 128 +- total_train_batch_size: 256 +- total_eval_batch_size: 2 +- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_steps: 0.1 +- num_epochs: 3 + +### Training results + +| Training Loss | Epoch | Step | Validation Loss | +|:-------------:|:------:|:----:|:---------------:| +| 0.4906 | 0.7111 | 50 | 0.5049 | +| 0.4582 | 1.4124 | 100 | 0.4757 | +| 0.4293 | 2.1138 | 150 | 0.4662 | +| 0.4300 | 2.8249 | 200 | 0.4637 | + + +### Framework versions + +- Transformers 5.2.0 +- Pytorch 2.11.0+cu128 +- Datasets 4.0.0 +- Tokenizers 0.22.2 diff --git a/qwen2.5-3b-instruct/all_results.json b/qwen2.5-3b-instruct/all_results.json new file mode 100644 index 0000000000000000000000000000000000000000..b5ea85e6562f75166ee13bb71b55a9babdfacb21 --- /dev/null +++ b/qwen2.5-3b-instruct/all_results.json @@ -0,0 +1,12 @@ +{ + "epoch": 3.0, + "eval_loss": 0.4636715054512024, + "eval_runtime": 39.8733, + "eval_samples_per_second": 5.016, + "eval_steps_per_second": 2.508, + "total_flos": 5.2362750737986355e+17, + "train_loss": 0.4823716257659482, + "train_runtime": 11702.1548, + "train_samples_per_second": 4.615, + "train_steps_per_second": 0.018 +} \ No newline at end of file diff --git a/qwen2.5-3b-instruct/chat_template.jinja b/qwen2.5-3b-instruct/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e --- /dev/null +++ b/qwen2.5-3b-instruct/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/qwen2.5-3b-instruct/checkpoint-142/chat_template.jinja b/qwen2.5-3b-instruct/checkpoint-142/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-142/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/qwen2.5-3b-instruct/checkpoint-142/config.json b/qwen2.5-3b-instruct/checkpoint-142/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4814292863d1e5aa2f4fda1db18f4082cd61aa93 --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-142/config.json @@ -0,0 +1,69 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 11008, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 70, + "model_type": "qwen2", + "num_attention_heads": 16, + "num_hidden_layers": 36, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/qwen2.5-3b-instruct/checkpoint-142/generation_config.json b/qwen2.5-3b-instruct/checkpoint-142/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..47205fbf3386a215e7c2b9d62df6576f49b77e7b --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-142/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.05, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.2.0" +} diff --git a/qwen2.5-3b-instruct/checkpoint-142/model.safetensors b/qwen2.5-3b-instruct/checkpoint-142/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2230048607369a62598ac950a9dc04e2122e453f --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-142/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29f6814f95d3ff87978af4457e7798e9e59f96bddfe938ed3b507d30e88bd870 +size 12343804296 diff --git a/qwen2.5-3b-instruct/checkpoint-142/optimizer.pt b/qwen2.5-3b-instruct/checkpoint-142/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..2098f3b4d9e6dda9926dcabf886c3c97792c0e25 --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-142/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63210209fbb6633262a45fca68d5aad7418012636bd5e4bdca9fcf6de6a6e2fb +size 24687895753 diff --git a/qwen2.5-3b-instruct/checkpoint-142/rng_state_0.pth b/qwen2.5-3b-instruct/checkpoint-142/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..870021e3fa5ac35c2f711adf0c93a556ab4842da --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-142/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95e5fc2074c0df31522a514f862c86cb00d71c946a7f15cc9ec0e53a69fb28a7 +size 14917 diff --git a/qwen2.5-3b-instruct/checkpoint-142/rng_state_1.pth b/qwen2.5-3b-instruct/checkpoint-142/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..21f20da5eb1da017f08aaa88bd19cf24d40e3fbf --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-142/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e7153eae67b6c9232a41bc996a2bf5b83229b8c7230d61911ac0fd40e64154e +size 14917 diff --git a/qwen2.5-3b-instruct/checkpoint-142/scheduler.pt b/qwen2.5-3b-instruct/checkpoint-142/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..2967e7243507a115f9424b1b386bf3657f047c75 --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-142/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:922deb71175797a96c4bddc5e1cef1c097df38c0dfa52334e97c89e0f573f2bb +size 1465 diff --git a/qwen2.5-3b-instruct/checkpoint-142/tokenizer.json b/qwen2.5-3b-instruct/checkpoint-142/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..34510ff0037cd50428af467a17ead5a96140a32c --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-142/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/qwen2.5-3b-instruct/checkpoint-142/tokenizer_config.json b/qwen2.5-3b-instruct/checkpoint-142/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..145e2c7a68b109ae0253dda29fc2e66cb01b4b85 --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-142/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen2.5-3b-instruct/checkpoint-142/trainer_state.json b/qwen2.5-3b-instruct/checkpoint-142/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..91022a4afe6e02a2017f0f2f2c53e056e5787755 --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-142/trainer_state.json @@ -0,0 +1,1044 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 50, + "global_step": 142, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.014222222222222223, + "grad_norm": 0.09173738956451416, + "learning_rate": 0.0, + "loss": 1.005206823348999, + "step": 1 + }, + { + "epoch": 0.028444444444444446, + "grad_norm": 0.09365915507078171, + "learning_rate": 4.5454545454545457e-07, + "loss": 1.0057916641235352, + "step": 2 + }, + { + "epoch": 0.042666666666666665, + "grad_norm": 0.0689043328166008, + "learning_rate": 9.090909090909091e-07, + "loss": 0.9406883716583252, + "step": 3 + }, + { + "epoch": 0.05688888888888889, + "grad_norm": 0.04593552276492119, + "learning_rate": 1.3636363636363636e-06, + "loss": 0.8941981792449951, + "step": 4 + }, + { + "epoch": 0.07111111111111111, + "grad_norm": 0.03204488381743431, + "learning_rate": 1.8181818181818183e-06, + "loss": 0.7926980257034302, + "step": 5 + }, + { + "epoch": 0.08533333333333333, + "grad_norm": 0.027753565460443497, + "learning_rate": 2.2727272727272728e-06, + "loss": 0.8305954933166504, + "step": 6 + }, + { + "epoch": 0.09955555555555555, + "grad_norm": 0.03171968460083008, + "learning_rate": 2.7272727272727272e-06, + "loss": 0.7948952913284302, + "step": 7 + }, + { + "epoch": 0.11377777777777778, + "grad_norm": 0.026748575270175934, + "learning_rate": 3.181818181818182e-06, + "loss": 0.7530462741851807, + "step": 8 + }, + { + "epoch": 0.128, + "grad_norm": 0.023413579910993576, + "learning_rate": 3.6363636363636366e-06, + "loss": 0.7157606482505798, + "step": 9 + }, + { + "epoch": 0.14222222222222222, + "grad_norm": 0.020688239485025406, + "learning_rate": 4.0909090909090915e-06, + "loss": 0.7008587718009949, + "step": 10 + }, + { + "epoch": 0.15644444444444444, + "grad_norm": 0.014185823500156403, + "learning_rate": 4.5454545454545455e-06, + "loss": 0.6668463945388794, + "step": 11 + }, + { + "epoch": 0.17066666666666666, + "grad_norm": 0.012938937172293663, + "learning_rate": 5e-06, + "loss": 0.6507903337478638, + "step": 12 + }, + { + "epoch": 0.18488888888888888, + "grad_norm": 0.012045802548527718, + "learning_rate": 5.4545454545454545e-06, + "loss": 0.6217454671859741, + "step": 13 + }, + { + "epoch": 0.1991111111111111, + "grad_norm": 0.009660671465098858, + "learning_rate": 5.90909090909091e-06, + "loss": 0.6269917488098145, + "step": 14 + }, + { + "epoch": 0.21333333333333335, + "grad_norm": 0.008694784715771675, + "learning_rate": 6.363636363636364e-06, + "loss": 0.5836964845657349, + "step": 15 + }, + { + "epoch": 0.22755555555555557, + "grad_norm": 0.008789200335741043, + "learning_rate": 6.818181818181818e-06, + "loss": 0.6187424659729004, + "step": 16 + }, + { + "epoch": 0.24177777777777779, + "grad_norm": 0.008197025395929813, + "learning_rate": 7.272727272727273e-06, + "loss": 0.5914188027381897, + "step": 17 + }, + { + "epoch": 0.256, + "grad_norm": 0.007580601144582033, + "learning_rate": 7.727272727272727e-06, + "loss": 0.5802223682403564, + "step": 18 + }, + { + "epoch": 0.2702222222222222, + "grad_norm": 0.007013861555606127, + "learning_rate": 8.181818181818183e-06, + "loss": 0.5884265899658203, + "step": 19 + }, + { + "epoch": 0.28444444444444444, + "grad_norm": 0.00683254050090909, + "learning_rate": 8.636363636363637e-06, + "loss": 0.5850374698638916, + "step": 20 + }, + { + "epoch": 0.2986666666666667, + "grad_norm": 0.006111981347203255, + "learning_rate": 9.090909090909091e-06, + "loss": 0.5651419162750244, + "step": 21 + }, + { + "epoch": 0.3128888888888889, + "grad_norm": 0.005564771592617035, + "learning_rate": 9.545454545454547e-06, + "loss": 0.553795576095581, + "step": 22 + }, + { + "epoch": 0.32711111111111113, + "grad_norm": 0.0057683708146214485, + "learning_rate": 1e-05, + "loss": 0.5542658567428589, + "step": 23 + }, + { + "epoch": 0.3413333333333333, + "grad_norm": 0.006690182723104954, + "learning_rate": 9.999323662872998e-06, + "loss": 0.5586982369422913, + "step": 24 + }, + { + "epoch": 0.35555555555555557, + "grad_norm": 0.006314165890216827, + "learning_rate": 9.99729483446475e-06, + "loss": 0.5584714412689209, + "step": 25 + }, + { + "epoch": 0.36977777777777776, + "grad_norm": 0.005813857074826956, + "learning_rate": 9.993914063644053e-06, + "loss": 0.5466614961624146, + "step": 26 + }, + { + "epoch": 0.384, + "grad_norm": 0.005381743889302015, + "learning_rate": 9.989182265027232e-06, + "loss": 0.5380265712738037, + "step": 27 + }, + { + "epoch": 0.3982222222222222, + "grad_norm": 0.0049252682365477085, + "learning_rate": 9.98310071873072e-06, + "loss": 0.535520613193512, + "step": 28 + }, + { + "epoch": 0.41244444444444445, + "grad_norm": 0.0048560877330601215, + "learning_rate": 9.975671070024741e-06, + "loss": 0.5377035140991211, + "step": 29 + }, + { + "epoch": 0.4266666666666667, + "grad_norm": 0.004911178257316351, + "learning_rate": 9.966895328888195e-06, + "loss": 0.5570093989372253, + "step": 30 + }, + { + "epoch": 0.4408888888888889, + "grad_norm": 0.004746970720589161, + "learning_rate": 9.956775869464901e-06, + "loss": 0.5285611748695374, + "step": 31 + }, + { + "epoch": 0.45511111111111113, + "grad_norm": 0.0047261337749660015, + "learning_rate": 9.945315429421307e-06, + "loss": 0.5199858546257019, + "step": 32 + }, + { + "epoch": 0.4693333333333333, + "grad_norm": 0.004447608254849911, + "learning_rate": 9.932517109205849e-06, + "loss": 0.5250071287155151, + "step": 33 + }, + { + "epoch": 0.48355555555555557, + "grad_norm": 0.004617375787347555, + "learning_rate": 9.918384371210178e-06, + "loss": 0.5516105890274048, + "step": 34 + }, + { + "epoch": 0.49777777777777776, + "grad_norm": 0.004378544166684151, + "learning_rate": 9.902921038832456e-06, + "loss": 0.5298895835876465, + "step": 35 + }, + { + "epoch": 0.512, + "grad_norm": 0.004164176061749458, + "learning_rate": 9.886131295443003e-06, + "loss": 0.5044555068016052, + "step": 36 + }, + { + "epoch": 0.5262222222222223, + "grad_norm": 0.004092546179890633, + "learning_rate": 9.868019683252543e-06, + "loss": 0.5102939009666443, + "step": 37 + }, + { + "epoch": 0.5404444444444444, + "grad_norm": 0.004554086830466986, + "learning_rate": 9.848591102083375e-06, + "loss": 0.5421494245529175, + "step": 38 + }, + { + "epoch": 0.5546666666666666, + "grad_norm": 0.004217132460325956, + "learning_rate": 9.82785080804381e-06, + "loss": 0.5239347219467163, + "step": 39 + }, + { + "epoch": 0.5688888888888889, + "grad_norm": 0.004286583047360182, + "learning_rate": 9.805804412106197e-06, + "loss": 0.5437066555023193, + "step": 40 + }, + { + "epoch": 0.5831111111111111, + "grad_norm": 0.004261231981217861, + "learning_rate": 9.782457878588977e-06, + "loss": 0.5146951079368591, + "step": 41 + }, + { + "epoch": 0.5973333333333334, + "grad_norm": 0.004209685139358044, + "learning_rate": 9.75781752354311e-06, + "loss": 0.4995099604129791, + "step": 42 + }, + { + "epoch": 0.6115555555555555, + "grad_norm": 0.0039211539551615715, + "learning_rate": 9.731890013043367e-06, + "loss": 0.5249044299125671, + "step": 43 + }, + { + "epoch": 0.6257777777777778, + "grad_norm": 0.0038641595747321844, + "learning_rate": 9.704682361384941e-06, + "loss": 0.4980819821357727, + "step": 44 + }, + { + "epoch": 0.64, + "grad_norm": 0.004069323651492596, + "learning_rate": 9.676201929185809e-06, + "loss": 0.5113821625709534, + "step": 45 + }, + { + "epoch": 0.6542222222222223, + "grad_norm": 0.0038530679885298014, + "learning_rate": 9.646456421395447e-06, + "loss": 0.5072203278541565, + "step": 46 + }, + { + "epoch": 0.6684444444444444, + "grad_norm": 0.003951605875045061, + "learning_rate": 9.615453885210368e-06, + "loss": 0.509921669960022, + "step": 47 + }, + { + "epoch": 0.6826666666666666, + "grad_norm": 0.0038154355715960264, + "learning_rate": 9.583202707897075e-06, + "loss": 0.49035143852233887, + "step": 48 + }, + { + "epoch": 0.6968888888888889, + "grad_norm": 0.004143900703638792, + "learning_rate": 9.549711614523007e-06, + "loss": 0.4858332574367523, + "step": 49 + }, + { + "epoch": 0.7111111111111111, + "grad_norm": 0.0038300366140902042, + "learning_rate": 9.514989665596114e-06, + "loss": 0.49062296748161316, + "step": 50 + }, + { + "epoch": 0.7111111111111111, + "eval_loss": 0.5049259662628174, + "eval_runtime": 38.0442, + "eval_samples_per_second": 5.257, + "eval_steps_per_second": 2.629, + "step": 50 + }, + { + "epoch": 0.7253333333333334, + "grad_norm": 0.003965203184634447, + "learning_rate": 9.479046254613673e-06, + "loss": 0.5013079643249512, + "step": 51 + }, + { + "epoch": 0.7395555555555555, + "grad_norm": 0.003626414341852069, + "learning_rate": 9.441891105521005e-06, + "loss": 0.46776771545410156, + "step": 52 + }, + { + "epoch": 0.7537777777777778, + "grad_norm": 0.003647371893748641, + "learning_rate": 9.40353427008083e-06, + "loss": 0.4870305061340332, + "step": 53 + }, + { + "epoch": 0.768, + "grad_norm": 0.0036924544256180525, + "learning_rate": 9.3639861251539e-06, + "loss": 0.48354169726371765, + "step": 54 + }, + { + "epoch": 0.7822222222222223, + "grad_norm": 0.0037534767761826515, + "learning_rate": 9.323257369891702e-06, + "loss": 0.5097652673721313, + "step": 55 + }, + { + "epoch": 0.7964444444444444, + "grad_norm": 0.003901001764461398, + "learning_rate": 9.281359022841966e-06, + "loss": 0.5023170709609985, + "step": 56 + }, + { + "epoch": 0.8106666666666666, + "grad_norm": 0.0037305736914277077, + "learning_rate": 9.238302418967757e-06, + "loss": 0.49597933888435364, + "step": 57 + }, + { + "epoch": 0.8248888888888889, + "grad_norm": 0.003764534369111061, + "learning_rate": 9.194099206580981e-06, + "loss": 0.504522442817688, + "step": 58 + }, + { + "epoch": 0.8391111111111111, + "grad_norm": 0.004040914587676525, + "learning_rate": 9.14876134419111e-06, + "loss": 0.49795830249786377, + "step": 59 + }, + { + "epoch": 0.8533333333333334, + "grad_norm": 0.003855582093819976, + "learning_rate": 9.102301097269974e-06, + "loss": 0.49624279141426086, + "step": 60 + }, + { + "epoch": 0.8675555555555555, + "grad_norm": 0.003851535264402628, + "learning_rate": 9.05473103493355e-06, + "loss": 0.47714120149612427, + "step": 61 + }, + { + "epoch": 0.8817777777777778, + "grad_norm": 0.004122862592339516, + "learning_rate": 9.006064026541549e-06, + "loss": 0.5046075582504272, + "step": 62 + }, + { + "epoch": 0.896, + "grad_norm": 0.0036710964050143957, + "learning_rate": 8.956313238215824e-06, + "loss": 0.45711296796798706, + "step": 63 + }, + { + "epoch": 0.9102222222222223, + "grad_norm": 0.004037045873701572, + "learning_rate": 8.905492129278478e-06, + "loss": 0.4839310050010681, + "step": 64 + }, + { + "epoch": 0.9244444444444444, + "grad_norm": 0.004081980790942907, + "learning_rate": 8.85361444861063e-06, + "loss": 0.4877704381942749, + "step": 65 + }, + { + "epoch": 0.9386666666666666, + "grad_norm": 0.0037577205803245306, + "learning_rate": 8.800694230932885e-06, + "loss": 0.49369895458221436, + "step": 66 + }, + { + "epoch": 0.9528888888888889, + "grad_norm": 0.0037781032733619213, + "learning_rate": 8.74674579300843e-06, + "loss": 0.4812237620353699, + "step": 67 + }, + { + "epoch": 0.9671111111111111, + "grad_norm": 0.0046208929270505905, + "learning_rate": 8.691783729769874e-06, + "loss": 0.4722365736961365, + "step": 68 + }, + { + "epoch": 0.9813333333333333, + "grad_norm": 0.004026597365736961, + "learning_rate": 8.635822910370793e-06, + "loss": 0.4962988495826721, + "step": 69 + }, + { + "epoch": 0.9955555555555555, + "grad_norm": 0.004095569718629122, + "learning_rate": 8.578878474163115e-06, + "loss": 0.45991775393486023, + "step": 70 + }, + { + "epoch": 1.0, + "grad_norm": 0.006300764624029398, + "learning_rate": 8.520965826601394e-06, + "loss": 0.4934382140636444, + "step": 71 + }, + { + "epoch": 1.0142222222222221, + "grad_norm": 0.00428728898987174, + "learning_rate": 8.462100635075097e-06, + "loss": 0.4847542643547058, + "step": 72 + }, + { + "epoch": 1.0284444444444445, + "grad_norm": 0.004186135251075029, + "learning_rate": 8.40229882467003e-06, + "loss": 0.4636518955230713, + "step": 73 + }, + { + "epoch": 1.0426666666666666, + "grad_norm": 0.003916833084076643, + "learning_rate": 8.341576573860049e-06, + "loss": 0.44187992811203003, + "step": 74 + }, + { + "epoch": 1.056888888888889, + "grad_norm": 0.004144088365137577, + "learning_rate": 8.279950310130218e-06, + "loss": 0.467815637588501, + "step": 75 + }, + { + "epoch": 1.0711111111111111, + "grad_norm": 0.004077525809407234, + "learning_rate": 8.2174367055326e-06, + "loss": 0.451981782913208, + "step": 76 + }, + { + "epoch": 1.0853333333333333, + "grad_norm": 0.004103589337319136, + "learning_rate": 8.154052672175888e-06, + "loss": 0.4449639320373535, + "step": 77 + }, + { + "epoch": 1.0995555555555556, + "grad_norm": 0.00439891591668129, + "learning_rate": 8.08981535765009e-06, + "loss": 0.47661933302879333, + "step": 78 + }, + { + "epoch": 1.1137777777777778, + "grad_norm": 0.004592400509864092, + "learning_rate": 8.024742140387506e-06, + "loss": 0.45776277780532837, + "step": 79 + }, + { + "epoch": 1.1280000000000001, + "grad_norm": 0.0037961499765515327, + "learning_rate": 7.95885062496126e-06, + "loss": 0.44757023453712463, + "step": 80 + }, + { + "epoch": 1.1422222222222222, + "grad_norm": 0.003865304868668318, + "learning_rate": 7.892158637322647e-06, + "loss": 0.419955849647522, + "step": 81 + }, + { + "epoch": 1.1564444444444444, + "grad_norm": 0.00407387875020504, + "learning_rate": 7.824684219978591e-06, + "loss": 0.4457795023918152, + "step": 82 + }, + { + "epoch": 1.1706666666666667, + "grad_norm": 0.00370654184371233, + "learning_rate": 7.756445627110523e-06, + "loss": 0.45605674386024475, + "step": 83 + }, + { + "epoch": 1.1848888888888889, + "grad_norm": 0.003941345028579235, + "learning_rate": 7.68746131963598e-06, + "loss": 0.46289366483688354, + "step": 84 + }, + { + "epoch": 1.199111111111111, + "grad_norm": 0.0039403438568115234, + "learning_rate": 7.6177499602143e-06, + "loss": 0.45568281412124634, + "step": 85 + }, + { + "epoch": 1.2133333333333334, + "grad_norm": 0.003944277763366699, + "learning_rate": 7.547330408197695e-06, + "loss": 0.44618040323257446, + "step": 86 + }, + { + "epoch": 1.2275555555555555, + "grad_norm": 0.0040544732473790646, + "learning_rate": 7.476221714529167e-06, + "loss": 0.45724251866340637, + "step": 87 + }, + { + "epoch": 1.2417777777777779, + "grad_norm": 0.004533152561634779, + "learning_rate": 7.404443116588548e-06, + "loss": 0.4806937575340271, + "step": 88 + }, + { + "epoch": 1.256, + "grad_norm": 0.004002939909696579, + "learning_rate": 7.332014032988123e-06, + "loss": 0.4561174213886261, + "step": 89 + }, + { + "epoch": 1.2702222222222221, + "grad_norm": 0.003825302002951503, + "learning_rate": 7.2589540583192165e-06, + "loss": 0.4558497667312622, + "step": 90 + }, + { + "epoch": 1.2844444444444445, + "grad_norm": 0.00397696765139699, + "learning_rate": 7.185282957851175e-06, + "loss": 0.4397018551826477, + "step": 91 + }, + { + "epoch": 1.2986666666666666, + "grad_norm": 0.004289410542696714, + "learning_rate": 7.111020662184174e-06, + "loss": 0.45822006464004517, + "step": 92 + }, + { + "epoch": 1.3128888888888888, + "grad_norm": 0.004002283327281475, + "learning_rate": 7.036187261857289e-06, + "loss": 0.4479483366012573, + "step": 93 + }, + { + "epoch": 1.3271111111111111, + "grad_norm": 0.004231768660247326, + "learning_rate": 6.960803001913315e-06, + "loss": 0.4738925099372864, + "step": 94 + }, + { + "epoch": 1.3413333333333333, + "grad_norm": 0.0040559363551437855, + "learning_rate": 6.884888276421766e-06, + "loss": 0.4614931643009186, + "step": 95 + }, + { + "epoch": 1.3555555555555556, + "grad_norm": 0.003587141167372465, + "learning_rate": 6.8084636229615786e-06, + "loss": 0.44328922033309937, + "step": 96 + }, + { + "epoch": 1.3697777777777778, + "grad_norm": 0.004009770695120096, + "learning_rate": 6.731549717064975e-06, + "loss": 0.44612404704093933, + "step": 97 + }, + { + "epoch": 1.384, + "grad_norm": 0.004132336005568504, + "learning_rate": 6.654167366624009e-06, + "loss": 0.4554000794887543, + "step": 98 + }, + { + "epoch": 1.3982222222222223, + "grad_norm": 0.0041816201992332935, + "learning_rate": 6.576337506261314e-06, + "loss": 0.45616668462753296, + "step": 99 + }, + { + "epoch": 1.4124444444444444, + "grad_norm": 0.003568898420780897, + "learning_rate": 6.498081191666549e-06, + "loss": 0.45816588401794434, + "step": 100 + }, + { + "epoch": 1.4124444444444444, + "eval_loss": 0.4757018983364105, + "eval_runtime": 39.6302, + "eval_samples_per_second": 5.047, + "eval_steps_per_second": 2.523, + "step": 100 + }, + { + "epoch": 1.4266666666666667, + "grad_norm": 0.004014050588011742, + "learning_rate": 6.419419593900109e-06, + "loss": 0.4430053234100342, + "step": 101 + }, + { + "epoch": 1.4408888888888889, + "grad_norm": 0.0039361282251775265, + "learning_rate": 6.340373993665607e-06, + "loss": 0.43451493978500366, + "step": 102 + }, + { + "epoch": 1.455111111111111, + "grad_norm": 0.0040025534108281136, + "learning_rate": 6.2609657755527135e-06, + "loss": 0.4296550154685974, + "step": 103 + }, + { + "epoch": 1.4693333333333334, + "grad_norm": 0.004081938415765762, + "learning_rate": 6.1812164222518626e-06, + "loss": 0.46728163957595825, + "step": 104 + }, + { + "epoch": 1.4835555555555555, + "grad_norm": 0.0043087489902973175, + "learning_rate": 6.101147508742456e-06, + "loss": 0.446887731552124, + "step": 105 + }, + { + "epoch": 1.4977777777777779, + "grad_norm": 0.0037424617912620306, + "learning_rate": 6.020780696456059e-06, + "loss": 0.43969088792800903, + "step": 106 + }, + { + "epoch": 1.512, + "grad_norm": 0.003990524914115667, + "learning_rate": 5.940137727416247e-06, + "loss": 0.4489642381668091, + "step": 107 + }, + { + "epoch": 1.5262222222222221, + "grad_norm": 0.00401579262688756, + "learning_rate": 5.859240418356614e-06, + "loss": 0.4380025863647461, + "step": 108 + }, + { + "epoch": 1.5404444444444443, + "grad_norm": 0.0042803785763680935, + "learning_rate": 5.778110654818602e-06, + "loss": 0.47752195596694946, + "step": 109 + }, + { + "epoch": 1.5546666666666666, + "grad_norm": 0.003838231321424246, + "learning_rate": 5.696770385230679e-06, + "loss": 0.4498862028121948, + "step": 110 + }, + { + "epoch": 1.568888888888889, + "grad_norm": 0.0041479310020804405, + "learning_rate": 5.615241614970546e-06, + "loss": 0.44400298595428467, + "step": 111 + }, + { + "epoch": 1.5831111111111111, + "grad_norm": 0.0038208707701414824, + "learning_rate": 5.533546400411899e-06, + "loss": 0.4457319378852844, + "step": 112 + }, + { + "epoch": 1.5973333333333333, + "grad_norm": 0.0038792924024164677, + "learning_rate": 5.4517068429574215e-06, + "loss": 0.4610831141471863, + "step": 113 + }, + { + "epoch": 1.6115555555555554, + "grad_norm": 0.004012539051473141, + "learning_rate": 5.3697450830595775e-06, + "loss": 0.4425020217895508, + "step": 114 + }, + { + "epoch": 1.6257777777777778, + "grad_norm": 0.003996311221271753, + "learning_rate": 5.287683294230855e-06, + "loss": 0.44555309414863586, + "step": 115 + }, + { + "epoch": 1.6400000000000001, + "grad_norm": 0.0037504970096051693, + "learning_rate": 5.20554367704505e-06, + "loss": 0.417566180229187, + "step": 116 + }, + { + "epoch": 1.6542222222222223, + "grad_norm": 0.003828976536169648, + "learning_rate": 5.123348453131242e-06, + "loss": 0.45680171251296997, + "step": 117 + }, + { + "epoch": 1.6684444444444444, + "grad_norm": 0.0040869563817977905, + "learning_rate": 5.041119859162068e-06, + "loss": 0.4432540535926819, + "step": 118 + }, + { + "epoch": 1.6826666666666665, + "grad_norm": 0.004195951856672764, + "learning_rate": 4.958880140837934e-06, + "loss": 0.44924670457839966, + "step": 119 + }, + { + "epoch": 1.696888888888889, + "grad_norm": 0.003907010890543461, + "learning_rate": 4.876651546868759e-06, + "loss": 0.43826115131378174, + "step": 120 + }, + { + "epoch": 1.7111111111111112, + "grad_norm": 0.003893865505233407, + "learning_rate": 4.794456322954953e-06, + "loss": 0.4698702096939087, + "step": 121 + }, + { + "epoch": 1.7253333333333334, + "grad_norm": 0.003987904172390699, + "learning_rate": 4.7123167057691446e-06, + "loss": 0.4665527045726776, + "step": 122 + }, + { + "epoch": 1.7395555555555555, + "grad_norm": 0.003826166968792677, + "learning_rate": 4.630254916940424e-06, + "loss": 0.46612560749053955, + "step": 123 + }, + { + "epoch": 1.7537777777777777, + "grad_norm": 0.0037139400374144316, + "learning_rate": 4.548293157042581e-06, + "loss": 0.4559047818183899, + "step": 124 + }, + { + "epoch": 1.768, + "grad_norm": 0.0038443440571427345, + "learning_rate": 4.466453599588103e-06, + "loss": 0.4595223069190979, + "step": 125 + }, + { + "epoch": 1.7822222222222224, + "grad_norm": 0.0039060397539287806, + "learning_rate": 4.384758385029457e-06, + "loss": 0.42560791969299316, + "step": 126 + }, + { + "epoch": 1.7964444444444445, + "grad_norm": 0.0041647544130682945, + "learning_rate": 4.3032296147693225e-06, + "loss": 0.43939000368118286, + "step": 127 + }, + { + "epoch": 1.8106666666666666, + "grad_norm": 0.004102814476937056, + "learning_rate": 4.2218893451814005e-06, + "loss": 0.4399089217185974, + "step": 128 + }, + { + "epoch": 1.8248888888888888, + "grad_norm": 0.003889351151883602, + "learning_rate": 4.140759581643386e-06, + "loss": 0.4451609253883362, + "step": 129 + }, + { + "epoch": 1.8391111111111111, + "grad_norm": 0.004501548130065203, + "learning_rate": 4.059862272583755e-06, + "loss": 0.4488486051559448, + "step": 130 + }, + { + "epoch": 1.8533333333333335, + "grad_norm": 0.0038232814986258745, + "learning_rate": 3.979219303543942e-06, + "loss": 0.42662614583969116, + "step": 131 + }, + { + "epoch": 1.8675555555555556, + "grad_norm": 0.003998646512627602, + "learning_rate": 3.898852491257547e-06, + "loss": 0.4547004997730255, + "step": 132 + }, + { + "epoch": 1.8817777777777778, + "grad_norm": 0.00403416994959116, + "learning_rate": 3.818783577748138e-06, + "loss": 0.4510434567928314, + "step": 133 + }, + { + "epoch": 1.896, + "grad_norm": 0.003799683880060911, + "learning_rate": 3.7390342244472886e-06, + "loss": 0.45353883504867554, + "step": 134 + }, + { + "epoch": 1.9102222222222223, + "grad_norm": 0.003921311814337969, + "learning_rate": 3.659626006334395e-06, + "loss": 0.4302864670753479, + "step": 135 + }, + { + "epoch": 1.9244444444444444, + "grad_norm": 0.00398287083953619, + "learning_rate": 3.5805804060998926e-06, + "loss": 0.4398406744003296, + "step": 136 + }, + { + "epoch": 1.9386666666666668, + "grad_norm": 0.0037779631093144417, + "learning_rate": 3.501918808333453e-06, + "loss": 0.465728759765625, + "step": 137 + }, + { + "epoch": 1.952888888888889, + "grad_norm": 0.003967550117522478, + "learning_rate": 3.4236624937386874e-06, + "loss": 0.4686010777950287, + "step": 138 + }, + { + "epoch": 1.967111111111111, + "grad_norm": 0.00354909454472363, + "learning_rate": 3.3458326333759927e-06, + "loss": 0.43872472643852234, + "step": 139 + }, + { + "epoch": 1.9813333333333332, + "grad_norm": 0.00383565342053771, + "learning_rate": 3.268450282935026e-06, + "loss": 0.4455334544181824, + "step": 140 + }, + { + "epoch": 1.9955555555555555, + "grad_norm": 0.0037798655685037374, + "learning_rate": 3.1915363770384223e-06, + "loss": 0.43381941318511963, + "step": 141 + }, + { + "epoch": 2.0, + "grad_norm": 0.00621951324865222, + "learning_rate": 3.1151117235782346e-06, + "loss": 0.4559262990951538, + "step": 142 + } + ], + "logging_steps": 1, + "max_steps": 213, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.49085005277823e+17, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/qwen2.5-3b-instruct/checkpoint-142/training_args.bin b/qwen2.5-3b-instruct/checkpoint-142/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..078e4bb631cb6e7996cf1a54cb10c1a5978aa53a --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-142/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c156fe01067afc4b06ba073aac841107d9e7b73798856df6f0417af272038b3d +size 5649 diff --git a/qwen2.5-3b-instruct/checkpoint-213/chat_template.jinja b/qwen2.5-3b-instruct/checkpoint-213/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-213/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/qwen2.5-3b-instruct/checkpoint-213/config.json b/qwen2.5-3b-instruct/checkpoint-213/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4814292863d1e5aa2f4fda1db18f4082cd61aa93 --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-213/config.json @@ -0,0 +1,69 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 11008, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 70, + "model_type": "qwen2", + "num_attention_heads": 16, + "num_hidden_layers": 36, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/qwen2.5-3b-instruct/checkpoint-213/generation_config.json b/qwen2.5-3b-instruct/checkpoint-213/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..47205fbf3386a215e7c2b9d62df6576f49b77e7b --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-213/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.05, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.2.0" +} diff --git a/qwen2.5-3b-instruct/checkpoint-213/model.safetensors b/qwen2.5-3b-instruct/checkpoint-213/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d60af65d2fc092fb0b22b79c826e7c7484a6a18c --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-213/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a3c8407d8c355dd9ca4bd171fb93c8bc925caeb151fcf8fd44b85decaa7a1a5 +size 12343804296 diff --git a/qwen2.5-3b-instruct/checkpoint-213/optimizer.pt b/qwen2.5-3b-instruct/checkpoint-213/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ee3fbc0c65d44ae425255fedab850842f051c16 --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-213/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39f1ee063036ba59301a10ce2cd87ad2cbc5b2e8217a10a35f3cb02f3da48d58 +size 24687895753 diff --git a/qwen2.5-3b-instruct/checkpoint-213/rng_state_0.pth b/qwen2.5-3b-instruct/checkpoint-213/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..870021e3fa5ac35c2f711adf0c93a556ab4842da --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-213/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95e5fc2074c0df31522a514f862c86cb00d71c946a7f15cc9ec0e53a69fb28a7 +size 14917 diff --git a/qwen2.5-3b-instruct/checkpoint-213/rng_state_1.pth b/qwen2.5-3b-instruct/checkpoint-213/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..21f20da5eb1da017f08aaa88bd19cf24d40e3fbf --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-213/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e7153eae67b6c9232a41bc996a2bf5b83229b8c7230d61911ac0fd40e64154e +size 14917 diff --git a/qwen2.5-3b-instruct/checkpoint-213/scheduler.pt b/qwen2.5-3b-instruct/checkpoint-213/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..39cf87775c3a71d10e7311e0cab769e112f357ba --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-213/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c38a67b98c17be81d4c6d00b60029cdedd43e9aba35ca28ec28ef946c400f268 +size 1465 diff --git a/qwen2.5-3b-instruct/checkpoint-213/tokenizer.json b/qwen2.5-3b-instruct/checkpoint-213/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..34510ff0037cd50428af467a17ead5a96140a32c --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-213/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/qwen2.5-3b-instruct/checkpoint-213/tokenizer_config.json b/qwen2.5-3b-instruct/checkpoint-213/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..145e2c7a68b109ae0253dda29fc2e66cb01b4b85 --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-213/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen2.5-3b-instruct/checkpoint-213/trainer_state.json b/qwen2.5-3b-instruct/checkpoint-213/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..43c6af31dc7a89c4f38264ca3cc64b60964ae783 --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-213/trainer_state.json @@ -0,0 +1,1557 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 50, + "global_step": 213, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.014222222222222223, + "grad_norm": 0.09173738956451416, + "learning_rate": 0.0, + "loss": 1.005206823348999, + "step": 1 + }, + { + "epoch": 0.028444444444444446, + "grad_norm": 0.09365915507078171, + "learning_rate": 4.5454545454545457e-07, + "loss": 1.0057916641235352, + "step": 2 + }, + { + "epoch": 0.042666666666666665, + "grad_norm": 0.0689043328166008, + "learning_rate": 9.090909090909091e-07, + "loss": 0.9406883716583252, + "step": 3 + }, + { + "epoch": 0.05688888888888889, + "grad_norm": 0.04593552276492119, + "learning_rate": 1.3636363636363636e-06, + "loss": 0.8941981792449951, + "step": 4 + }, + { + "epoch": 0.07111111111111111, + "grad_norm": 0.03204488381743431, + "learning_rate": 1.8181818181818183e-06, + "loss": 0.7926980257034302, + "step": 5 + }, + { + "epoch": 0.08533333333333333, + "grad_norm": 0.027753565460443497, + "learning_rate": 2.2727272727272728e-06, + "loss": 0.8305954933166504, + "step": 6 + }, + { + "epoch": 0.09955555555555555, + "grad_norm": 0.03171968460083008, + "learning_rate": 2.7272727272727272e-06, + "loss": 0.7948952913284302, + "step": 7 + }, + { + "epoch": 0.11377777777777778, + "grad_norm": 0.026748575270175934, + "learning_rate": 3.181818181818182e-06, + "loss": 0.7530462741851807, + "step": 8 + }, + { + "epoch": 0.128, + "grad_norm": 0.023413579910993576, + "learning_rate": 3.6363636363636366e-06, + "loss": 0.7157606482505798, + "step": 9 + }, + { + "epoch": 0.14222222222222222, + "grad_norm": 0.020688239485025406, + "learning_rate": 4.0909090909090915e-06, + "loss": 0.7008587718009949, + "step": 10 + }, + { + "epoch": 0.15644444444444444, + "grad_norm": 0.014185823500156403, + "learning_rate": 4.5454545454545455e-06, + "loss": 0.6668463945388794, + "step": 11 + }, + { + "epoch": 0.17066666666666666, + "grad_norm": 0.012938937172293663, + "learning_rate": 5e-06, + "loss": 0.6507903337478638, + "step": 12 + }, + { + "epoch": 0.18488888888888888, + "grad_norm": 0.012045802548527718, + "learning_rate": 5.4545454545454545e-06, + "loss": 0.6217454671859741, + "step": 13 + }, + { + "epoch": 0.1991111111111111, + "grad_norm": 0.009660671465098858, + "learning_rate": 5.90909090909091e-06, + "loss": 0.6269917488098145, + "step": 14 + }, + { + "epoch": 0.21333333333333335, + "grad_norm": 0.008694784715771675, + "learning_rate": 6.363636363636364e-06, + "loss": 0.5836964845657349, + "step": 15 + }, + { + "epoch": 0.22755555555555557, + "grad_norm": 0.008789200335741043, + "learning_rate": 6.818181818181818e-06, + "loss": 0.6187424659729004, + "step": 16 + }, + { + "epoch": 0.24177777777777779, + "grad_norm": 0.008197025395929813, + "learning_rate": 7.272727272727273e-06, + "loss": 0.5914188027381897, + "step": 17 + }, + { + "epoch": 0.256, + "grad_norm": 0.007580601144582033, + "learning_rate": 7.727272727272727e-06, + "loss": 0.5802223682403564, + "step": 18 + }, + { + "epoch": 0.2702222222222222, + "grad_norm": 0.007013861555606127, + "learning_rate": 8.181818181818183e-06, + "loss": 0.5884265899658203, + "step": 19 + }, + { + "epoch": 0.28444444444444444, + "grad_norm": 0.00683254050090909, + "learning_rate": 8.636363636363637e-06, + "loss": 0.5850374698638916, + "step": 20 + }, + { + "epoch": 0.2986666666666667, + "grad_norm": 0.006111981347203255, + "learning_rate": 9.090909090909091e-06, + "loss": 0.5651419162750244, + "step": 21 + }, + { + "epoch": 0.3128888888888889, + "grad_norm": 0.005564771592617035, + "learning_rate": 9.545454545454547e-06, + "loss": 0.553795576095581, + "step": 22 + }, + { + "epoch": 0.32711111111111113, + "grad_norm": 0.0057683708146214485, + "learning_rate": 1e-05, + "loss": 0.5542658567428589, + "step": 23 + }, + { + "epoch": 0.3413333333333333, + "grad_norm": 0.006690182723104954, + "learning_rate": 9.999323662872998e-06, + "loss": 0.5586982369422913, + "step": 24 + }, + { + "epoch": 0.35555555555555557, + "grad_norm": 0.006314165890216827, + "learning_rate": 9.99729483446475e-06, + "loss": 0.5584714412689209, + "step": 25 + }, + { + "epoch": 0.36977777777777776, + "grad_norm": 0.005813857074826956, + "learning_rate": 9.993914063644053e-06, + "loss": 0.5466614961624146, + "step": 26 + }, + { + "epoch": 0.384, + "grad_norm": 0.005381743889302015, + "learning_rate": 9.989182265027232e-06, + "loss": 0.5380265712738037, + "step": 27 + }, + { + "epoch": 0.3982222222222222, + "grad_norm": 0.0049252682365477085, + "learning_rate": 9.98310071873072e-06, + "loss": 0.535520613193512, + "step": 28 + }, + { + "epoch": 0.41244444444444445, + "grad_norm": 0.0048560877330601215, + "learning_rate": 9.975671070024741e-06, + "loss": 0.5377035140991211, + "step": 29 + }, + { + "epoch": 0.4266666666666667, + "grad_norm": 0.004911178257316351, + "learning_rate": 9.966895328888195e-06, + "loss": 0.5570093989372253, + "step": 30 + }, + { + "epoch": 0.4408888888888889, + "grad_norm": 0.004746970720589161, + "learning_rate": 9.956775869464901e-06, + "loss": 0.5285611748695374, + "step": 31 + }, + { + "epoch": 0.45511111111111113, + "grad_norm": 0.0047261337749660015, + "learning_rate": 9.945315429421307e-06, + "loss": 0.5199858546257019, + "step": 32 + }, + { + "epoch": 0.4693333333333333, + "grad_norm": 0.004447608254849911, + "learning_rate": 9.932517109205849e-06, + "loss": 0.5250071287155151, + "step": 33 + }, + { + "epoch": 0.48355555555555557, + "grad_norm": 0.004617375787347555, + "learning_rate": 9.918384371210178e-06, + "loss": 0.5516105890274048, + "step": 34 + }, + { + "epoch": 0.49777777777777776, + "grad_norm": 0.004378544166684151, + "learning_rate": 9.902921038832456e-06, + "loss": 0.5298895835876465, + "step": 35 + }, + { + "epoch": 0.512, + "grad_norm": 0.004164176061749458, + "learning_rate": 9.886131295443003e-06, + "loss": 0.5044555068016052, + "step": 36 + }, + { + "epoch": 0.5262222222222223, + "grad_norm": 0.004092546179890633, + "learning_rate": 9.868019683252543e-06, + "loss": 0.5102939009666443, + "step": 37 + }, + { + "epoch": 0.5404444444444444, + "grad_norm": 0.004554086830466986, + "learning_rate": 9.848591102083375e-06, + "loss": 0.5421494245529175, + "step": 38 + }, + { + "epoch": 0.5546666666666666, + "grad_norm": 0.004217132460325956, + "learning_rate": 9.82785080804381e-06, + "loss": 0.5239347219467163, + "step": 39 + }, + { + "epoch": 0.5688888888888889, + "grad_norm": 0.004286583047360182, + "learning_rate": 9.805804412106197e-06, + "loss": 0.5437066555023193, + "step": 40 + }, + { + "epoch": 0.5831111111111111, + "grad_norm": 0.004261231981217861, + "learning_rate": 9.782457878588977e-06, + "loss": 0.5146951079368591, + "step": 41 + }, + { + "epoch": 0.5973333333333334, + "grad_norm": 0.004209685139358044, + "learning_rate": 9.75781752354311e-06, + "loss": 0.4995099604129791, + "step": 42 + }, + { + "epoch": 0.6115555555555555, + "grad_norm": 0.0039211539551615715, + "learning_rate": 9.731890013043367e-06, + "loss": 0.5249044299125671, + "step": 43 + }, + { + "epoch": 0.6257777777777778, + "grad_norm": 0.0038641595747321844, + "learning_rate": 9.704682361384941e-06, + "loss": 0.4980819821357727, + "step": 44 + }, + { + "epoch": 0.64, + "grad_norm": 0.004069323651492596, + "learning_rate": 9.676201929185809e-06, + "loss": 0.5113821625709534, + "step": 45 + }, + { + "epoch": 0.6542222222222223, + "grad_norm": 0.0038530679885298014, + "learning_rate": 9.646456421395447e-06, + "loss": 0.5072203278541565, + "step": 46 + }, + { + "epoch": 0.6684444444444444, + "grad_norm": 0.003951605875045061, + "learning_rate": 9.615453885210368e-06, + "loss": 0.509921669960022, + "step": 47 + }, + { + "epoch": 0.6826666666666666, + "grad_norm": 0.0038154355715960264, + "learning_rate": 9.583202707897075e-06, + "loss": 0.49035143852233887, + "step": 48 + }, + { + "epoch": 0.6968888888888889, + "grad_norm": 0.004143900703638792, + "learning_rate": 9.549711614523007e-06, + "loss": 0.4858332574367523, + "step": 49 + }, + { + "epoch": 0.7111111111111111, + "grad_norm": 0.0038300366140902042, + "learning_rate": 9.514989665596114e-06, + "loss": 0.49062296748161316, + "step": 50 + }, + { + "epoch": 0.7111111111111111, + "eval_loss": 0.5049259662628174, + "eval_runtime": 38.0442, + "eval_samples_per_second": 5.257, + "eval_steps_per_second": 2.629, + "step": 50 + }, + { + "epoch": 0.7253333333333334, + "grad_norm": 0.003965203184634447, + "learning_rate": 9.479046254613673e-06, + "loss": 0.5013079643249512, + "step": 51 + }, + { + "epoch": 0.7395555555555555, + "grad_norm": 0.003626414341852069, + "learning_rate": 9.441891105521005e-06, + "loss": 0.46776771545410156, + "step": 52 + }, + { + "epoch": 0.7537777777777778, + "grad_norm": 0.003647371893748641, + "learning_rate": 9.40353427008083e-06, + "loss": 0.4870305061340332, + "step": 53 + }, + { + "epoch": 0.768, + "grad_norm": 0.0036924544256180525, + "learning_rate": 9.3639861251539e-06, + "loss": 0.48354169726371765, + "step": 54 + }, + { + "epoch": 0.7822222222222223, + "grad_norm": 0.0037534767761826515, + "learning_rate": 9.323257369891702e-06, + "loss": 0.5097652673721313, + "step": 55 + }, + { + "epoch": 0.7964444444444444, + "grad_norm": 0.003901001764461398, + "learning_rate": 9.281359022841966e-06, + "loss": 0.5023170709609985, + "step": 56 + }, + { + "epoch": 0.8106666666666666, + "grad_norm": 0.0037305736914277077, + "learning_rate": 9.238302418967757e-06, + "loss": 0.49597933888435364, + "step": 57 + }, + { + "epoch": 0.8248888888888889, + "grad_norm": 0.003764534369111061, + "learning_rate": 9.194099206580981e-06, + "loss": 0.504522442817688, + "step": 58 + }, + { + "epoch": 0.8391111111111111, + "grad_norm": 0.004040914587676525, + "learning_rate": 9.14876134419111e-06, + "loss": 0.49795830249786377, + "step": 59 + }, + { + "epoch": 0.8533333333333334, + "grad_norm": 0.003855582093819976, + "learning_rate": 9.102301097269974e-06, + "loss": 0.49624279141426086, + "step": 60 + }, + { + "epoch": 0.8675555555555555, + "grad_norm": 0.003851535264402628, + "learning_rate": 9.05473103493355e-06, + "loss": 0.47714120149612427, + "step": 61 + }, + { + "epoch": 0.8817777777777778, + "grad_norm": 0.004122862592339516, + "learning_rate": 9.006064026541549e-06, + "loss": 0.5046075582504272, + "step": 62 + }, + { + "epoch": 0.896, + "grad_norm": 0.0036710964050143957, + "learning_rate": 8.956313238215824e-06, + "loss": 0.45711296796798706, + "step": 63 + }, + { + "epoch": 0.9102222222222223, + "grad_norm": 0.004037045873701572, + "learning_rate": 8.905492129278478e-06, + "loss": 0.4839310050010681, + "step": 64 + }, + { + "epoch": 0.9244444444444444, + "grad_norm": 0.004081980790942907, + "learning_rate": 8.85361444861063e-06, + "loss": 0.4877704381942749, + "step": 65 + }, + { + "epoch": 0.9386666666666666, + "grad_norm": 0.0037577205803245306, + "learning_rate": 8.800694230932885e-06, + "loss": 0.49369895458221436, + "step": 66 + }, + { + "epoch": 0.9528888888888889, + "grad_norm": 0.0037781032733619213, + "learning_rate": 8.74674579300843e-06, + "loss": 0.4812237620353699, + "step": 67 + }, + { + "epoch": 0.9671111111111111, + "grad_norm": 0.0046208929270505905, + "learning_rate": 8.691783729769874e-06, + "loss": 0.4722365736961365, + "step": 68 + }, + { + "epoch": 0.9813333333333333, + "grad_norm": 0.004026597365736961, + "learning_rate": 8.635822910370793e-06, + "loss": 0.4962988495826721, + "step": 69 + }, + { + "epoch": 0.9955555555555555, + "grad_norm": 0.004095569718629122, + "learning_rate": 8.578878474163115e-06, + "loss": 0.45991775393486023, + "step": 70 + }, + { + "epoch": 1.0, + "grad_norm": 0.006300764624029398, + "learning_rate": 8.520965826601394e-06, + "loss": 0.4934382140636444, + "step": 71 + }, + { + "epoch": 1.0142222222222221, + "grad_norm": 0.00428728898987174, + "learning_rate": 8.462100635075097e-06, + "loss": 0.4847542643547058, + "step": 72 + }, + { + "epoch": 1.0284444444444445, + "grad_norm": 0.004186135251075029, + "learning_rate": 8.40229882467003e-06, + "loss": 0.4636518955230713, + "step": 73 + }, + { + "epoch": 1.0426666666666666, + "grad_norm": 0.003916833084076643, + "learning_rate": 8.341576573860049e-06, + "loss": 0.44187992811203003, + "step": 74 + }, + { + "epoch": 1.056888888888889, + "grad_norm": 0.004144088365137577, + "learning_rate": 8.279950310130218e-06, + "loss": 0.467815637588501, + "step": 75 + }, + { + "epoch": 1.0711111111111111, + "grad_norm": 0.004077525809407234, + "learning_rate": 8.2174367055326e-06, + "loss": 0.451981782913208, + "step": 76 + }, + { + "epoch": 1.0853333333333333, + "grad_norm": 0.004103589337319136, + "learning_rate": 8.154052672175888e-06, + "loss": 0.4449639320373535, + "step": 77 + }, + { + "epoch": 1.0995555555555556, + "grad_norm": 0.00439891591668129, + "learning_rate": 8.08981535765009e-06, + "loss": 0.47661933302879333, + "step": 78 + }, + { + "epoch": 1.1137777777777778, + "grad_norm": 0.004592400509864092, + "learning_rate": 8.024742140387506e-06, + "loss": 0.45776277780532837, + "step": 79 + }, + { + "epoch": 1.1280000000000001, + "grad_norm": 0.0037961499765515327, + "learning_rate": 7.95885062496126e-06, + "loss": 0.44757023453712463, + "step": 80 + }, + { + "epoch": 1.1422222222222222, + "grad_norm": 0.003865304868668318, + "learning_rate": 7.892158637322647e-06, + "loss": 0.419955849647522, + "step": 81 + }, + { + "epoch": 1.1564444444444444, + "grad_norm": 0.00407387875020504, + "learning_rate": 7.824684219978591e-06, + "loss": 0.4457795023918152, + "step": 82 + }, + { + "epoch": 1.1706666666666667, + "grad_norm": 0.00370654184371233, + "learning_rate": 7.756445627110523e-06, + "loss": 0.45605674386024475, + "step": 83 + }, + { + "epoch": 1.1848888888888889, + "grad_norm": 0.003941345028579235, + "learning_rate": 7.68746131963598e-06, + "loss": 0.46289366483688354, + "step": 84 + }, + { + "epoch": 1.199111111111111, + "grad_norm": 0.0039403438568115234, + "learning_rate": 7.6177499602143e-06, + "loss": 0.45568281412124634, + "step": 85 + }, + { + "epoch": 1.2133333333333334, + "grad_norm": 0.003944277763366699, + "learning_rate": 7.547330408197695e-06, + "loss": 0.44618040323257446, + "step": 86 + }, + { + "epoch": 1.2275555555555555, + "grad_norm": 0.0040544732473790646, + "learning_rate": 7.476221714529167e-06, + "loss": 0.45724251866340637, + "step": 87 + }, + { + "epoch": 1.2417777777777779, + "grad_norm": 0.004533152561634779, + "learning_rate": 7.404443116588548e-06, + "loss": 0.4806937575340271, + "step": 88 + }, + { + "epoch": 1.256, + "grad_norm": 0.004002939909696579, + "learning_rate": 7.332014032988123e-06, + "loss": 0.4561174213886261, + "step": 89 + }, + { + "epoch": 1.2702222222222221, + "grad_norm": 0.003825302002951503, + "learning_rate": 7.2589540583192165e-06, + "loss": 0.4558497667312622, + "step": 90 + }, + { + "epoch": 1.2844444444444445, + "grad_norm": 0.00397696765139699, + "learning_rate": 7.185282957851175e-06, + "loss": 0.4397018551826477, + "step": 91 + }, + { + "epoch": 1.2986666666666666, + "grad_norm": 0.004289410542696714, + "learning_rate": 7.111020662184174e-06, + "loss": 0.45822006464004517, + "step": 92 + }, + { + "epoch": 1.3128888888888888, + "grad_norm": 0.004002283327281475, + "learning_rate": 7.036187261857289e-06, + "loss": 0.4479483366012573, + "step": 93 + }, + { + "epoch": 1.3271111111111111, + "grad_norm": 0.004231768660247326, + "learning_rate": 6.960803001913315e-06, + "loss": 0.4738925099372864, + "step": 94 + }, + { + "epoch": 1.3413333333333333, + "grad_norm": 0.0040559363551437855, + "learning_rate": 6.884888276421766e-06, + "loss": 0.4614931643009186, + "step": 95 + }, + { + "epoch": 1.3555555555555556, + "grad_norm": 0.003587141167372465, + "learning_rate": 6.8084636229615786e-06, + "loss": 0.44328922033309937, + "step": 96 + }, + { + "epoch": 1.3697777777777778, + "grad_norm": 0.004009770695120096, + "learning_rate": 6.731549717064975e-06, + "loss": 0.44612404704093933, + "step": 97 + }, + { + "epoch": 1.384, + "grad_norm": 0.004132336005568504, + "learning_rate": 6.654167366624009e-06, + "loss": 0.4554000794887543, + "step": 98 + }, + { + "epoch": 1.3982222222222223, + "grad_norm": 0.0041816201992332935, + "learning_rate": 6.576337506261314e-06, + "loss": 0.45616668462753296, + "step": 99 + }, + { + "epoch": 1.4124444444444444, + "grad_norm": 0.003568898420780897, + "learning_rate": 6.498081191666549e-06, + "loss": 0.45816588401794434, + "step": 100 + }, + { + "epoch": 1.4124444444444444, + "eval_loss": 0.4757018983364105, + "eval_runtime": 39.6302, + "eval_samples_per_second": 5.047, + "eval_steps_per_second": 2.523, + "step": 100 + }, + { + "epoch": 1.4266666666666667, + "grad_norm": 0.004014050588011742, + "learning_rate": 6.419419593900109e-06, + "loss": 0.4430053234100342, + "step": 101 + }, + { + "epoch": 1.4408888888888889, + "grad_norm": 0.0039361282251775265, + "learning_rate": 6.340373993665607e-06, + "loss": 0.43451493978500366, + "step": 102 + }, + { + "epoch": 1.455111111111111, + "grad_norm": 0.0040025534108281136, + "learning_rate": 6.2609657755527135e-06, + "loss": 0.4296550154685974, + "step": 103 + }, + { + "epoch": 1.4693333333333334, + "grad_norm": 0.004081938415765762, + "learning_rate": 6.1812164222518626e-06, + "loss": 0.46728163957595825, + "step": 104 + }, + { + "epoch": 1.4835555555555555, + "grad_norm": 0.0043087489902973175, + "learning_rate": 6.101147508742456e-06, + "loss": 0.446887731552124, + "step": 105 + }, + { + "epoch": 1.4977777777777779, + "grad_norm": 0.0037424617912620306, + "learning_rate": 6.020780696456059e-06, + "loss": 0.43969088792800903, + "step": 106 + }, + { + "epoch": 1.512, + "grad_norm": 0.003990524914115667, + "learning_rate": 5.940137727416247e-06, + "loss": 0.4489642381668091, + "step": 107 + }, + { + "epoch": 1.5262222222222221, + "grad_norm": 0.00401579262688756, + "learning_rate": 5.859240418356614e-06, + "loss": 0.4380025863647461, + "step": 108 + }, + { + "epoch": 1.5404444444444443, + "grad_norm": 0.0042803785763680935, + "learning_rate": 5.778110654818602e-06, + "loss": 0.47752195596694946, + "step": 109 + }, + { + "epoch": 1.5546666666666666, + "grad_norm": 0.003838231321424246, + "learning_rate": 5.696770385230679e-06, + "loss": 0.4498862028121948, + "step": 110 + }, + { + "epoch": 1.568888888888889, + "grad_norm": 0.0041479310020804405, + "learning_rate": 5.615241614970546e-06, + "loss": 0.44400298595428467, + "step": 111 + }, + { + "epoch": 1.5831111111111111, + "grad_norm": 0.0038208707701414824, + "learning_rate": 5.533546400411899e-06, + "loss": 0.4457319378852844, + "step": 112 + }, + { + "epoch": 1.5973333333333333, + "grad_norm": 0.0038792924024164677, + "learning_rate": 5.4517068429574215e-06, + "loss": 0.4610831141471863, + "step": 113 + }, + { + "epoch": 1.6115555555555554, + "grad_norm": 0.004012539051473141, + "learning_rate": 5.3697450830595775e-06, + "loss": 0.4425020217895508, + "step": 114 + }, + { + "epoch": 1.6257777777777778, + "grad_norm": 0.003996311221271753, + "learning_rate": 5.287683294230855e-06, + "loss": 0.44555309414863586, + "step": 115 + }, + { + "epoch": 1.6400000000000001, + "grad_norm": 0.0037504970096051693, + "learning_rate": 5.20554367704505e-06, + "loss": 0.417566180229187, + "step": 116 + }, + { + "epoch": 1.6542222222222223, + "grad_norm": 0.003828976536169648, + "learning_rate": 5.123348453131242e-06, + "loss": 0.45680171251296997, + "step": 117 + }, + { + "epoch": 1.6684444444444444, + "grad_norm": 0.0040869563817977905, + "learning_rate": 5.041119859162068e-06, + "loss": 0.4432540535926819, + "step": 118 + }, + { + "epoch": 1.6826666666666665, + "grad_norm": 0.004195951856672764, + "learning_rate": 4.958880140837934e-06, + "loss": 0.44924670457839966, + "step": 119 + }, + { + "epoch": 1.696888888888889, + "grad_norm": 0.003907010890543461, + "learning_rate": 4.876651546868759e-06, + "loss": 0.43826115131378174, + "step": 120 + }, + { + "epoch": 1.7111111111111112, + "grad_norm": 0.003893865505233407, + "learning_rate": 4.794456322954953e-06, + "loss": 0.4698702096939087, + "step": 121 + }, + { + "epoch": 1.7253333333333334, + "grad_norm": 0.003987904172390699, + "learning_rate": 4.7123167057691446e-06, + "loss": 0.4665527045726776, + "step": 122 + }, + { + "epoch": 1.7395555555555555, + "grad_norm": 0.003826166968792677, + "learning_rate": 4.630254916940424e-06, + "loss": 0.46612560749053955, + "step": 123 + }, + { + "epoch": 1.7537777777777777, + "grad_norm": 0.0037139400374144316, + "learning_rate": 4.548293157042581e-06, + "loss": 0.4559047818183899, + "step": 124 + }, + { + "epoch": 1.768, + "grad_norm": 0.0038443440571427345, + "learning_rate": 4.466453599588103e-06, + "loss": 0.4595223069190979, + "step": 125 + }, + { + "epoch": 1.7822222222222224, + "grad_norm": 0.0039060397539287806, + "learning_rate": 4.384758385029457e-06, + "loss": 0.42560791969299316, + "step": 126 + }, + { + "epoch": 1.7964444444444445, + "grad_norm": 0.0041647544130682945, + "learning_rate": 4.3032296147693225e-06, + "loss": 0.43939000368118286, + "step": 127 + }, + { + "epoch": 1.8106666666666666, + "grad_norm": 0.004102814476937056, + "learning_rate": 4.2218893451814005e-06, + "loss": 0.4399089217185974, + "step": 128 + }, + { + "epoch": 1.8248888888888888, + "grad_norm": 0.003889351151883602, + "learning_rate": 4.140759581643386e-06, + "loss": 0.4451609253883362, + "step": 129 + }, + { + "epoch": 1.8391111111111111, + "grad_norm": 0.004501548130065203, + "learning_rate": 4.059862272583755e-06, + "loss": 0.4488486051559448, + "step": 130 + }, + { + "epoch": 1.8533333333333335, + "grad_norm": 0.0038232814986258745, + "learning_rate": 3.979219303543942e-06, + "loss": 0.42662614583969116, + "step": 131 + }, + { + "epoch": 1.8675555555555556, + "grad_norm": 0.003998646512627602, + "learning_rate": 3.898852491257547e-06, + "loss": 0.4547004997730255, + "step": 132 + }, + { + "epoch": 1.8817777777777778, + "grad_norm": 0.00403416994959116, + "learning_rate": 3.818783577748138e-06, + "loss": 0.4510434567928314, + "step": 133 + }, + { + "epoch": 1.896, + "grad_norm": 0.003799683880060911, + "learning_rate": 3.7390342244472886e-06, + "loss": 0.45353883504867554, + "step": 134 + }, + { + "epoch": 1.9102222222222223, + "grad_norm": 0.003921311814337969, + "learning_rate": 3.659626006334395e-06, + "loss": 0.4302864670753479, + "step": 135 + }, + { + "epoch": 1.9244444444444444, + "grad_norm": 0.00398287083953619, + "learning_rate": 3.5805804060998926e-06, + "loss": 0.4398406744003296, + "step": 136 + }, + { + "epoch": 1.9386666666666668, + "grad_norm": 0.0037779631093144417, + "learning_rate": 3.501918808333453e-06, + "loss": 0.465728759765625, + "step": 137 + }, + { + "epoch": 1.952888888888889, + "grad_norm": 0.003967550117522478, + "learning_rate": 3.4236624937386874e-06, + "loss": 0.4686010777950287, + "step": 138 + }, + { + "epoch": 1.967111111111111, + "grad_norm": 0.00354909454472363, + "learning_rate": 3.3458326333759927e-06, + "loss": 0.43872472643852234, + "step": 139 + }, + { + "epoch": 1.9813333333333332, + "grad_norm": 0.00383565342053771, + "learning_rate": 3.268450282935026e-06, + "loss": 0.4455334544181824, + "step": 140 + }, + { + "epoch": 1.9955555555555555, + "grad_norm": 0.0037798655685037374, + "learning_rate": 3.1915363770384223e-06, + "loss": 0.43381941318511963, + "step": 141 + }, + { + "epoch": 2.0, + "grad_norm": 0.00621951324865222, + "learning_rate": 3.1151117235782346e-06, + "loss": 0.4559262990951538, + "step": 142 + }, + { + "epoch": 2.014222222222222, + "grad_norm": 0.004304260015487671, + "learning_rate": 3.0391969980866874e-06, + "loss": 0.43779897689819336, + "step": 143 + }, + { + "epoch": 2.0284444444444443, + "grad_norm": 0.003964607138186693, + "learning_rate": 2.963812738142713e-06, + "loss": 0.4199046492576599, + "step": 144 + }, + { + "epoch": 2.042666666666667, + "grad_norm": 0.004212403669953346, + "learning_rate": 2.8889793378158284e-06, + "loss": 0.43830418586730957, + "step": 145 + }, + { + "epoch": 2.056888888888889, + "grad_norm": 0.0037585427053272724, + "learning_rate": 2.814717042148827e-06, + "loss": 0.4249774217605591, + "step": 146 + }, + { + "epoch": 2.071111111111111, + "grad_norm": 0.003818828146904707, + "learning_rate": 2.7410459416807856e-06, + "loss": 0.41759681701660156, + "step": 147 + }, + { + "epoch": 2.0853333333333333, + "grad_norm": 0.0038647297769784927, + "learning_rate": 2.6679859670118785e-06, + "loss": 0.43880075216293335, + "step": 148 + }, + { + "epoch": 2.0995555555555554, + "grad_norm": 0.0037169770803302526, + "learning_rate": 2.5955568834114523e-06, + "loss": 0.4053642153739929, + "step": 149 + }, + { + "epoch": 2.113777777777778, + "grad_norm": 0.0037585999816656113, + "learning_rate": 2.523778285470835e-06, + "loss": 0.4293328523635864, + "step": 150 + }, + { + "epoch": 2.113777777777778, + "eval_loss": 0.46622127294540405, + "eval_runtime": 42.1452, + "eval_samples_per_second": 4.746, + "eval_steps_per_second": 2.373, + "step": 150 + }, + { + "epoch": 2.128, + "grad_norm": 0.004196058958768845, + "learning_rate": 2.452669591802307e-06, + "loss": 0.4054480195045471, + "step": 151 + }, + { + "epoch": 2.1422222222222222, + "grad_norm": 0.003635152243077755, + "learning_rate": 2.3822500397857016e-06, + "loss": 0.41965848207473755, + "step": 152 + }, + { + "epoch": 2.1564444444444444, + "grad_norm": 0.0042378562502563, + "learning_rate": 2.3125386803640188e-06, + "loss": 0.4354519248008728, + "step": 153 + }, + { + "epoch": 2.1706666666666665, + "grad_norm": 0.003634256776422262, + "learning_rate": 2.243554372889479e-06, + "loss": 0.4178982675075531, + "step": 154 + }, + { + "epoch": 2.1848888888888887, + "grad_norm": 0.003790440736338496, + "learning_rate": 2.1753157800214107e-06, + "loss": 0.4098745584487915, + "step": 155 + }, + { + "epoch": 2.1991111111111112, + "grad_norm": 0.0038817040622234344, + "learning_rate": 2.1078413626773547e-06, + "loss": 0.43672817945480347, + "step": 156 + }, + { + "epoch": 2.2133333333333334, + "grad_norm": 0.003961282316595316, + "learning_rate": 2.0411493750387423e-06, + "loss": 0.39173248410224915, + "step": 157 + }, + { + "epoch": 2.2275555555555555, + "grad_norm": 0.003964588046073914, + "learning_rate": 1.9752578596124955e-06, + "loss": 0.44043296575546265, + "step": 158 + }, + { + "epoch": 2.2417777777777776, + "grad_norm": 0.003892629873007536, + "learning_rate": 1.9101846423499113e-06, + "loss": 0.41284430027008057, + "step": 159 + }, + { + "epoch": 2.2560000000000002, + "grad_norm": 0.004033091943711042, + "learning_rate": 1.8459473278241125e-06, + "loss": 0.41392982006073, + "step": 160 + }, + { + "epoch": 2.2702222222222224, + "grad_norm": 0.003734015626832843, + "learning_rate": 1.7825632944674016e-06, + "loss": 0.41735541820526123, + "step": 161 + }, + { + "epoch": 2.2844444444444445, + "grad_norm": 0.003964587114751339, + "learning_rate": 1.7200496898697832e-06, + "loss": 0.43122196197509766, + "step": 162 + }, + { + "epoch": 2.2986666666666666, + "grad_norm": 0.0036877307575196028, + "learning_rate": 1.6584234261399535e-06, + "loss": 0.41986942291259766, + "step": 163 + }, + { + "epoch": 2.3128888888888888, + "grad_norm": 0.0038088378496468067, + "learning_rate": 1.5977011753299726e-06, + "loss": 0.41601645946502686, + "step": 164 + }, + { + "epoch": 2.327111111111111, + "grad_norm": 0.003943529445677996, + "learning_rate": 1.5378993649249053e-06, + "loss": 0.4430370330810547, + "step": 165 + }, + { + "epoch": 2.3413333333333335, + "grad_norm": 0.0038139254320412874, + "learning_rate": 1.4790341733986085e-06, + "loss": 0.41499751806259155, + "step": 166 + }, + { + "epoch": 2.3555555555555556, + "grad_norm": 0.003613486187532544, + "learning_rate": 1.4211215258368866e-06, + "loss": 0.40212416648864746, + "step": 167 + }, + { + "epoch": 2.3697777777777778, + "grad_norm": 0.0037240793462842703, + "learning_rate": 1.3641770896292083e-06, + "loss": 0.40819668769836426, + "step": 168 + }, + { + "epoch": 2.384, + "grad_norm": 0.003915995359420776, + "learning_rate": 1.3082162702301276e-06, + "loss": 0.4199577569961548, + "step": 169 + }, + { + "epoch": 2.398222222222222, + "grad_norm": 0.0038723198231309652, + "learning_rate": 1.2532542069915722e-06, + "loss": 0.4247972071170807, + "step": 170 + }, + { + "epoch": 2.4124444444444446, + "grad_norm": 0.0036193751730024815, + "learning_rate": 1.1993057690671174e-06, + "loss": 0.41396448016166687, + "step": 171 + }, + { + "epoch": 2.4266666666666667, + "grad_norm": 0.003564434824511409, + "learning_rate": 1.1463855513893695e-06, + "loss": 0.43425244092941284, + "step": 172 + }, + { + "epoch": 2.440888888888889, + "grad_norm": 0.003745888127014041, + "learning_rate": 1.0945078707215224e-06, + "loss": 0.42147308588027954, + "step": 173 + }, + { + "epoch": 2.455111111111111, + "grad_norm": 0.003768347902223468, + "learning_rate": 1.0436867617841768e-06, + "loss": 0.4213670492172241, + "step": 174 + }, + { + "epoch": 2.469333333333333, + "grad_norm": 0.0036286180838942528, + "learning_rate": 9.939359734584552e-07, + "loss": 0.4306361675262451, + "step": 175 + }, + { + "epoch": 2.4835555555555557, + "grad_norm": 0.0037941799964755774, + "learning_rate": 9.452689650664515e-07, + "loss": 0.4333479404449463, + "step": 176 + }, + { + "epoch": 2.497777777777778, + "grad_norm": 0.003679306246340275, + "learning_rate": 8.976989027300265e-07, + "loss": 0.4236246347427368, + "step": 177 + }, + { + "epoch": 2.512, + "grad_norm": 0.003801122773438692, + "learning_rate": 8.512386558088919e-07, + "loss": 0.4386535882949829, + "step": 178 + }, + { + "epoch": 2.526222222222222, + "grad_norm": 0.00364946317858994, + "learning_rate": 8.059007934190194e-07, + "loss": 0.42091357707977295, + "step": 179 + }, + { + "epoch": 2.5404444444444443, + "grad_norm": 0.003728381125256419, + "learning_rate": 7.61697581032243e-07, + "loss": 0.44629859924316406, + "step": 180 + }, + { + "epoch": 2.554666666666667, + "grad_norm": 0.003702825866639614, + "learning_rate": 7.186409771580355e-07, + "loss": 0.4341578185558319, + "step": 181 + }, + { + "epoch": 2.568888888888889, + "grad_norm": 0.00379473646171391, + "learning_rate": 6.76742630108298e-07, + "loss": 0.4224834442138672, + "step": 182 + }, + { + "epoch": 2.583111111111111, + "grad_norm": 0.003699215827509761, + "learning_rate": 6.360138748461015e-07, + "loss": 0.4217371344566345, + "step": 183 + }, + { + "epoch": 2.5973333333333333, + "grad_norm": 0.0036750342696905136, + "learning_rate": 5.964657299191712e-07, + "loss": 0.4117254614830017, + "step": 184 + }, + { + "epoch": 2.6115555555555554, + "grad_norm": 0.003708022413775325, + "learning_rate": 5.581088944789953e-07, + "loss": 0.4190234839916229, + "step": 185 + }, + { + "epoch": 2.6257777777777775, + "grad_norm": 0.00363738089799881, + "learning_rate": 5.20953745386329e-07, + "loss": 0.4357784390449524, + "step": 186 + }, + { + "epoch": 2.64, + "grad_norm": 0.0036364425905048847, + "learning_rate": 4.850103344038853e-07, + "loss": 0.43076783418655396, + "step": 187 + }, + { + "epoch": 2.6542222222222223, + "grad_norm": 0.003594007110223174, + "learning_rate": 4.502883854769935e-07, + "loss": 0.4010687470436096, + "step": 188 + }, + { + "epoch": 2.6684444444444444, + "grad_norm": 0.00375847308896482, + "learning_rate": 4.167972921029262e-07, + "loss": 0.4280475378036499, + "step": 189 + }, + { + "epoch": 2.6826666666666665, + "grad_norm": 0.003887615166604519, + "learning_rate": 3.8454611478963235e-07, + "loss": 0.4384361803531647, + "step": 190 + }, + { + "epoch": 2.696888888888889, + "grad_norm": 0.0036584900226444006, + "learning_rate": 3.535435786045538e-07, + "loss": 0.4107455611228943, + "step": 191 + }, + { + "epoch": 2.7111111111111112, + "grad_norm": 0.003653391497209668, + "learning_rate": 3.237980708141919e-07, + "loss": 0.42101866006851196, + "step": 192 + }, + { + "epoch": 2.7253333333333334, + "grad_norm": 0.003794454736635089, + "learning_rate": 2.9531763861505967e-07, + "loss": 0.43159422278404236, + "step": 193 + }, + { + "epoch": 2.7395555555555555, + "grad_norm": 0.003839928889647126, + "learning_rate": 2.681099869566328e-07, + "loss": 0.4079360365867615, + "step": 194 + }, + { + "epoch": 2.7537777777777777, + "grad_norm": 0.0035740381572395563, + "learning_rate": 2.4218247645689306e-07, + "loss": 0.4277493357658386, + "step": 195 + }, + { + "epoch": 2.768, + "grad_norm": 0.0035776360891759396, + "learning_rate": 2.175421214110235e-07, + "loss": 0.4117092490196228, + "step": 196 + }, + { + "epoch": 2.7822222222222224, + "grad_norm": 0.0038700539153069258, + "learning_rate": 1.941955878938029e-07, + "loss": 0.42654484510421753, + "step": 197 + }, + { + "epoch": 2.7964444444444445, + "grad_norm": 0.003715057857334614, + "learning_rate": 1.721491919561913e-07, + "loss": 0.43574994802474976, + "step": 198 + }, + { + "epoch": 2.8106666666666666, + "grad_norm": 0.0036776140332221985, + "learning_rate": 1.514088979166256e-07, + "loss": 0.41729098558425903, + "step": 199 + }, + { + "epoch": 2.824888888888889, + "grad_norm": 0.003615819849073887, + "learning_rate": 1.3198031674745814e-07, + "loss": 0.42997270822525024, + "step": 200 + }, + { + "epoch": 2.824888888888889, + "eval_loss": 0.4636807143688202, + "eval_runtime": 38.1939, + "eval_samples_per_second": 5.236, + "eval_steps_per_second": 2.618, + "step": 200 + }, + { + "epoch": 2.8391111111111114, + "grad_norm": 0.00375831825658679, + "learning_rate": 1.138687045569975e-07, + "loss": 0.42529386281967163, + "step": 201 + }, + { + "epoch": 2.8533333333333335, + "grad_norm": 0.0035394008737057447, + "learning_rate": 9.70789611675449e-08, + "loss": 0.4284723401069641, + "step": 202 + }, + { + "epoch": 2.8675555555555556, + "grad_norm": 0.0037534094881266356, + "learning_rate": 8.161562878982399e-08, + "loss": 0.41172438859939575, + "step": 203 + }, + { + "epoch": 2.8817777777777778, + "grad_norm": 0.003908019047230482, + "learning_rate": 6.74828907941516e-08, + "loss": 0.43681836128234863, + "step": 204 + }, + { + "epoch": 2.896, + "grad_norm": 0.003631765488535166, + "learning_rate": 5.468457057869358e-08, + "loss": 0.43698325753211975, + "step": 205 + }, + { + "epoch": 2.910222222222222, + "grad_norm": 0.003675674321129918, + "learning_rate": 4.322413053509944e-08, + "loss": 0.4135843515396118, + "step": 206 + }, + { + "epoch": 2.924444444444444, + "grad_norm": 0.0037287427112460136, + "learning_rate": 3.31046711118066e-08, + "loss": 0.41366809606552124, + "step": 207 + }, + { + "epoch": 2.9386666666666668, + "grad_norm": 0.0037086675874888897, + "learning_rate": 2.4328929975260262e-08, + "loss": 0.4246295094490051, + "step": 208 + }, + { + "epoch": 2.952888888888889, + "grad_norm": 0.0035672816447913647, + "learning_rate": 1.6899281269279756e-08, + "loss": 0.4110904037952423, + "step": 209 + }, + { + "epoch": 2.967111111111111, + "grad_norm": 0.0037544071674346924, + "learning_rate": 1.0817734972768946e-08, + "loss": 0.4201211929321289, + "step": 210 + }, + { + "epoch": 2.981333333333333, + "grad_norm": 0.004007895011454821, + "learning_rate": 6.085936355947897e-09, + "loss": 0.43755394220352173, + "step": 211 + }, + { + "epoch": 2.9955555555555557, + "grad_norm": 0.003630666760727763, + "learning_rate": 2.7051655352494654e-09, + "loss": 0.4097408652305603, + "step": 212 + }, + { + "epoch": 3.0, + "grad_norm": 0.00691953394562006, + "learning_rate": 6.763371270035457e-10, + "loss": 0.5003441572189331, + "step": 213 + } + ], + "logging_steps": 1, + "max_steps": 213, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 5.2362750737986355e+17, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/qwen2.5-3b-instruct/checkpoint-213/training_args.bin b/qwen2.5-3b-instruct/checkpoint-213/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..078e4bb631cb6e7996cf1a54cb10c1a5978aa53a --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-213/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c156fe01067afc4b06ba073aac841107d9e7b73798856df6f0417af272038b3d +size 5649 diff --git a/qwen2.5-3b-instruct/checkpoint-71/chat_template.jinja b/qwen2.5-3b-instruct/checkpoint-71/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-71/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/qwen2.5-3b-instruct/checkpoint-71/config.json b/qwen2.5-3b-instruct/checkpoint-71/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4814292863d1e5aa2f4fda1db18f4082cd61aa93 --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-71/config.json @@ -0,0 +1,69 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 11008, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 70, + "model_type": "qwen2", + "num_attention_heads": 16, + "num_hidden_layers": 36, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/qwen2.5-3b-instruct/checkpoint-71/generation_config.json b/qwen2.5-3b-instruct/checkpoint-71/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..47205fbf3386a215e7c2b9d62df6576f49b77e7b --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-71/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.05, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.2.0" +} diff --git a/qwen2.5-3b-instruct/checkpoint-71/model.safetensors b/qwen2.5-3b-instruct/checkpoint-71/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fa8facc6427ab3cee5f7c15fafb92eba5b217452 --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-71/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a7dad42a8618609ce94b577df9ac290cb8c6b2c73e70dfe05e96dc1fae770c2 +size 12343804296 diff --git a/qwen2.5-3b-instruct/checkpoint-71/optimizer.pt b/qwen2.5-3b-instruct/checkpoint-71/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..12a13c61a7d5340f79ca99642f9d60e775072d04 --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-71/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd111a153a80bd01394b36f569913b3e7966ea3f311a07bc8b2c22a637f80c54 +size 24687895753 diff --git a/qwen2.5-3b-instruct/checkpoint-71/rng_state_0.pth b/qwen2.5-3b-instruct/checkpoint-71/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..870021e3fa5ac35c2f711adf0c93a556ab4842da --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-71/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95e5fc2074c0df31522a514f862c86cb00d71c946a7f15cc9ec0e53a69fb28a7 +size 14917 diff --git a/qwen2.5-3b-instruct/checkpoint-71/rng_state_1.pth b/qwen2.5-3b-instruct/checkpoint-71/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..21f20da5eb1da017f08aaa88bd19cf24d40e3fbf --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-71/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e7153eae67b6c9232a41bc996a2bf5b83229b8c7230d61911ac0fd40e64154e +size 14917 diff --git a/qwen2.5-3b-instruct/checkpoint-71/scheduler.pt b/qwen2.5-3b-instruct/checkpoint-71/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3be1d29bde40f225713ff5084d3d9d69858fcfd --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-71/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c96c28f12671e2f5db493417daefb3ff0b1bc36602843e0d7ef6df3649c27b3 +size 1465 diff --git a/qwen2.5-3b-instruct/checkpoint-71/tokenizer.json b/qwen2.5-3b-instruct/checkpoint-71/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..34510ff0037cd50428af467a17ead5a96140a32c --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-71/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/qwen2.5-3b-instruct/checkpoint-71/tokenizer_config.json b/qwen2.5-3b-instruct/checkpoint-71/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..145e2c7a68b109ae0253dda29fc2e66cb01b4b85 --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-71/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen2.5-3b-instruct/checkpoint-71/trainer_state.json b/qwen2.5-3b-instruct/checkpoint-71/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..16cd1bfaf3f29c508d22f0fef6ef45e0e31fd255 --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-71/trainer_state.json @@ -0,0 +1,539 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 50, + "global_step": 71, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.014222222222222223, + "grad_norm": 0.09173738956451416, + "learning_rate": 0.0, + "loss": 1.005206823348999, + "step": 1 + }, + { + "epoch": 0.028444444444444446, + "grad_norm": 0.09365915507078171, + "learning_rate": 4.5454545454545457e-07, + "loss": 1.0057916641235352, + "step": 2 + }, + { + "epoch": 0.042666666666666665, + "grad_norm": 0.0689043328166008, + "learning_rate": 9.090909090909091e-07, + "loss": 0.9406883716583252, + "step": 3 + }, + { + "epoch": 0.05688888888888889, + "grad_norm": 0.04593552276492119, + "learning_rate": 1.3636363636363636e-06, + "loss": 0.8941981792449951, + "step": 4 + }, + { + "epoch": 0.07111111111111111, + "grad_norm": 0.03204488381743431, + "learning_rate": 1.8181818181818183e-06, + "loss": 0.7926980257034302, + "step": 5 + }, + { + "epoch": 0.08533333333333333, + "grad_norm": 0.027753565460443497, + "learning_rate": 2.2727272727272728e-06, + "loss": 0.8305954933166504, + "step": 6 + }, + { + "epoch": 0.09955555555555555, + "grad_norm": 0.03171968460083008, + "learning_rate": 2.7272727272727272e-06, + "loss": 0.7948952913284302, + "step": 7 + }, + { + "epoch": 0.11377777777777778, + "grad_norm": 0.026748575270175934, + "learning_rate": 3.181818181818182e-06, + "loss": 0.7530462741851807, + "step": 8 + }, + { + "epoch": 0.128, + "grad_norm": 0.023413579910993576, + "learning_rate": 3.6363636363636366e-06, + "loss": 0.7157606482505798, + "step": 9 + }, + { + "epoch": 0.14222222222222222, + "grad_norm": 0.020688239485025406, + "learning_rate": 4.0909090909090915e-06, + "loss": 0.7008587718009949, + "step": 10 + }, + { + "epoch": 0.15644444444444444, + "grad_norm": 0.014185823500156403, + "learning_rate": 4.5454545454545455e-06, + "loss": 0.6668463945388794, + "step": 11 + }, + { + "epoch": 0.17066666666666666, + "grad_norm": 0.012938937172293663, + "learning_rate": 5e-06, + "loss": 0.6507903337478638, + "step": 12 + }, + { + "epoch": 0.18488888888888888, + "grad_norm": 0.012045802548527718, + "learning_rate": 5.4545454545454545e-06, + "loss": 0.6217454671859741, + "step": 13 + }, + { + "epoch": 0.1991111111111111, + "grad_norm": 0.009660671465098858, + "learning_rate": 5.90909090909091e-06, + "loss": 0.6269917488098145, + "step": 14 + }, + { + "epoch": 0.21333333333333335, + "grad_norm": 0.008694784715771675, + "learning_rate": 6.363636363636364e-06, + "loss": 0.5836964845657349, + "step": 15 + }, + { + "epoch": 0.22755555555555557, + "grad_norm": 0.008789200335741043, + "learning_rate": 6.818181818181818e-06, + "loss": 0.6187424659729004, + "step": 16 + }, + { + "epoch": 0.24177777777777779, + "grad_norm": 0.008197025395929813, + "learning_rate": 7.272727272727273e-06, + "loss": 0.5914188027381897, + "step": 17 + }, + { + "epoch": 0.256, + "grad_norm": 0.007580601144582033, + "learning_rate": 7.727272727272727e-06, + "loss": 0.5802223682403564, + "step": 18 + }, + { + "epoch": 0.2702222222222222, + "grad_norm": 0.007013861555606127, + "learning_rate": 8.181818181818183e-06, + "loss": 0.5884265899658203, + "step": 19 + }, + { + "epoch": 0.28444444444444444, + "grad_norm": 0.00683254050090909, + "learning_rate": 8.636363636363637e-06, + "loss": 0.5850374698638916, + "step": 20 + }, + { + "epoch": 0.2986666666666667, + "grad_norm": 0.006111981347203255, + "learning_rate": 9.090909090909091e-06, + "loss": 0.5651419162750244, + "step": 21 + }, + { + "epoch": 0.3128888888888889, + "grad_norm": 0.005564771592617035, + "learning_rate": 9.545454545454547e-06, + "loss": 0.553795576095581, + "step": 22 + }, + { + "epoch": 0.32711111111111113, + "grad_norm": 0.0057683708146214485, + "learning_rate": 1e-05, + "loss": 0.5542658567428589, + "step": 23 + }, + { + "epoch": 0.3413333333333333, + "grad_norm": 0.006690182723104954, + "learning_rate": 9.999323662872998e-06, + "loss": 0.5586982369422913, + "step": 24 + }, + { + "epoch": 0.35555555555555557, + "grad_norm": 0.006314165890216827, + "learning_rate": 9.99729483446475e-06, + "loss": 0.5584714412689209, + "step": 25 + }, + { + "epoch": 0.36977777777777776, + "grad_norm": 0.005813857074826956, + "learning_rate": 9.993914063644053e-06, + "loss": 0.5466614961624146, + "step": 26 + }, + { + "epoch": 0.384, + "grad_norm": 0.005381743889302015, + "learning_rate": 9.989182265027232e-06, + "loss": 0.5380265712738037, + "step": 27 + }, + { + "epoch": 0.3982222222222222, + "grad_norm": 0.0049252682365477085, + "learning_rate": 9.98310071873072e-06, + "loss": 0.535520613193512, + "step": 28 + }, + { + "epoch": 0.41244444444444445, + "grad_norm": 0.0048560877330601215, + "learning_rate": 9.975671070024741e-06, + "loss": 0.5377035140991211, + "step": 29 + }, + { + "epoch": 0.4266666666666667, + "grad_norm": 0.004911178257316351, + "learning_rate": 9.966895328888195e-06, + "loss": 0.5570093989372253, + "step": 30 + }, + { + "epoch": 0.4408888888888889, + "grad_norm": 0.004746970720589161, + "learning_rate": 9.956775869464901e-06, + "loss": 0.5285611748695374, + "step": 31 + }, + { + "epoch": 0.45511111111111113, + "grad_norm": 0.0047261337749660015, + "learning_rate": 9.945315429421307e-06, + "loss": 0.5199858546257019, + "step": 32 + }, + { + "epoch": 0.4693333333333333, + "grad_norm": 0.004447608254849911, + "learning_rate": 9.932517109205849e-06, + "loss": 0.5250071287155151, + "step": 33 + }, + { + "epoch": 0.48355555555555557, + "grad_norm": 0.004617375787347555, + "learning_rate": 9.918384371210178e-06, + "loss": 0.5516105890274048, + "step": 34 + }, + { + "epoch": 0.49777777777777776, + "grad_norm": 0.004378544166684151, + "learning_rate": 9.902921038832456e-06, + "loss": 0.5298895835876465, + "step": 35 + }, + { + "epoch": 0.512, + "grad_norm": 0.004164176061749458, + "learning_rate": 9.886131295443003e-06, + "loss": 0.5044555068016052, + "step": 36 + }, + { + "epoch": 0.5262222222222223, + "grad_norm": 0.004092546179890633, + "learning_rate": 9.868019683252543e-06, + "loss": 0.5102939009666443, + "step": 37 + }, + { + "epoch": 0.5404444444444444, + "grad_norm": 0.004554086830466986, + "learning_rate": 9.848591102083375e-06, + "loss": 0.5421494245529175, + "step": 38 + }, + { + "epoch": 0.5546666666666666, + "grad_norm": 0.004217132460325956, + "learning_rate": 9.82785080804381e-06, + "loss": 0.5239347219467163, + "step": 39 + }, + { + "epoch": 0.5688888888888889, + "grad_norm": 0.004286583047360182, + "learning_rate": 9.805804412106197e-06, + "loss": 0.5437066555023193, + "step": 40 + }, + { + "epoch": 0.5831111111111111, + "grad_norm": 0.004261231981217861, + "learning_rate": 9.782457878588977e-06, + "loss": 0.5146951079368591, + "step": 41 + }, + { + "epoch": 0.5973333333333334, + "grad_norm": 0.004209685139358044, + "learning_rate": 9.75781752354311e-06, + "loss": 0.4995099604129791, + "step": 42 + }, + { + "epoch": 0.6115555555555555, + "grad_norm": 0.0039211539551615715, + "learning_rate": 9.731890013043367e-06, + "loss": 0.5249044299125671, + "step": 43 + }, + { + "epoch": 0.6257777777777778, + "grad_norm": 0.0038641595747321844, + "learning_rate": 9.704682361384941e-06, + "loss": 0.4980819821357727, + "step": 44 + }, + { + "epoch": 0.64, + "grad_norm": 0.004069323651492596, + "learning_rate": 9.676201929185809e-06, + "loss": 0.5113821625709534, + "step": 45 + }, + { + "epoch": 0.6542222222222223, + "grad_norm": 0.0038530679885298014, + "learning_rate": 9.646456421395447e-06, + "loss": 0.5072203278541565, + "step": 46 + }, + { + "epoch": 0.6684444444444444, + "grad_norm": 0.003951605875045061, + "learning_rate": 9.615453885210368e-06, + "loss": 0.509921669960022, + "step": 47 + }, + { + "epoch": 0.6826666666666666, + "grad_norm": 0.0038154355715960264, + "learning_rate": 9.583202707897075e-06, + "loss": 0.49035143852233887, + "step": 48 + }, + { + "epoch": 0.6968888888888889, + "grad_norm": 0.004143900703638792, + "learning_rate": 9.549711614523007e-06, + "loss": 0.4858332574367523, + "step": 49 + }, + { + "epoch": 0.7111111111111111, + "grad_norm": 0.0038300366140902042, + "learning_rate": 9.514989665596114e-06, + "loss": 0.49062296748161316, + "step": 50 + }, + { + "epoch": 0.7111111111111111, + "eval_loss": 0.5049259662628174, + "eval_runtime": 38.0442, + "eval_samples_per_second": 5.257, + "eval_steps_per_second": 2.629, + "step": 50 + }, + { + "epoch": 0.7253333333333334, + "grad_norm": 0.003965203184634447, + "learning_rate": 9.479046254613673e-06, + "loss": 0.5013079643249512, + "step": 51 + }, + { + "epoch": 0.7395555555555555, + "grad_norm": 0.003626414341852069, + "learning_rate": 9.441891105521005e-06, + "loss": 0.46776771545410156, + "step": 52 + }, + { + "epoch": 0.7537777777777778, + "grad_norm": 0.003647371893748641, + "learning_rate": 9.40353427008083e-06, + "loss": 0.4870305061340332, + "step": 53 + }, + { + "epoch": 0.768, + "grad_norm": 0.0036924544256180525, + "learning_rate": 9.3639861251539e-06, + "loss": 0.48354169726371765, + "step": 54 + }, + { + "epoch": 0.7822222222222223, + "grad_norm": 0.0037534767761826515, + "learning_rate": 9.323257369891702e-06, + "loss": 0.5097652673721313, + "step": 55 + }, + { + "epoch": 0.7964444444444444, + "grad_norm": 0.003901001764461398, + "learning_rate": 9.281359022841966e-06, + "loss": 0.5023170709609985, + "step": 56 + }, + { + "epoch": 0.8106666666666666, + "grad_norm": 0.0037305736914277077, + "learning_rate": 9.238302418967757e-06, + "loss": 0.49597933888435364, + "step": 57 + }, + { + "epoch": 0.8248888888888889, + "grad_norm": 0.003764534369111061, + "learning_rate": 9.194099206580981e-06, + "loss": 0.504522442817688, + "step": 58 + }, + { + "epoch": 0.8391111111111111, + "grad_norm": 0.004040914587676525, + "learning_rate": 9.14876134419111e-06, + "loss": 0.49795830249786377, + "step": 59 + }, + { + "epoch": 0.8533333333333334, + "grad_norm": 0.003855582093819976, + "learning_rate": 9.102301097269974e-06, + "loss": 0.49624279141426086, + "step": 60 + }, + { + "epoch": 0.8675555555555555, + "grad_norm": 0.003851535264402628, + "learning_rate": 9.05473103493355e-06, + "loss": 0.47714120149612427, + "step": 61 + }, + { + "epoch": 0.8817777777777778, + "grad_norm": 0.004122862592339516, + "learning_rate": 9.006064026541549e-06, + "loss": 0.5046075582504272, + "step": 62 + }, + { + "epoch": 0.896, + "grad_norm": 0.0036710964050143957, + "learning_rate": 8.956313238215824e-06, + "loss": 0.45711296796798706, + "step": 63 + }, + { + "epoch": 0.9102222222222223, + "grad_norm": 0.004037045873701572, + "learning_rate": 8.905492129278478e-06, + "loss": 0.4839310050010681, + "step": 64 + }, + { + "epoch": 0.9244444444444444, + "grad_norm": 0.004081980790942907, + "learning_rate": 8.85361444861063e-06, + "loss": 0.4877704381942749, + "step": 65 + }, + { + "epoch": 0.9386666666666666, + "grad_norm": 0.0037577205803245306, + "learning_rate": 8.800694230932885e-06, + "loss": 0.49369895458221436, + "step": 66 + }, + { + "epoch": 0.9528888888888889, + "grad_norm": 0.0037781032733619213, + "learning_rate": 8.74674579300843e-06, + "loss": 0.4812237620353699, + "step": 67 + }, + { + "epoch": 0.9671111111111111, + "grad_norm": 0.0046208929270505905, + "learning_rate": 8.691783729769874e-06, + "loss": 0.4722365736961365, + "step": 68 + }, + { + "epoch": 0.9813333333333333, + "grad_norm": 0.004026597365736961, + "learning_rate": 8.635822910370793e-06, + "loss": 0.4962988495826721, + "step": 69 + }, + { + "epoch": 0.9955555555555555, + "grad_norm": 0.004095569718629122, + "learning_rate": 8.578878474163115e-06, + "loss": 0.45991775393486023, + "step": 70 + }, + { + "epoch": 1.0, + "grad_norm": 0.006300764624029398, + "learning_rate": 8.520965826601394e-06, + "loss": 0.4934382140636444, + "step": 71 + } + ], + "logging_steps": 1, + "max_steps": 213, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.7454250364554445e+17, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/qwen2.5-3b-instruct/checkpoint-71/training_args.bin b/qwen2.5-3b-instruct/checkpoint-71/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..078e4bb631cb6e7996cf1a54cb10c1a5978aa53a --- /dev/null +++ b/qwen2.5-3b-instruct/checkpoint-71/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c156fe01067afc4b06ba073aac841107d9e7b73798856df6f0417af272038b3d +size 5649 diff --git a/qwen2.5-3b-instruct/config.json b/qwen2.5-3b-instruct/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4814292863d1e5aa2f4fda1db18f4082cd61aa93 --- /dev/null +++ b/qwen2.5-3b-instruct/config.json @@ -0,0 +1,69 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 11008, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 70, + "model_type": "qwen2", + "num_attention_heads": 16, + "num_hidden_layers": 36, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.2.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/qwen2.5-3b-instruct/eval_results.json b/qwen2.5-3b-instruct/eval_results.json new file mode 100644 index 0000000000000000000000000000000000000000..ebdc0248c196d9e3491299e2be7b64f789391863 --- /dev/null +++ b/qwen2.5-3b-instruct/eval_results.json @@ -0,0 +1,7 @@ +{ + "epoch": 3.0, + "eval_loss": 0.4636715054512024, + "eval_runtime": 39.8733, + "eval_samples_per_second": 5.016, + "eval_steps_per_second": 2.508 +} \ No newline at end of file diff --git a/qwen2.5-3b-instruct/generation_config.json b/qwen2.5-3b-instruct/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..47205fbf3386a215e7c2b9d62df6576f49b77e7b --- /dev/null +++ b/qwen2.5-3b-instruct/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.05, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.2.0" +} diff --git a/qwen2.5-3b-instruct/model.safetensors b/qwen2.5-3b-instruct/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d60af65d2fc092fb0b22b79c826e7c7484a6a18c --- /dev/null +++ b/qwen2.5-3b-instruct/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a3c8407d8c355dd9ca4bd171fb93c8bc925caeb151fcf8fd44b85decaa7a1a5 +size 12343804296 diff --git a/qwen2.5-3b-instruct/tokenizer.json b/qwen2.5-3b-instruct/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..34510ff0037cd50428af467a17ead5a96140a32c --- /dev/null +++ b/qwen2.5-3b-instruct/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/qwen2.5-3b-instruct/tokenizer_config.json b/qwen2.5-3b-instruct/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..145e2c7a68b109ae0253dda29fc2e66cb01b4b85 --- /dev/null +++ b/qwen2.5-3b-instruct/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen2.5-3b-instruct/train_results.json b/qwen2.5-3b-instruct/train_results.json new file mode 100644 index 0000000000000000000000000000000000000000..3a3a8f253f8a572092e794082b87b1e06fe1b923 --- /dev/null +++ b/qwen2.5-3b-instruct/train_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 3.0, + "total_flos": 5.2362750737986355e+17, + "train_loss": 0.4823716257659482, + "train_runtime": 11702.1548, + "train_samples_per_second": 4.615, + "train_steps_per_second": 0.018 +} \ No newline at end of file diff --git a/qwen2.5-3b-instruct/trainer_log.jsonl b/qwen2.5-3b-instruct/trainer_log.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..98cbc7f25bdc442f44451f302104831b84b31a84 --- /dev/null +++ b/qwen2.5-3b-instruct/trainer_log.jsonl @@ -0,0 +1,218 @@ +{"current_steps": 1, "total_steps": 213, "loss": 1.005206823348999, "lr": 0.0, "epoch": 0.014222222222222223, "percentage": 0.47, "elapsed_time": "0:01:02", "remaining_time": "3:42:13"} +{"current_steps": 2, "total_steps": 213, "loss": 1.0057916641235352, "lr": 4.5454545454545457e-07, "epoch": 0.028444444444444446, "percentage": 0.94, "elapsed_time": "0:02:01", "remaining_time": "3:32:57"} +{"current_steps": 3, "total_steps": 213, "loss": 0.9406883716583252, "lr": 9.090909090909091e-07, "epoch": 0.042666666666666665, "percentage": 1.41, "elapsed_time": "0:02:54", "remaining_time": "3:23:38"} +{"current_steps": 4, "total_steps": 213, "loss": 0.8941981792449951, "lr": 1.3636363636363636e-06, "epoch": 0.05688888888888889, "percentage": 1.88, "elapsed_time": "0:03:44", "remaining_time": "3:15:53"} +{"current_steps": 5, "total_steps": 213, "loss": 0.7926980257034302, "lr": 1.8181818181818183e-06, "epoch": 0.07111111111111111, "percentage": 2.35, "elapsed_time": "0:04:39", "remaining_time": "3:13:39"} +{"current_steps": 6, "total_steps": 213, "loss": 0.8305954933166504, "lr": 2.2727272727272728e-06, "epoch": 0.08533333333333333, "percentage": 2.82, "elapsed_time": "0:05:30", "remaining_time": "3:10:10"} +{"current_steps": 7, "total_steps": 213, "loss": 0.7948952913284302, "lr": 2.7272727272727272e-06, "epoch": 0.09955555555555555, "percentage": 3.29, "elapsed_time": "0:06:20", "remaining_time": "3:06:44"} +{"current_steps": 8, "total_steps": 213, "loss": 0.7530462741851807, "lr": 3.181818181818182e-06, "epoch": 0.11377777777777778, "percentage": 3.76, "elapsed_time": "0:07:12", "remaining_time": "3:04:52"} +{"current_steps": 9, "total_steps": 213, "loss": 0.7157606482505798, "lr": 3.6363636363636366e-06, "epoch": 0.128, "percentage": 4.23, "elapsed_time": "0:08:05", "remaining_time": "3:03:18"} +{"current_steps": 10, "total_steps": 213, "loss": 0.7008587718009949, "lr": 4.0909090909090915e-06, "epoch": 0.14222222222222222, "percentage": 4.69, "elapsed_time": "0:09:00", "remaining_time": "3:02:44"} +{"current_steps": 11, "total_steps": 213, "loss": 0.6668463945388794, "lr": 4.5454545454545455e-06, "epoch": 0.15644444444444444, "percentage": 5.16, "elapsed_time": "0:09:53", "remaining_time": "3:01:38"} +{"current_steps": 12, "total_steps": 213, "loss": 0.6507903337478638, "lr": 5e-06, "epoch": 0.17066666666666666, "percentage": 5.63, "elapsed_time": "0:10:47", "remaining_time": "3:00:47"} +{"current_steps": 13, "total_steps": 213, "loss": 0.6217454671859741, "lr": 5.4545454545454545e-06, "epoch": 0.18488888888888888, "percentage": 6.1, "elapsed_time": "0:11:38", "remaining_time": "2:59:05"} +{"current_steps": 14, "total_steps": 213, "loss": 0.6269917488098145, "lr": 5.90909090909091e-06, "epoch": 0.1991111111111111, "percentage": 6.57, "elapsed_time": "0:12:32", "remaining_time": "2:58:19"} +{"current_steps": 15, "total_steps": 213, "loss": 0.5836964845657349, "lr": 6.363636363636364e-06, "epoch": 0.21333333333333335, "percentage": 7.04, "elapsed_time": "0:13:23", "remaining_time": "2:56:46"} +{"current_steps": 16, "total_steps": 213, "loss": 0.6187424659729004, "lr": 6.818181818181818e-06, "epoch": 0.22755555555555557, "percentage": 7.51, "elapsed_time": "0:14:15", "remaining_time": "2:55:34"} +{"current_steps": 17, "total_steps": 213, "loss": 0.5914188027381897, "lr": 7.272727272727273e-06, "epoch": 0.24177777777777779, "percentage": 7.98, "elapsed_time": "0:15:09", "remaining_time": "2:54:41"} +{"current_steps": 18, "total_steps": 213, "loss": 0.5802223682403564, "lr": 7.727272727272727e-06, "epoch": 0.256, "percentage": 8.45, "elapsed_time": "0:15:59", "remaining_time": "2:53:18"} +{"current_steps": 19, "total_steps": 213, "loss": 0.5884265899658203, "lr": 8.181818181818183e-06, "epoch": 0.2702222222222222, "percentage": 8.92, "elapsed_time": "0:16:54", "remaining_time": "2:52:36"} +{"current_steps": 20, "total_steps": 213, "loss": 0.5850374698638916, "lr": 8.636363636363637e-06, "epoch": 0.28444444444444444, "percentage": 9.39, "elapsed_time": "0:17:46", "remaining_time": "2:51:34"} +{"current_steps": 21, "total_steps": 213, "loss": 0.5651419162750244, "lr": 9.090909090909091e-06, "epoch": 0.2986666666666667, "percentage": 9.86, "elapsed_time": "0:18:41", "remaining_time": "2:50:52"} +{"current_steps": 22, "total_steps": 213, "loss": 0.553795576095581, "lr": 9.545454545454547e-06, "epoch": 0.3128888888888889, "percentage": 10.33, "elapsed_time": "0:19:33", "remaining_time": "2:49:51"} +{"current_steps": 23, "total_steps": 213, "loss": 0.5542658567428589, "lr": 1e-05, "epoch": 0.32711111111111113, "percentage": 10.8, "elapsed_time": "0:20:27", "remaining_time": "2:49:01"} +{"current_steps": 24, "total_steps": 213, "loss": 0.5586982369422913, "lr": 9.999323662872998e-06, "epoch": 0.3413333333333333, "percentage": 11.27, "elapsed_time": "0:21:17", "remaining_time": "2:47:43"} +{"current_steps": 25, "total_steps": 213, "loss": 0.5584714412689209, "lr": 9.99729483446475e-06, "epoch": 0.35555555555555557, "percentage": 11.74, "elapsed_time": "0:22:11", "remaining_time": "2:46:53"} +{"current_steps": 26, "total_steps": 213, "loss": 0.5466614961624146, "lr": 9.993914063644053e-06, "epoch": 0.36977777777777776, "percentage": 12.21, "elapsed_time": "0:23:03", "remaining_time": "2:45:51"} +{"current_steps": 27, "total_steps": 213, "loss": 0.5380265712738037, "lr": 9.989182265027232e-06, "epoch": 0.384, "percentage": 12.68, "elapsed_time": "0:23:57", "remaining_time": "2:45:00"} +{"current_steps": 28, "total_steps": 213, "loss": 0.535520613193512, "lr": 9.98310071873072e-06, "epoch": 0.3982222222222222, "percentage": 13.15, "elapsed_time": "0:24:49", "remaining_time": "2:44:02"} +{"current_steps": 29, "total_steps": 213, "loss": 0.5377035140991211, "lr": 9.975671070024741e-06, "epoch": 0.41244444444444445, "percentage": 13.62, "elapsed_time": "0:25:40", "remaining_time": "2:42:57"} +{"current_steps": 30, "total_steps": 213, "loss": 0.5570093989372253, "lr": 9.966895328888195e-06, "epoch": 0.4266666666666667, "percentage": 14.08, "elapsed_time": "0:26:34", "remaining_time": "2:42:04"} +{"current_steps": 31, "total_steps": 213, "loss": 0.5285611748695374, "lr": 9.956775869464901e-06, "epoch": 0.4408888888888889, "percentage": 14.55, "elapsed_time": "0:27:26", "remaining_time": "2:41:08"} +{"current_steps": 32, "total_steps": 213, "loss": 0.5199858546257019, "lr": 9.945315429421307e-06, "epoch": 0.45511111111111113, "percentage": 15.02, "elapsed_time": "0:28:20", "remaining_time": "2:40:15"} +{"current_steps": 33, "total_steps": 213, "loss": 0.5250071287155151, "lr": 9.932517109205849e-06, "epoch": 0.4693333333333333, "percentage": 15.49, "elapsed_time": "0:29:11", "remaining_time": "2:39:15"} +{"current_steps": 34, "total_steps": 213, "loss": 0.5516105890274048, "lr": 9.918384371210178e-06, "epoch": 0.48355555555555557, "percentage": 15.96, "elapsed_time": "0:30:02", "remaining_time": "2:38:10"} +{"current_steps": 35, "total_steps": 213, "loss": 0.5298895835876465, "lr": 9.902921038832456e-06, "epoch": 0.49777777777777776, "percentage": 16.43, "elapsed_time": "0:30:53", "remaining_time": "2:37:07"} +{"current_steps": 36, "total_steps": 213, "loss": 0.5044555068016052, "lr": 9.886131295443003e-06, "epoch": 0.512, "percentage": 16.9, "elapsed_time": "0:31:45", "remaining_time": "2:36:10"} +{"current_steps": 37, "total_steps": 213, "loss": 0.5102939009666443, "lr": 9.868019683252543e-06, "epoch": 0.5262222222222223, "percentage": 17.37, "elapsed_time": "0:32:37", "remaining_time": "2:35:11"} +{"current_steps": 38, "total_steps": 213, "loss": 0.5421494245529175, "lr": 9.848591102083375e-06, "epoch": 0.5404444444444444, "percentage": 17.84, "elapsed_time": "0:33:28", "remaining_time": "2:34:10"} +{"current_steps": 39, "total_steps": 213, "loss": 0.5239347219467163, "lr": 9.82785080804381e-06, "epoch": 0.5546666666666666, "percentage": 18.31, "elapsed_time": "0:34:21", "remaining_time": "2:33:15"} +{"current_steps": 40, "total_steps": 213, "loss": 0.5437066555023193, "lr": 9.805804412106197e-06, "epoch": 0.5688888888888889, "percentage": 18.78, "elapsed_time": "0:35:11", "remaining_time": "2:32:13"} +{"current_steps": 41, "total_steps": 213, "loss": 0.5146951079368591, "lr": 9.782457878588977e-06, "epoch": 0.5831111111111111, "percentage": 19.25, "elapsed_time": "0:36:05", "remaining_time": "2:31:25"} +{"current_steps": 42, "total_steps": 213, "loss": 0.4995099604129791, "lr": 9.75781752354311e-06, "epoch": 0.5973333333333334, "percentage": 19.72, "elapsed_time": "0:36:59", "remaining_time": "2:30:37"} +{"current_steps": 43, "total_steps": 213, "loss": 0.5249044299125671, "lr": 9.731890013043367e-06, "epoch": 0.6115555555555555, "percentage": 20.19, "elapsed_time": "0:37:52", "remaining_time": "2:29:44"} +{"current_steps": 44, "total_steps": 213, "loss": 0.4980819821357727, "lr": 9.704682361384941e-06, "epoch": 0.6257777777777778, "percentage": 20.66, "elapsed_time": "0:38:49", "remaining_time": "2:29:05"} +{"current_steps": 45, "total_steps": 213, "loss": 0.5113821625709534, "lr": 9.676201929185809e-06, "epoch": 0.64, "percentage": 21.13, "elapsed_time": "0:39:39", "remaining_time": "2:28:04"} +{"current_steps": 46, "total_steps": 213, "loss": 0.5072203278541565, "lr": 9.646456421395447e-06, "epoch": 0.6542222222222223, "percentage": 21.6, "elapsed_time": "0:40:33", "remaining_time": "2:27:16"} +{"current_steps": 47, "total_steps": 213, "loss": 0.509921669960022, "lr": 9.615453885210368e-06, "epoch": 0.6684444444444444, "percentage": 22.07, "elapsed_time": "0:41:26", "remaining_time": "2:26:21"} +{"current_steps": 48, "total_steps": 213, "loss": 0.49035143852233887, "lr": 9.583202707897075e-06, "epoch": 0.6826666666666666, "percentage": 22.54, "elapsed_time": "0:42:20", "remaining_time": "2:25:32"} +{"current_steps": 49, "total_steps": 213, "loss": 0.4858332574367523, "lr": 9.549711614523007e-06, "epoch": 0.6968888888888889, "percentage": 23.0, "elapsed_time": "0:43:09", "remaining_time": "2:24:26"} +{"current_steps": 50, "total_steps": 213, "loss": 0.49062296748161316, "lr": 9.514989665596114e-06, "epoch": 0.7111111111111111, "percentage": 23.47, "elapsed_time": "0:44:03", "remaining_time": "2:23:37"} +{"current_steps": 50, "total_steps": 213, "eval_loss": 0.5049259662628174, "epoch": 0.7111111111111111, "percentage": 23.47, "elapsed_time": "0:44:41", "remaining_time": "2:25:41"} +{"current_steps": 51, "total_steps": 213, "loss": 0.5013079643249512, "lr": 9.479046254613673e-06, "epoch": 0.7253333333333334, "percentage": 23.94, "elapsed_time": "0:45:37", "remaining_time": "2:24:54"} +{"current_steps": 52, "total_steps": 213, "loss": 0.46776771545410156, "lr": 9.441891105521005e-06, "epoch": 0.7395555555555555, "percentage": 24.41, "elapsed_time": "0:46:33", "remaining_time": "2:24:07"} +{"current_steps": 53, "total_steps": 213, "loss": 0.4870305061340332, "lr": 9.40353427008083e-06, "epoch": 0.7537777777777778, "percentage": 24.88, "elapsed_time": "0:47:27", "remaining_time": "2:23:15"} +{"current_steps": 54, "total_steps": 213, "loss": 0.48354169726371765, "lr": 9.3639861251539e-06, "epoch": 0.768, "percentage": 25.35, "elapsed_time": "0:48:23", "remaining_time": "2:22:28"} +{"current_steps": 55, "total_steps": 213, "loss": 0.5097652673721313, "lr": 9.323257369891702e-06, "epoch": 0.7822222222222223, "percentage": 25.82, "elapsed_time": "0:49:14", "remaining_time": "2:21:27"} +{"current_steps": 56, "total_steps": 213, "loss": 0.5023170709609985, "lr": 9.281359022841966e-06, "epoch": 0.7964444444444444, "percentage": 26.29, "elapsed_time": "0:50:06", "remaining_time": "2:20:28"} +{"current_steps": 57, "total_steps": 213, "loss": 0.49597933888435364, "lr": 9.238302418967757e-06, "epoch": 0.8106666666666666, "percentage": 26.76, "elapsed_time": "0:50:57", "remaining_time": "2:19:29"} +{"current_steps": 58, "total_steps": 213, "loss": 0.504522442817688, "lr": 9.194099206580981e-06, "epoch": 0.8248888888888889, "percentage": 27.23, "elapsed_time": "0:51:50", "remaining_time": "2:18:33"} +{"current_steps": 59, "total_steps": 213, "loss": 0.49795830249786377, "lr": 9.14876134419111e-06, "epoch": 0.8391111111111111, "percentage": 27.7, "elapsed_time": "0:52:42", "remaining_time": "2:17:33"} +{"current_steps": 60, "total_steps": 213, "loss": 0.49624279141426086, "lr": 9.102301097269974e-06, "epoch": 0.8533333333333334, "percentage": 28.17, "elapsed_time": "0:53:35", "remaining_time": "2:16:39"} +{"current_steps": 61, "total_steps": 213, "loss": 0.47714120149612427, "lr": 9.05473103493355e-06, "epoch": 0.8675555555555555, "percentage": 28.64, "elapsed_time": "0:54:27", "remaining_time": "2:15:42"} +{"current_steps": 62, "total_steps": 213, "loss": 0.5046075582504272, "lr": 9.006064026541549e-06, "epoch": 0.8817777777777778, "percentage": 29.11, "elapsed_time": "0:55:18", "remaining_time": "2:14:43"} +{"current_steps": 63, "total_steps": 213, "loss": 0.45711296796798706, "lr": 8.956313238215824e-06, "epoch": 0.896, "percentage": 29.58, "elapsed_time": "0:56:16", "remaining_time": "2:14:00"} +{"current_steps": 64, "total_steps": 213, "loss": 0.4839310050010681, "lr": 8.905492129278478e-06, "epoch": 0.9102222222222223, "percentage": 30.05, "elapsed_time": "0:57:08", "remaining_time": "2:13:00"} +{"current_steps": 65, "total_steps": 213, "loss": 0.4877704381942749, "lr": 8.85361444861063e-06, "epoch": 0.9244444444444444, "percentage": 30.52, "elapsed_time": "0:58:04", "remaining_time": "2:12:13"} +{"current_steps": 66, "total_steps": 213, "loss": 0.49369895458221436, "lr": 8.800694230932885e-06, "epoch": 0.9386666666666666, "percentage": 30.99, "elapsed_time": "0:58:56", "remaining_time": "2:11:17"} +{"current_steps": 67, "total_steps": 213, "loss": 0.4812237620353699, "lr": 8.74674579300843e-06, "epoch": 0.9528888888888889, "percentage": 31.46, "elapsed_time": "0:59:51", "remaining_time": "2:10:26"} +{"current_steps": 68, "total_steps": 213, "loss": 0.4722365736961365, "lr": 8.691783729769874e-06, "epoch": 0.9671111111111111, "percentage": 31.92, "elapsed_time": "1:00:46", "remaining_time": "2:09:35"} +{"current_steps": 69, "total_steps": 213, "loss": 0.4962988495826721, "lr": 8.635822910370793e-06, "epoch": 0.9813333333333333, "percentage": 32.39, "elapsed_time": "1:01:40", "remaining_time": "2:08:42"} +{"current_steps": 70, "total_steps": 213, "loss": 0.45991775393486023, "lr": 8.578878474163115e-06, "epoch": 0.9955555555555555, "percentage": 32.86, "elapsed_time": "1:02:32", "remaining_time": "2:07:46"} +{"current_steps": 71, "total_steps": 213, "loss": 0.4934382140636444, "lr": 8.520965826601394e-06, "epoch": 1.0, "percentage": 33.33, "elapsed_time": "1:02:50", "remaining_time": "2:05:40"} +{"current_steps": 72, "total_steps": 213, "loss": 0.4847542643547058, "lr": 8.462100635075097e-06, "epoch": 1.0142222222222221, "percentage": 33.8, "elapsed_time": "1:05:40", "remaining_time": "2:08:36"} +{"current_steps": 73, "total_steps": 213, "loss": 0.4636518955230713, "lr": 8.40229882467003e-06, "epoch": 1.0284444444444445, "percentage": 34.27, "elapsed_time": "1:06:33", "remaining_time": "2:07:38"} +{"current_steps": 74, "total_steps": 213, "loss": 0.44187992811203003, "lr": 8.341576573860049e-06, "epoch": 1.0426666666666666, "percentage": 34.74, "elapsed_time": "1:07:30", "remaining_time": "2:06:48"} +{"current_steps": 75, "total_steps": 213, "loss": 0.467815637588501, "lr": 8.279950310130218e-06, "epoch": 1.056888888888889, "percentage": 35.21, "elapsed_time": "1:08:24", "remaining_time": "2:05:52"} +{"current_steps": 76, "total_steps": 213, "loss": 0.451981782913208, "lr": 8.2174367055326e-06, "epoch": 1.0711111111111111, "percentage": 35.68, "elapsed_time": "1:09:16", "remaining_time": "2:04:52"} +{"current_steps": 77, "total_steps": 213, "loss": 0.4449639320373535, "lr": 8.154052672175888e-06, "epoch": 1.0853333333333333, "percentage": 36.15, "elapsed_time": "1:10:08", "remaining_time": "2:03:54"} +{"current_steps": 78, "total_steps": 213, "loss": 0.47661933302879333, "lr": 8.08981535765009e-06, "epoch": 1.0995555555555556, "percentage": 36.62, "elapsed_time": "1:11:03", "remaining_time": "2:02:59"} +{"current_steps": 79, "total_steps": 213, "loss": 0.45776277780532837, "lr": 8.024742140387506e-06, "epoch": 1.1137777777777778, "percentage": 37.09, "elapsed_time": "1:11:54", "remaining_time": "2:01:58"} +{"current_steps": 80, "total_steps": 213, "loss": 0.44757023453712463, "lr": 7.95885062496126e-06, "epoch": 1.1280000000000001, "percentage": 37.56, "elapsed_time": "1:12:48", "remaining_time": "2:01:02"} +{"current_steps": 81, "total_steps": 213, "loss": 0.419955849647522, "lr": 7.892158637322647e-06, "epoch": 1.1422222222222222, "percentage": 38.03, "elapsed_time": "1:13:39", "remaining_time": "2:00:02"} +{"current_steps": 82, "total_steps": 213, "loss": 0.4457795023918152, "lr": 7.824684219978591e-06, "epoch": 1.1564444444444444, "percentage": 38.5, "elapsed_time": "1:14:32", "remaining_time": "1:59:05"} +{"current_steps": 83, "total_steps": 213, "loss": 0.45605674386024475, "lr": 7.756445627110523e-06, "epoch": 1.1706666666666667, "percentage": 38.97, "elapsed_time": "1:15:27", "remaining_time": "1:58:10"} +{"current_steps": 84, "total_steps": 213, "loss": 0.46289366483688354, "lr": 7.68746131963598e-06, "epoch": 1.1848888888888889, "percentage": 39.44, "elapsed_time": "1:16:19", "remaining_time": "1:57:12"} +{"current_steps": 85, "total_steps": 213, "loss": 0.45568281412124634, "lr": 7.6177499602143e-06, "epoch": 1.199111111111111, "percentage": 39.91, "elapsed_time": "1:17:10", "remaining_time": "1:56:13"} +{"current_steps": 86, "total_steps": 213, "loss": 0.44618040323257446, "lr": 7.547330408197695e-06, "epoch": 1.2133333333333334, "percentage": 40.38, "elapsed_time": "1:18:06", "remaining_time": "1:55:20"} +{"current_steps": 87, "total_steps": 213, "loss": 0.45724251866340637, "lr": 7.476221714529167e-06, "epoch": 1.2275555555555555, "percentage": 40.85, "elapsed_time": "1:18:59", "remaining_time": "1:54:24"} +{"current_steps": 88, "total_steps": 213, "loss": 0.4806937575340271, "lr": 7.404443116588548e-06, "epoch": 1.2417777777777779, "percentage": 41.31, "elapsed_time": "1:19:50", "remaining_time": "1:53:24"} +{"current_steps": 89, "total_steps": 213, "loss": 0.4561174213886261, "lr": 7.332014032988123e-06, "epoch": 1.256, "percentage": 41.78, "elapsed_time": "1:20:42", "remaining_time": "1:52:27"} +{"current_steps": 90, "total_steps": 213, "loss": 0.4558497667312622, "lr": 7.2589540583192165e-06, "epoch": 1.2702222222222221, "percentage": 42.25, "elapsed_time": "1:21:35", "remaining_time": "1:51:30"} +{"current_steps": 91, "total_steps": 213, "loss": 0.4397018551826477, "lr": 7.185282957851175e-06, "epoch": 1.2844444444444445, "percentage": 42.72, "elapsed_time": "1:22:30", "remaining_time": "1:50:36"} +{"current_steps": 92, "total_steps": 213, "loss": 0.45822006464004517, "lr": 7.111020662184174e-06, "epoch": 1.2986666666666666, "percentage": 43.19, "elapsed_time": "1:23:24", "remaining_time": "1:49:42"} +{"current_steps": 93, "total_steps": 213, "loss": 0.4479483366012573, "lr": 7.036187261857289e-06, "epoch": 1.3128888888888888, "percentage": 43.66, "elapsed_time": "1:24:17", "remaining_time": "1:48:45"} +{"current_steps": 94, "total_steps": 213, "loss": 0.4738925099372864, "lr": 6.960803001913315e-06, "epoch": 1.3271111111111111, "percentage": 44.13, "elapsed_time": "1:25:06", "remaining_time": "1:47:45"} +{"current_steps": 95, "total_steps": 213, "loss": 0.4614931643009186, "lr": 6.884888276421766e-06, "epoch": 1.3413333333333333, "percentage": 44.6, "elapsed_time": "1:25:59", "remaining_time": "1:46:48"} +{"current_steps": 96, "total_steps": 213, "loss": 0.44328922033309937, "lr": 6.8084636229615786e-06, "epoch": 1.3555555555555556, "percentage": 45.07, "elapsed_time": "1:26:52", "remaining_time": "1:45:52"} +{"current_steps": 97, "total_steps": 213, "loss": 0.44612404704093933, "lr": 6.731549717064975e-06, "epoch": 1.3697777777777778, "percentage": 45.54, "elapsed_time": "1:27:46", "remaining_time": "1:44:57"} +{"current_steps": 98, "total_steps": 213, "loss": 0.4554000794887543, "lr": 6.654167366624009e-06, "epoch": 1.384, "percentage": 46.01, "elapsed_time": "1:28:35", "remaining_time": "1:43:58"} +{"current_steps": 99, "total_steps": 213, "loss": 0.45616668462753296, "lr": 6.576337506261314e-06, "epoch": 1.3982222222222223, "percentage": 46.48, "elapsed_time": "1:29:28", "remaining_time": "1:43:02"} +{"current_steps": 100, "total_steps": 213, "loss": 0.45816588401794434, "lr": 6.498081191666549e-06, "epoch": 1.4124444444444444, "percentage": 46.95, "elapsed_time": "1:30:24", "remaining_time": "1:42:09"} +{"current_steps": 100, "total_steps": 213, "eval_loss": 0.4757018983364105, "epoch": 1.4124444444444444, "percentage": 46.95, "elapsed_time": "1:31:03", "remaining_time": "1:42:54"} +{"current_steps": 101, "total_steps": 213, "loss": 0.4430053234100342, "lr": 6.419419593900109e-06, "epoch": 1.4266666666666667, "percentage": 47.42, "elapsed_time": "1:31:54", "remaining_time": "1:41:54"} +{"current_steps": 102, "total_steps": 213, "loss": 0.43451493978500366, "lr": 6.340373993665607e-06, "epoch": 1.4408888888888889, "percentage": 47.89, "elapsed_time": "1:32:48", "remaining_time": "1:40:59"} +{"current_steps": 103, "total_steps": 213, "loss": 0.4296550154685974, "lr": 6.2609657755527135e-06, "epoch": 1.455111111111111, "percentage": 48.36, "elapsed_time": "1:33:39", "remaining_time": "1:40:01"} +{"current_steps": 104, "total_steps": 213, "loss": 0.46728163957595825, "lr": 6.1812164222518626e-06, "epoch": 1.4693333333333334, "percentage": 48.83, "elapsed_time": "1:34:31", "remaining_time": "1:39:04"} +{"current_steps": 105, "total_steps": 213, "loss": 0.446887731552124, "lr": 6.101147508742456e-06, "epoch": 1.4835555555555555, "percentage": 49.3, "elapsed_time": "1:35:23", "remaining_time": "1:38:07"} +{"current_steps": 106, "total_steps": 213, "loss": 0.43969088792800903, "lr": 6.020780696456059e-06, "epoch": 1.4977777777777779, "percentage": 49.77, "elapsed_time": "1:36:18", "remaining_time": "1:37:12"} +{"current_steps": 107, "total_steps": 213, "loss": 0.4489642381668091, "lr": 5.940137727416247e-06, "epoch": 1.512, "percentage": 50.23, "elapsed_time": "1:37:12", "remaining_time": "1:36:18"} +{"current_steps": 108, "total_steps": 213, "loss": 0.4380025863647461, "lr": 5.859240418356614e-06, "epoch": 1.5262222222222221, "percentage": 50.7, "elapsed_time": "1:38:07", "remaining_time": "1:35:23"} +{"current_steps": 109, "total_steps": 213, "loss": 0.47752195596694946, "lr": 5.778110654818602e-06, "epoch": 1.5404444444444443, "percentage": 51.17, "elapsed_time": "1:38:57", "remaining_time": "1:34:25"} +{"current_steps": 110, "total_steps": 213, "loss": 0.4498862028121948, "lr": 5.696770385230679e-06, "epoch": 1.5546666666666666, "percentage": 51.64, "elapsed_time": "1:39:50", "remaining_time": "1:33:29"} +{"current_steps": 111, "total_steps": 213, "loss": 0.44400298595428467, "lr": 5.615241614970546e-06, "epoch": 1.568888888888889, "percentage": 52.11, "elapsed_time": "1:40:42", "remaining_time": "1:32:32"} +{"current_steps": 112, "total_steps": 213, "loss": 0.4457319378852844, "lr": 5.533546400411899e-06, "epoch": 1.5831111111111111, "percentage": 52.58, "elapsed_time": "1:41:34", "remaining_time": "1:31:36"} +{"current_steps": 113, "total_steps": 213, "loss": 0.4610831141471863, "lr": 5.4517068429574215e-06, "epoch": 1.5973333333333333, "percentage": 53.05, "elapsed_time": "1:42:29", "remaining_time": "1:30:42"} +{"current_steps": 114, "total_steps": 213, "loss": 0.4425020217895508, "lr": 5.3697450830595775e-06, "epoch": 1.6115555555555554, "percentage": 53.52, "elapsed_time": "1:43:24", "remaining_time": "1:29:48"} +{"current_steps": 115, "total_steps": 213, "loss": 0.44555309414863586, "lr": 5.287683294230855e-06, "epoch": 1.6257777777777778, "percentage": 53.99, "elapsed_time": "1:44:15", "remaining_time": "1:28:50"} +{"current_steps": 116, "total_steps": 213, "loss": 0.417566180229187, "lr": 5.20554367704505e-06, "epoch": 1.6400000000000001, "percentage": 54.46, "elapsed_time": "1:45:10", "remaining_time": "1:27:56"} +{"current_steps": 117, "total_steps": 213, "loss": 0.45680171251296997, "lr": 5.123348453131242e-06, "epoch": 1.6542222222222223, "percentage": 54.93, "elapsed_time": "1:46:03", "remaining_time": "1:27:01"} +{"current_steps": 118, "total_steps": 213, "loss": 0.4432540535926819, "lr": 5.041119859162068e-06, "epoch": 1.6684444444444444, "percentage": 55.4, "elapsed_time": "1:46:56", "remaining_time": "1:26:05"} +{"current_steps": 119, "total_steps": 213, "loss": 0.44924670457839966, "lr": 4.958880140837934e-06, "epoch": 1.6826666666666665, "percentage": 55.87, "elapsed_time": "1:47:48", "remaining_time": "1:25:09"} +{"current_steps": 120, "total_steps": 213, "loss": 0.43826115131378174, "lr": 4.876651546868759e-06, "epoch": 1.696888888888889, "percentage": 56.34, "elapsed_time": "1:48:41", "remaining_time": "1:24:14"} +{"current_steps": 121, "total_steps": 213, "loss": 0.4698702096939087, "lr": 4.794456322954953e-06, "epoch": 1.7111111111111112, "percentage": 56.81, "elapsed_time": "1:49:34", "remaining_time": "1:23:19"} +{"current_steps": 122, "total_steps": 213, "loss": 0.4665527045726776, "lr": 4.7123167057691446e-06, "epoch": 1.7253333333333334, "percentage": 57.28, "elapsed_time": "1:50:27", "remaining_time": "1:22:23"} +{"current_steps": 123, "total_steps": 213, "loss": 0.46612560749053955, "lr": 4.630254916940424e-06, "epoch": 1.7395555555555555, "percentage": 57.75, "elapsed_time": "1:51:23", "remaining_time": "1:21:30"} +{"current_steps": 124, "total_steps": 213, "loss": 0.4559047818183899, "lr": 4.548293157042581e-06, "epoch": 1.7537777777777777, "percentage": 58.22, "elapsed_time": "1:52:20", "remaining_time": "1:20:38"} +{"current_steps": 125, "total_steps": 213, "loss": 0.4595223069190979, "lr": 4.466453599588103e-06, "epoch": 1.768, "percentage": 58.69, "elapsed_time": "1:53:14", "remaining_time": "1:19:43"} +{"current_steps": 126, "total_steps": 213, "loss": 0.42560791969299316, "lr": 4.384758385029457e-06, "epoch": 1.7822222222222224, "percentage": 59.15, "elapsed_time": "1:54:08", "remaining_time": "1:18:48"} +{"current_steps": 127, "total_steps": 213, "loss": 0.43939000368118286, "lr": 4.3032296147693225e-06, "epoch": 1.7964444444444445, "percentage": 59.62, "elapsed_time": "1:54:59", "remaining_time": "1:17:52"} +{"current_steps": 128, "total_steps": 213, "loss": 0.4399089217185974, "lr": 4.2218893451814005e-06, "epoch": 1.8106666666666666, "percentage": 60.09, "elapsed_time": "1:55:54", "remaining_time": "1:16:58"} +{"current_steps": 129, "total_steps": 213, "loss": 0.4451609253883362, "lr": 4.140759581643386e-06, "epoch": 1.8248888888888888, "percentage": 60.56, "elapsed_time": "1:56:48", "remaining_time": "1:16:03"} +{"current_steps": 130, "total_steps": 213, "loss": 0.4488486051559448, "lr": 4.059862272583755e-06, "epoch": 1.8391111111111111, "percentage": 61.03, "elapsed_time": "1:57:43", "remaining_time": "1:15:09"} +{"current_steps": 131, "total_steps": 213, "loss": 0.42662614583969116, "lr": 3.979219303543942e-06, "epoch": 1.8533333333333335, "percentage": 61.5, "elapsed_time": "1:58:39", "remaining_time": "1:14:16"} +{"current_steps": 132, "total_steps": 213, "loss": 0.4547004997730255, "lr": 3.898852491257547e-06, "epoch": 1.8675555555555556, "percentage": 61.97, "elapsed_time": "1:59:31", "remaining_time": "1:13:20"} +{"current_steps": 133, "total_steps": 213, "loss": 0.4510434567928314, "lr": 3.818783577748138e-06, "epoch": 1.8817777777777778, "percentage": 62.44, "elapsed_time": "2:00:24", "remaining_time": "1:12:25"} +{"current_steps": 134, "total_steps": 213, "loss": 0.45353883504867554, "lr": 3.7390342244472886e-06, "epoch": 1.896, "percentage": 62.91, "elapsed_time": "2:01:15", "remaining_time": "1:11:29"} +{"current_steps": 135, "total_steps": 213, "loss": 0.4302864670753479, "lr": 3.659626006334395e-06, "epoch": 1.9102222222222223, "percentage": 63.38, "elapsed_time": "2:02:07", "remaining_time": "1:10:33"} +{"current_steps": 136, "total_steps": 213, "loss": 0.4398406744003296, "lr": 3.5805804060998926e-06, "epoch": 1.9244444444444444, "percentage": 63.85, "elapsed_time": "2:02:57", "remaining_time": "1:09:37"} +{"current_steps": 137, "total_steps": 213, "loss": 0.465728759765625, "lr": 3.501918808333453e-06, "epoch": 1.9386666666666668, "percentage": 64.32, "elapsed_time": "2:03:49", "remaining_time": "1:08:41"} +{"current_steps": 138, "total_steps": 213, "loss": 0.4686010777950287, "lr": 3.4236624937386874e-06, "epoch": 1.952888888888889, "percentage": 64.79, "elapsed_time": "2:04:42", "remaining_time": "1:07:46"} +{"current_steps": 139, "total_steps": 213, "loss": 0.43872472643852234, "lr": 3.3458326333759927e-06, "epoch": 1.967111111111111, "percentage": 65.26, "elapsed_time": "2:05:37", "remaining_time": "1:06:52"} +{"current_steps": 140, "total_steps": 213, "loss": 0.4455334544181824, "lr": 3.268450282935026e-06, "epoch": 1.9813333333333332, "percentage": 65.73, "elapsed_time": "2:06:27", "remaining_time": "1:05:56"} +{"current_steps": 141, "total_steps": 213, "loss": 0.43381941318511963, "lr": 3.1915363770384223e-06, "epoch": 1.9955555555555555, "percentage": 66.2, "elapsed_time": "2:07:23", "remaining_time": "1:05:03"} +{"current_steps": 142, "total_steps": 213, "loss": 0.4559262990951538, "lr": 3.1151117235782346e-06, "epoch": 2.0, "percentage": 66.67, "elapsed_time": "2:07:42", "remaining_time": "1:03:51"} +{"current_steps": 143, "total_steps": 213, "loss": 0.43779897689819336, "lr": 3.0391969980866874e-06, "epoch": 2.014222222222222, "percentage": 67.14, "elapsed_time": "2:10:32", "remaining_time": "1:03:53"} +{"current_steps": 144, "total_steps": 213, "loss": 0.4199046492576599, "lr": 2.963812738142713e-06, "epoch": 2.0284444444444443, "percentage": 67.61, "elapsed_time": "2:11:25", "remaining_time": "1:02:58"} +{"current_steps": 145, "total_steps": 213, "loss": 0.43830418586730957, "lr": 2.8889793378158284e-06, "epoch": 2.042666666666667, "percentage": 68.08, "elapsed_time": "2:12:18", "remaining_time": "1:02:02"} +{"current_steps": 146, "total_steps": 213, "loss": 0.4249774217605591, "lr": 2.814717042148827e-06, "epoch": 2.056888888888889, "percentage": 68.54, "elapsed_time": "2:13:13", "remaining_time": "1:01:08"} +{"current_steps": 147, "total_steps": 213, "loss": 0.41759681701660156, "lr": 2.7410459416807856e-06, "epoch": 2.071111111111111, "percentage": 69.01, "elapsed_time": "2:14:06", "remaining_time": "1:00:12"} +{"current_steps": 148, "total_steps": 213, "loss": 0.43880075216293335, "lr": 2.6679859670118785e-06, "epoch": 2.0853333333333333, "percentage": 69.48, "elapsed_time": "2:14:58", "remaining_time": "0:59:16"} +{"current_steps": 149, "total_steps": 213, "loss": 0.4053642153739929, "lr": 2.5955568834114523e-06, "epoch": 2.0995555555555554, "percentage": 69.95, "elapsed_time": "2:15:52", "remaining_time": "0:58:21"} +{"current_steps": 150, "total_steps": 213, "loss": 0.4293328523635864, "lr": 2.523778285470835e-06, "epoch": 2.113777777777778, "percentage": 70.42, "elapsed_time": "2:16:47", "remaining_time": "0:57:27"} +{"current_steps": 150, "total_steps": 213, "eval_loss": 0.46622127294540405, "epoch": 2.113777777777778, "percentage": 70.42, "elapsed_time": "2:17:29", "remaining_time": "0:57:44"} +{"current_steps": 151, "total_steps": 213, "loss": 0.4054480195045471, "lr": 2.452669591802307e-06, "epoch": 2.128, "percentage": 70.89, "elapsed_time": "2:18:19", "remaining_time": "0:56:47"} +{"current_steps": 152, "total_steps": 213, "loss": 0.41965848207473755, "lr": 2.3822500397857016e-06, "epoch": 2.1422222222222222, "percentage": 71.36, "elapsed_time": "2:19:13", "remaining_time": "0:55:52"} +{"current_steps": 153, "total_steps": 213, "loss": 0.4354519248008728, "lr": 2.3125386803640188e-06, "epoch": 2.1564444444444444, "percentage": 71.83, "elapsed_time": "2:20:07", "remaining_time": "0:54:57"} +{"current_steps": 154, "total_steps": 213, "loss": 0.4178982675075531, "lr": 2.243554372889479e-06, "epoch": 2.1706666666666665, "percentage": 72.3, "elapsed_time": "2:21:01", "remaining_time": "0:54:01"} +{"current_steps": 155, "total_steps": 213, "loss": 0.4098745584487915, "lr": 2.1753157800214107e-06, "epoch": 2.1848888888888887, "percentage": 72.77, "elapsed_time": "2:21:55", "remaining_time": "0:53:06"} +{"current_steps": 156, "total_steps": 213, "loss": 0.43672817945480347, "lr": 2.1078413626773547e-06, "epoch": 2.1991111111111112, "percentage": 73.24, "elapsed_time": "2:22:48", "remaining_time": "0:52:10"} +{"current_steps": 157, "total_steps": 213, "loss": 0.39173248410224915, "lr": 2.0411493750387423e-06, "epoch": 2.2133333333333334, "percentage": 73.71, "elapsed_time": "2:23:39", "remaining_time": "0:51:14"} +{"current_steps": 158, "total_steps": 213, "loss": 0.44043296575546265, "lr": 1.9752578596124955e-06, "epoch": 2.2275555555555555, "percentage": 74.18, "elapsed_time": "2:24:31", "remaining_time": "0:50:18"} +{"current_steps": 159, "total_steps": 213, "loss": 0.41284430027008057, "lr": 1.9101846423499113e-06, "epoch": 2.2417777777777776, "percentage": 74.65, "elapsed_time": "2:25:22", "remaining_time": "0:49:22"} +{"current_steps": 160, "total_steps": 213, "loss": 0.41392982006073, "lr": 1.8459473278241125e-06, "epoch": 2.2560000000000002, "percentage": 75.12, "elapsed_time": "2:26:15", "remaining_time": "0:48:26"} +{"current_steps": 161, "total_steps": 213, "loss": 0.41735541820526123, "lr": 1.7825632944674016e-06, "epoch": 2.2702222222222224, "percentage": 75.59, "elapsed_time": "2:27:10", "remaining_time": "0:47:32"} +{"current_steps": 162, "total_steps": 213, "loss": 0.43122196197509766, "lr": 1.7200496898697832e-06, "epoch": 2.2844444444444445, "percentage": 76.06, "elapsed_time": "2:28:01", "remaining_time": "0:46:36"} +{"current_steps": 163, "total_steps": 213, "loss": 0.41986942291259766, "lr": 1.6584234261399535e-06, "epoch": 2.2986666666666666, "percentage": 76.53, "elapsed_time": "2:28:57", "remaining_time": "0:45:41"} +{"current_steps": 164, "total_steps": 213, "loss": 0.41601645946502686, "lr": 1.5977011753299726e-06, "epoch": 2.3128888888888888, "percentage": 77.0, "elapsed_time": "2:29:48", "remaining_time": "0:44:45"} +{"current_steps": 165, "total_steps": 213, "loss": 0.4430370330810547, "lr": 1.5378993649249053e-06, "epoch": 2.327111111111111, "percentage": 77.46, "elapsed_time": "2:30:37", "remaining_time": "0:43:49"} +{"current_steps": 166, "total_steps": 213, "loss": 0.41499751806259155, "lr": 1.4790341733986085e-06, "epoch": 2.3413333333333335, "percentage": 77.93, "elapsed_time": "2:31:30", "remaining_time": "0:42:53"} +{"current_steps": 167, "total_steps": 213, "loss": 0.40212416648864746, "lr": 1.4211215258368866e-06, "epoch": 2.3555555555555556, "percentage": 78.4, "elapsed_time": "2:32:24", "remaining_time": "0:41:58"} +{"current_steps": 168, "total_steps": 213, "loss": 0.40819668769836426, "lr": 1.3641770896292083e-06, "epoch": 2.3697777777777778, "percentage": 78.87, "elapsed_time": "2:33:20", "remaining_time": "0:41:04"} +{"current_steps": 169, "total_steps": 213, "loss": 0.4199577569961548, "lr": 1.3082162702301276e-06, "epoch": 2.384, "percentage": 79.34, "elapsed_time": "2:34:14", "remaining_time": "0:40:09"} +{"current_steps": 170, "total_steps": 213, "loss": 0.4247972071170807, "lr": 1.2532542069915722e-06, "epoch": 2.398222222222222, "percentage": 79.81, "elapsed_time": "2:35:06", "remaining_time": "0:39:13"} +{"current_steps": 171, "total_steps": 213, "loss": 0.41396448016166687, "lr": 1.1993057690671174e-06, "epoch": 2.4124444444444446, "percentage": 80.28, "elapsed_time": "2:35:57", "remaining_time": "0:38:18"} +{"current_steps": 172, "total_steps": 213, "loss": 0.43425244092941284, "lr": 1.1463855513893695e-06, "epoch": 2.4266666666666667, "percentage": 80.75, "elapsed_time": "2:36:52", "remaining_time": "0:37:23"} +{"current_steps": 173, "total_steps": 213, "loss": 0.42147308588027954, "lr": 1.0945078707215224e-06, "epoch": 2.440888888888889, "percentage": 81.22, "elapsed_time": "2:37:44", "remaining_time": "0:36:28"} +{"current_steps": 174, "total_steps": 213, "loss": 0.4213670492172241, "lr": 1.0436867617841768e-06, "epoch": 2.455111111111111, "percentage": 81.69, "elapsed_time": "2:38:35", "remaining_time": "0:35:32"} +{"current_steps": 175, "total_steps": 213, "loss": 0.4306361675262451, "lr": 9.939359734584552e-07, "epoch": 2.469333333333333, "percentage": 82.16, "elapsed_time": "2:39:27", "remaining_time": "0:34:37"} +{"current_steps": 176, "total_steps": 213, "loss": 0.4333479404449463, "lr": 9.452689650664515e-07, "epoch": 2.4835555555555557, "percentage": 82.63, "elapsed_time": "2:40:20", "remaining_time": "0:33:42"} +{"current_steps": 177, "total_steps": 213, "loss": 0.4236246347427368, "lr": 8.976989027300265e-07, "epoch": 2.497777777777778, "percentage": 83.1, "elapsed_time": "2:41:14", "remaining_time": "0:32:47"} +{"current_steps": 178, "total_steps": 213, "loss": 0.4386535882949829, "lr": 8.512386558088919e-07, "epoch": 2.512, "percentage": 83.57, "elapsed_time": "2:42:06", "remaining_time": "0:31:52"} +{"current_steps": 179, "total_steps": 213, "loss": 0.42091357707977295, "lr": 8.059007934190194e-07, "epoch": 2.526222222222222, "percentage": 84.04, "elapsed_time": "2:43:00", "remaining_time": "0:30:57"} +{"current_steps": 180, "total_steps": 213, "loss": 0.44629859924316406, "lr": 7.61697581032243e-07, "epoch": 2.5404444444444443, "percentage": 84.51, "elapsed_time": "2:43:51", "remaining_time": "0:30:02"} +{"current_steps": 181, "total_steps": 213, "loss": 0.4341578185558319, "lr": 7.186409771580355e-07, "epoch": 2.554666666666667, "percentage": 84.98, "elapsed_time": "2:44:47", "remaining_time": "0:29:08"} +{"current_steps": 182, "total_steps": 213, "loss": 0.4224834442138672, "lr": 6.76742630108298e-07, "epoch": 2.568888888888889, "percentage": 85.45, "elapsed_time": "2:45:41", "remaining_time": "0:28:13"} +{"current_steps": 183, "total_steps": 213, "loss": 0.4217371344566345, "lr": 6.360138748461015e-07, "epoch": 2.583111111111111, "percentage": 85.92, "elapsed_time": "2:46:36", "remaining_time": "0:27:18"} +{"current_steps": 184, "total_steps": 213, "loss": 0.4117254614830017, "lr": 5.964657299191712e-07, "epoch": 2.5973333333333333, "percentage": 86.38, "elapsed_time": "2:47:29", "remaining_time": "0:26:23"} +{"current_steps": 185, "total_steps": 213, "loss": 0.4190234839916229, "lr": 5.581088944789953e-07, "epoch": 2.6115555555555554, "percentage": 86.85, "elapsed_time": "2:48:25", "remaining_time": "0:25:29"} +{"current_steps": 186, "total_steps": 213, "loss": 0.4357784390449524, "lr": 5.20953745386329e-07, "epoch": 2.6257777777777775, "percentage": 87.32, "elapsed_time": "2:49:18", "remaining_time": "0:24:34"} +{"current_steps": 187, "total_steps": 213, "loss": 0.43076783418655396, "lr": 4.850103344038853e-07, "epoch": 2.64, "percentage": 87.79, "elapsed_time": "2:50:11", "remaining_time": "0:23:39"} +{"current_steps": 188, "total_steps": 213, "loss": 0.4010687470436096, "lr": 4.502883854769935e-07, "epoch": 2.6542222222222223, "percentage": 88.26, "elapsed_time": "2:51:04", "remaining_time": "0:22:44"} +{"current_steps": 189, "total_steps": 213, "loss": 0.4280475378036499, "lr": 4.167972921029262e-07, "epoch": 2.6684444444444444, "percentage": 88.73, "elapsed_time": "2:51:57", "remaining_time": "0:21:50"} +{"current_steps": 190, "total_steps": 213, "loss": 0.4384361803531647, "lr": 3.8454611478963235e-07, "epoch": 2.6826666666666665, "percentage": 89.2, "elapsed_time": "2:52:48", "remaining_time": "0:20:55"} +{"current_steps": 191, "total_steps": 213, "loss": 0.4107455611228943, "lr": 3.535435786045538e-07, "epoch": 2.696888888888889, "percentage": 89.67, "elapsed_time": "2:53:39", "remaining_time": "0:20:00"} +{"current_steps": 192, "total_steps": 213, "loss": 0.42101866006851196, "lr": 3.237980708141919e-07, "epoch": 2.7111111111111112, "percentage": 90.14, "elapsed_time": "2:54:31", "remaining_time": "0:19:05"} +{"current_steps": 193, "total_steps": 213, "loss": 0.43159422278404236, "lr": 2.9531763861505967e-07, "epoch": 2.7253333333333334, "percentage": 90.61, "elapsed_time": "2:55:24", "remaining_time": "0:18:10"} +{"current_steps": 194, "total_steps": 213, "loss": 0.4079360365867615, "lr": 2.681099869566328e-07, "epoch": 2.7395555555555555, "percentage": 91.08, "elapsed_time": "2:56:15", "remaining_time": "0:17:15"} +{"current_steps": 195, "total_steps": 213, "loss": 0.4277493357658386, "lr": 2.4218247645689306e-07, "epoch": 2.7537777777777777, "percentage": 91.55, "elapsed_time": "2:57:09", "remaining_time": "0:16:21"} +{"current_steps": 196, "total_steps": 213, "loss": 0.4117092490196228, "lr": 2.175421214110235e-07, "epoch": 2.768, "percentage": 92.02, "elapsed_time": "2:58:02", "remaining_time": "0:15:26"} +{"current_steps": 197, "total_steps": 213, "loss": 0.42654484510421753, "lr": 1.941955878938029e-07, "epoch": 2.7822222222222224, "percentage": 92.49, "elapsed_time": "2:58:55", "remaining_time": "0:14:31"} +{"current_steps": 198, "total_steps": 213, "loss": 0.43574994802474976, "lr": 1.721491919561913e-07, "epoch": 2.7964444444444445, "percentage": 92.96, "elapsed_time": "2:59:47", "remaining_time": "0:13:37"} +{"current_steps": 199, "total_steps": 213, "loss": 0.41729098558425903, "lr": 1.514088979166256e-07, "epoch": 2.8106666666666666, "percentage": 93.43, "elapsed_time": "3:00:41", "remaining_time": "0:12:42"} +{"current_steps": 200, "total_steps": 213, "loss": 0.42997270822525024, "lr": 1.3198031674745814e-07, "epoch": 2.824888888888889, "percentage": 93.9, "elapsed_time": "3:01:34", "remaining_time": "0:11:48"} +{"current_steps": 200, "total_steps": 213, "eval_loss": 0.4636807143688202, "epoch": 2.824888888888889, "percentage": 93.9, "elapsed_time": "3:02:12", "remaining_time": "0:11:50"} +{"current_steps": 201, "total_steps": 213, "loss": 0.42529386281967163, "lr": 1.138687045569975e-07, "epoch": 2.8391111111111114, "percentage": 94.37, "elapsed_time": "3:03:08", "remaining_time": "0:10:56"} +{"current_steps": 202, "total_steps": 213, "loss": 0.4284723401069641, "lr": 9.70789611675449e-08, "epoch": 2.8533333333333335, "percentage": 94.84, "elapsed_time": "3:04:00", "remaining_time": "0:10:01"} +{"current_steps": 203, "total_steps": 213, "loss": 0.41172438859939575, "lr": 8.161562878982399e-08, "epoch": 2.8675555555555556, "percentage": 95.31, "elapsed_time": "3:04:52", "remaining_time": "0:09:06"} +{"current_steps": 204, "total_steps": 213, "loss": 0.43681836128234863, "lr": 6.74828907941516e-08, "epoch": 2.8817777777777778, "percentage": 95.77, "elapsed_time": "3:05:43", "remaining_time": "0:08:11"} +{"current_steps": 205, "total_steps": 213, "loss": 0.43698325753211975, "lr": 5.468457057869358e-08, "epoch": 2.896, "percentage": 96.24, "elapsed_time": "3:06:38", "remaining_time": "0:07:16"} +{"current_steps": 206, "total_steps": 213, "loss": 0.4135843515396118, "lr": 4.322413053509944e-08, "epoch": 2.910222222222222, "percentage": 96.71, "elapsed_time": "3:07:34", "remaining_time": "0:06:22"} +{"current_steps": 207, "total_steps": 213, "loss": 0.41366809606552124, "lr": 3.31046711118066e-08, "epoch": 2.924444444444444, "percentage": 97.18, "elapsed_time": "3:08:25", "remaining_time": "0:05:27"} +{"current_steps": 208, "total_steps": 213, "loss": 0.4246295094490051, "lr": 2.4328929975260262e-08, "epoch": 2.9386666666666668, "percentage": 97.65, "elapsed_time": "3:09:19", "remaining_time": "0:04:33"} +{"current_steps": 209, "total_steps": 213, "loss": 0.4110904037952423, "lr": 1.6899281269279756e-08, "epoch": 2.952888888888889, "percentage": 98.12, "elapsed_time": "3:10:10", "remaining_time": "0:03:38"} +{"current_steps": 210, "total_steps": 213, "loss": 0.4201211929321289, "lr": 1.0817734972768946e-08, "epoch": 2.967111111111111, "percentage": 98.59, "elapsed_time": "3:11:01", "remaining_time": "0:02:43"} +{"current_steps": 211, "total_steps": 213, "loss": 0.43755394220352173, "lr": 6.085936355947897e-09, "epoch": 2.981333333333333, "percentage": 99.06, "elapsed_time": "3:11:53", "remaining_time": "0:01:49"} +{"current_steps": 212, "total_steps": 213, "loss": 0.4097408652305603, "lr": 2.7051655352494654e-09, "epoch": 2.9955555555555557, "percentage": 99.53, "elapsed_time": "3:12:45", "remaining_time": "0:00:54"} +{"current_steps": 213, "total_steps": 213, "loss": 0.5003441572189331, "lr": 6.763371270035457e-10, "epoch": 3.0, "percentage": 100.0, "elapsed_time": "3:13:02", "remaining_time": "0:00:00"} +{"current_steps": 213, "total_steps": 213, "epoch": 3.0, "percentage": 100.0, "elapsed_time": "3:14:58", "remaining_time": "0:00:00"} diff --git a/qwen2.5-3b-instruct/trainer_state.json b/qwen2.5-3b-instruct/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..94894e8fe45380a5331af35dd03b0053844ca027 --- /dev/null +++ b/qwen2.5-3b-instruct/trainer_state.json @@ -0,0 +1,1566 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 50, + "global_step": 213, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.014222222222222223, + "grad_norm": 0.09173738956451416, + "learning_rate": 0.0, + "loss": 1.005206823348999, + "step": 1 + }, + { + "epoch": 0.028444444444444446, + "grad_norm": 0.09365915507078171, + "learning_rate": 4.5454545454545457e-07, + "loss": 1.0057916641235352, + "step": 2 + }, + { + "epoch": 0.042666666666666665, + "grad_norm": 0.0689043328166008, + "learning_rate": 9.090909090909091e-07, + "loss": 0.9406883716583252, + "step": 3 + }, + { + "epoch": 0.05688888888888889, + "grad_norm": 0.04593552276492119, + "learning_rate": 1.3636363636363636e-06, + "loss": 0.8941981792449951, + "step": 4 + }, + { + "epoch": 0.07111111111111111, + "grad_norm": 0.03204488381743431, + "learning_rate": 1.8181818181818183e-06, + "loss": 0.7926980257034302, + "step": 5 + }, + { + "epoch": 0.08533333333333333, + "grad_norm": 0.027753565460443497, + "learning_rate": 2.2727272727272728e-06, + "loss": 0.8305954933166504, + "step": 6 + }, + { + "epoch": 0.09955555555555555, + "grad_norm": 0.03171968460083008, + "learning_rate": 2.7272727272727272e-06, + "loss": 0.7948952913284302, + "step": 7 + }, + { + "epoch": 0.11377777777777778, + "grad_norm": 0.026748575270175934, + "learning_rate": 3.181818181818182e-06, + "loss": 0.7530462741851807, + "step": 8 + }, + { + "epoch": 0.128, + "grad_norm": 0.023413579910993576, + "learning_rate": 3.6363636363636366e-06, + "loss": 0.7157606482505798, + "step": 9 + }, + { + "epoch": 0.14222222222222222, + "grad_norm": 0.020688239485025406, + "learning_rate": 4.0909090909090915e-06, + "loss": 0.7008587718009949, + "step": 10 + }, + { + "epoch": 0.15644444444444444, + "grad_norm": 0.014185823500156403, + "learning_rate": 4.5454545454545455e-06, + "loss": 0.6668463945388794, + "step": 11 + }, + { + "epoch": 0.17066666666666666, + "grad_norm": 0.012938937172293663, + "learning_rate": 5e-06, + "loss": 0.6507903337478638, + "step": 12 + }, + { + "epoch": 0.18488888888888888, + "grad_norm": 0.012045802548527718, + "learning_rate": 5.4545454545454545e-06, + "loss": 0.6217454671859741, + "step": 13 + }, + { + "epoch": 0.1991111111111111, + "grad_norm": 0.009660671465098858, + "learning_rate": 5.90909090909091e-06, + "loss": 0.6269917488098145, + "step": 14 + }, + { + "epoch": 0.21333333333333335, + "grad_norm": 0.008694784715771675, + "learning_rate": 6.363636363636364e-06, + "loss": 0.5836964845657349, + "step": 15 + }, + { + "epoch": 0.22755555555555557, + "grad_norm": 0.008789200335741043, + "learning_rate": 6.818181818181818e-06, + "loss": 0.6187424659729004, + "step": 16 + }, + { + "epoch": 0.24177777777777779, + "grad_norm": 0.008197025395929813, + "learning_rate": 7.272727272727273e-06, + "loss": 0.5914188027381897, + "step": 17 + }, + { + "epoch": 0.256, + "grad_norm": 0.007580601144582033, + "learning_rate": 7.727272727272727e-06, + "loss": 0.5802223682403564, + "step": 18 + }, + { + "epoch": 0.2702222222222222, + "grad_norm": 0.007013861555606127, + "learning_rate": 8.181818181818183e-06, + "loss": 0.5884265899658203, + "step": 19 + }, + { + "epoch": 0.28444444444444444, + "grad_norm": 0.00683254050090909, + "learning_rate": 8.636363636363637e-06, + "loss": 0.5850374698638916, + "step": 20 + }, + { + "epoch": 0.2986666666666667, + "grad_norm": 0.006111981347203255, + "learning_rate": 9.090909090909091e-06, + "loss": 0.5651419162750244, + "step": 21 + }, + { + "epoch": 0.3128888888888889, + "grad_norm": 0.005564771592617035, + "learning_rate": 9.545454545454547e-06, + "loss": 0.553795576095581, + "step": 22 + }, + { + "epoch": 0.32711111111111113, + "grad_norm": 0.0057683708146214485, + "learning_rate": 1e-05, + "loss": 0.5542658567428589, + "step": 23 + }, + { + "epoch": 0.3413333333333333, + "grad_norm": 0.006690182723104954, + "learning_rate": 9.999323662872998e-06, + "loss": 0.5586982369422913, + "step": 24 + }, + { + "epoch": 0.35555555555555557, + "grad_norm": 0.006314165890216827, + "learning_rate": 9.99729483446475e-06, + "loss": 0.5584714412689209, + "step": 25 + }, + { + "epoch": 0.36977777777777776, + "grad_norm": 0.005813857074826956, + "learning_rate": 9.993914063644053e-06, + "loss": 0.5466614961624146, + "step": 26 + }, + { + "epoch": 0.384, + "grad_norm": 0.005381743889302015, + "learning_rate": 9.989182265027232e-06, + "loss": 0.5380265712738037, + "step": 27 + }, + { + "epoch": 0.3982222222222222, + "grad_norm": 0.0049252682365477085, + "learning_rate": 9.98310071873072e-06, + "loss": 0.535520613193512, + "step": 28 + }, + { + "epoch": 0.41244444444444445, + "grad_norm": 0.0048560877330601215, + "learning_rate": 9.975671070024741e-06, + "loss": 0.5377035140991211, + "step": 29 + }, + { + "epoch": 0.4266666666666667, + "grad_norm": 0.004911178257316351, + "learning_rate": 9.966895328888195e-06, + "loss": 0.5570093989372253, + "step": 30 + }, + { + "epoch": 0.4408888888888889, + "grad_norm": 0.004746970720589161, + "learning_rate": 9.956775869464901e-06, + "loss": 0.5285611748695374, + "step": 31 + }, + { + "epoch": 0.45511111111111113, + "grad_norm": 0.0047261337749660015, + "learning_rate": 9.945315429421307e-06, + "loss": 0.5199858546257019, + "step": 32 + }, + { + "epoch": 0.4693333333333333, + "grad_norm": 0.004447608254849911, + "learning_rate": 9.932517109205849e-06, + "loss": 0.5250071287155151, + "step": 33 + }, + { + "epoch": 0.48355555555555557, + "grad_norm": 0.004617375787347555, + "learning_rate": 9.918384371210178e-06, + "loss": 0.5516105890274048, + "step": 34 + }, + { + "epoch": 0.49777777777777776, + "grad_norm": 0.004378544166684151, + "learning_rate": 9.902921038832456e-06, + "loss": 0.5298895835876465, + "step": 35 + }, + { + "epoch": 0.512, + "grad_norm": 0.004164176061749458, + "learning_rate": 9.886131295443003e-06, + "loss": 0.5044555068016052, + "step": 36 + }, + { + "epoch": 0.5262222222222223, + "grad_norm": 0.004092546179890633, + "learning_rate": 9.868019683252543e-06, + "loss": 0.5102939009666443, + "step": 37 + }, + { + "epoch": 0.5404444444444444, + "grad_norm": 0.004554086830466986, + "learning_rate": 9.848591102083375e-06, + "loss": 0.5421494245529175, + "step": 38 + }, + { + "epoch": 0.5546666666666666, + "grad_norm": 0.004217132460325956, + "learning_rate": 9.82785080804381e-06, + "loss": 0.5239347219467163, + "step": 39 + }, + { + "epoch": 0.5688888888888889, + "grad_norm": 0.004286583047360182, + "learning_rate": 9.805804412106197e-06, + "loss": 0.5437066555023193, + "step": 40 + }, + { + "epoch": 0.5831111111111111, + "grad_norm": 0.004261231981217861, + "learning_rate": 9.782457878588977e-06, + "loss": 0.5146951079368591, + "step": 41 + }, + { + "epoch": 0.5973333333333334, + "grad_norm": 0.004209685139358044, + "learning_rate": 9.75781752354311e-06, + "loss": 0.4995099604129791, + "step": 42 + }, + { + "epoch": 0.6115555555555555, + "grad_norm": 0.0039211539551615715, + "learning_rate": 9.731890013043367e-06, + "loss": 0.5249044299125671, + "step": 43 + }, + { + "epoch": 0.6257777777777778, + "grad_norm": 0.0038641595747321844, + "learning_rate": 9.704682361384941e-06, + "loss": 0.4980819821357727, + "step": 44 + }, + { + "epoch": 0.64, + "grad_norm": 0.004069323651492596, + "learning_rate": 9.676201929185809e-06, + "loss": 0.5113821625709534, + "step": 45 + }, + { + "epoch": 0.6542222222222223, + "grad_norm": 0.0038530679885298014, + "learning_rate": 9.646456421395447e-06, + "loss": 0.5072203278541565, + "step": 46 + }, + { + "epoch": 0.6684444444444444, + "grad_norm": 0.003951605875045061, + "learning_rate": 9.615453885210368e-06, + "loss": 0.509921669960022, + "step": 47 + }, + { + "epoch": 0.6826666666666666, + "grad_norm": 0.0038154355715960264, + "learning_rate": 9.583202707897075e-06, + "loss": 0.49035143852233887, + "step": 48 + }, + { + "epoch": 0.6968888888888889, + "grad_norm": 0.004143900703638792, + "learning_rate": 9.549711614523007e-06, + "loss": 0.4858332574367523, + "step": 49 + }, + { + "epoch": 0.7111111111111111, + "grad_norm": 0.0038300366140902042, + "learning_rate": 9.514989665596114e-06, + "loss": 0.49062296748161316, + "step": 50 + }, + { + "epoch": 0.7111111111111111, + "eval_loss": 0.5049259662628174, + "eval_runtime": 38.0442, + "eval_samples_per_second": 5.257, + "eval_steps_per_second": 2.629, + "step": 50 + }, + { + "epoch": 0.7253333333333334, + "grad_norm": 0.003965203184634447, + "learning_rate": 9.479046254613673e-06, + "loss": 0.5013079643249512, + "step": 51 + }, + { + "epoch": 0.7395555555555555, + "grad_norm": 0.003626414341852069, + "learning_rate": 9.441891105521005e-06, + "loss": 0.46776771545410156, + "step": 52 + }, + { + "epoch": 0.7537777777777778, + "grad_norm": 0.003647371893748641, + "learning_rate": 9.40353427008083e-06, + "loss": 0.4870305061340332, + "step": 53 + }, + { + "epoch": 0.768, + "grad_norm": 0.0036924544256180525, + "learning_rate": 9.3639861251539e-06, + "loss": 0.48354169726371765, + "step": 54 + }, + { + "epoch": 0.7822222222222223, + "grad_norm": 0.0037534767761826515, + "learning_rate": 9.323257369891702e-06, + "loss": 0.5097652673721313, + "step": 55 + }, + { + "epoch": 0.7964444444444444, + "grad_norm": 0.003901001764461398, + "learning_rate": 9.281359022841966e-06, + "loss": 0.5023170709609985, + "step": 56 + }, + { + "epoch": 0.8106666666666666, + "grad_norm": 0.0037305736914277077, + "learning_rate": 9.238302418967757e-06, + "loss": 0.49597933888435364, + "step": 57 + }, + { + "epoch": 0.8248888888888889, + "grad_norm": 0.003764534369111061, + "learning_rate": 9.194099206580981e-06, + "loss": 0.504522442817688, + "step": 58 + }, + { + "epoch": 0.8391111111111111, + "grad_norm": 0.004040914587676525, + "learning_rate": 9.14876134419111e-06, + "loss": 0.49795830249786377, + "step": 59 + }, + { + "epoch": 0.8533333333333334, + "grad_norm": 0.003855582093819976, + "learning_rate": 9.102301097269974e-06, + "loss": 0.49624279141426086, + "step": 60 + }, + { + "epoch": 0.8675555555555555, + "grad_norm": 0.003851535264402628, + "learning_rate": 9.05473103493355e-06, + "loss": 0.47714120149612427, + "step": 61 + }, + { + "epoch": 0.8817777777777778, + "grad_norm": 0.004122862592339516, + "learning_rate": 9.006064026541549e-06, + "loss": 0.5046075582504272, + "step": 62 + }, + { + "epoch": 0.896, + "grad_norm": 0.0036710964050143957, + "learning_rate": 8.956313238215824e-06, + "loss": 0.45711296796798706, + "step": 63 + }, + { + "epoch": 0.9102222222222223, + "grad_norm": 0.004037045873701572, + "learning_rate": 8.905492129278478e-06, + "loss": 0.4839310050010681, + "step": 64 + }, + { + "epoch": 0.9244444444444444, + "grad_norm": 0.004081980790942907, + "learning_rate": 8.85361444861063e-06, + "loss": 0.4877704381942749, + "step": 65 + }, + { + "epoch": 0.9386666666666666, + "grad_norm": 0.0037577205803245306, + "learning_rate": 8.800694230932885e-06, + "loss": 0.49369895458221436, + "step": 66 + }, + { + "epoch": 0.9528888888888889, + "grad_norm": 0.0037781032733619213, + "learning_rate": 8.74674579300843e-06, + "loss": 0.4812237620353699, + "step": 67 + }, + { + "epoch": 0.9671111111111111, + "grad_norm": 0.0046208929270505905, + "learning_rate": 8.691783729769874e-06, + "loss": 0.4722365736961365, + "step": 68 + }, + { + "epoch": 0.9813333333333333, + "grad_norm": 0.004026597365736961, + "learning_rate": 8.635822910370793e-06, + "loss": 0.4962988495826721, + "step": 69 + }, + { + "epoch": 0.9955555555555555, + "grad_norm": 0.004095569718629122, + "learning_rate": 8.578878474163115e-06, + "loss": 0.45991775393486023, + "step": 70 + }, + { + "epoch": 1.0, + "grad_norm": 0.006300764624029398, + "learning_rate": 8.520965826601394e-06, + "loss": 0.4934382140636444, + "step": 71 + }, + { + "epoch": 1.0142222222222221, + "grad_norm": 0.00428728898987174, + "learning_rate": 8.462100635075097e-06, + "loss": 0.4847542643547058, + "step": 72 + }, + { + "epoch": 1.0284444444444445, + "grad_norm": 0.004186135251075029, + "learning_rate": 8.40229882467003e-06, + "loss": 0.4636518955230713, + "step": 73 + }, + { + "epoch": 1.0426666666666666, + "grad_norm": 0.003916833084076643, + "learning_rate": 8.341576573860049e-06, + "loss": 0.44187992811203003, + "step": 74 + }, + { + "epoch": 1.056888888888889, + "grad_norm": 0.004144088365137577, + "learning_rate": 8.279950310130218e-06, + "loss": 0.467815637588501, + "step": 75 + }, + { + "epoch": 1.0711111111111111, + "grad_norm": 0.004077525809407234, + "learning_rate": 8.2174367055326e-06, + "loss": 0.451981782913208, + "step": 76 + }, + { + "epoch": 1.0853333333333333, + "grad_norm": 0.004103589337319136, + "learning_rate": 8.154052672175888e-06, + "loss": 0.4449639320373535, + "step": 77 + }, + { + "epoch": 1.0995555555555556, + "grad_norm": 0.00439891591668129, + "learning_rate": 8.08981535765009e-06, + "loss": 0.47661933302879333, + "step": 78 + }, + { + "epoch": 1.1137777777777778, + "grad_norm": 0.004592400509864092, + "learning_rate": 8.024742140387506e-06, + "loss": 0.45776277780532837, + "step": 79 + }, + { + "epoch": 1.1280000000000001, + "grad_norm": 0.0037961499765515327, + "learning_rate": 7.95885062496126e-06, + "loss": 0.44757023453712463, + "step": 80 + }, + { + "epoch": 1.1422222222222222, + "grad_norm": 0.003865304868668318, + "learning_rate": 7.892158637322647e-06, + "loss": 0.419955849647522, + "step": 81 + }, + { + "epoch": 1.1564444444444444, + "grad_norm": 0.00407387875020504, + "learning_rate": 7.824684219978591e-06, + "loss": 0.4457795023918152, + "step": 82 + }, + { + "epoch": 1.1706666666666667, + "grad_norm": 0.00370654184371233, + "learning_rate": 7.756445627110523e-06, + "loss": 0.45605674386024475, + "step": 83 + }, + { + "epoch": 1.1848888888888889, + "grad_norm": 0.003941345028579235, + "learning_rate": 7.68746131963598e-06, + "loss": 0.46289366483688354, + "step": 84 + }, + { + "epoch": 1.199111111111111, + "grad_norm": 0.0039403438568115234, + "learning_rate": 7.6177499602143e-06, + "loss": 0.45568281412124634, + "step": 85 + }, + { + "epoch": 1.2133333333333334, + "grad_norm": 0.003944277763366699, + "learning_rate": 7.547330408197695e-06, + "loss": 0.44618040323257446, + "step": 86 + }, + { + "epoch": 1.2275555555555555, + "grad_norm": 0.0040544732473790646, + "learning_rate": 7.476221714529167e-06, + "loss": 0.45724251866340637, + "step": 87 + }, + { + "epoch": 1.2417777777777779, + "grad_norm": 0.004533152561634779, + "learning_rate": 7.404443116588548e-06, + "loss": 0.4806937575340271, + "step": 88 + }, + { + "epoch": 1.256, + "grad_norm": 0.004002939909696579, + "learning_rate": 7.332014032988123e-06, + "loss": 0.4561174213886261, + "step": 89 + }, + { + "epoch": 1.2702222222222221, + "grad_norm": 0.003825302002951503, + "learning_rate": 7.2589540583192165e-06, + "loss": 0.4558497667312622, + "step": 90 + }, + { + "epoch": 1.2844444444444445, + "grad_norm": 0.00397696765139699, + "learning_rate": 7.185282957851175e-06, + "loss": 0.4397018551826477, + "step": 91 + }, + { + "epoch": 1.2986666666666666, + "grad_norm": 0.004289410542696714, + "learning_rate": 7.111020662184174e-06, + "loss": 0.45822006464004517, + "step": 92 + }, + { + "epoch": 1.3128888888888888, + "grad_norm": 0.004002283327281475, + "learning_rate": 7.036187261857289e-06, + "loss": 0.4479483366012573, + "step": 93 + }, + { + "epoch": 1.3271111111111111, + "grad_norm": 0.004231768660247326, + "learning_rate": 6.960803001913315e-06, + "loss": 0.4738925099372864, + "step": 94 + }, + { + "epoch": 1.3413333333333333, + "grad_norm": 0.0040559363551437855, + "learning_rate": 6.884888276421766e-06, + "loss": 0.4614931643009186, + "step": 95 + }, + { + "epoch": 1.3555555555555556, + "grad_norm": 0.003587141167372465, + "learning_rate": 6.8084636229615786e-06, + "loss": 0.44328922033309937, + "step": 96 + }, + { + "epoch": 1.3697777777777778, + "grad_norm": 0.004009770695120096, + "learning_rate": 6.731549717064975e-06, + "loss": 0.44612404704093933, + "step": 97 + }, + { + "epoch": 1.384, + "grad_norm": 0.004132336005568504, + "learning_rate": 6.654167366624009e-06, + "loss": 0.4554000794887543, + "step": 98 + }, + { + "epoch": 1.3982222222222223, + "grad_norm": 0.0041816201992332935, + "learning_rate": 6.576337506261314e-06, + "loss": 0.45616668462753296, + "step": 99 + }, + { + "epoch": 1.4124444444444444, + "grad_norm": 0.003568898420780897, + "learning_rate": 6.498081191666549e-06, + "loss": 0.45816588401794434, + "step": 100 + }, + { + "epoch": 1.4124444444444444, + "eval_loss": 0.4757018983364105, + "eval_runtime": 39.6302, + "eval_samples_per_second": 5.047, + "eval_steps_per_second": 2.523, + "step": 100 + }, + { + "epoch": 1.4266666666666667, + "grad_norm": 0.004014050588011742, + "learning_rate": 6.419419593900109e-06, + "loss": 0.4430053234100342, + "step": 101 + }, + { + "epoch": 1.4408888888888889, + "grad_norm": 0.0039361282251775265, + "learning_rate": 6.340373993665607e-06, + "loss": 0.43451493978500366, + "step": 102 + }, + { + "epoch": 1.455111111111111, + "grad_norm": 0.0040025534108281136, + "learning_rate": 6.2609657755527135e-06, + "loss": 0.4296550154685974, + "step": 103 + }, + { + "epoch": 1.4693333333333334, + "grad_norm": 0.004081938415765762, + "learning_rate": 6.1812164222518626e-06, + "loss": 0.46728163957595825, + "step": 104 + }, + { + "epoch": 1.4835555555555555, + "grad_norm": 0.0043087489902973175, + "learning_rate": 6.101147508742456e-06, + "loss": 0.446887731552124, + "step": 105 + }, + { + "epoch": 1.4977777777777779, + "grad_norm": 0.0037424617912620306, + "learning_rate": 6.020780696456059e-06, + "loss": 0.43969088792800903, + "step": 106 + }, + { + "epoch": 1.512, + "grad_norm": 0.003990524914115667, + "learning_rate": 5.940137727416247e-06, + "loss": 0.4489642381668091, + "step": 107 + }, + { + "epoch": 1.5262222222222221, + "grad_norm": 0.00401579262688756, + "learning_rate": 5.859240418356614e-06, + "loss": 0.4380025863647461, + "step": 108 + }, + { + "epoch": 1.5404444444444443, + "grad_norm": 0.0042803785763680935, + "learning_rate": 5.778110654818602e-06, + "loss": 0.47752195596694946, + "step": 109 + }, + { + "epoch": 1.5546666666666666, + "grad_norm": 0.003838231321424246, + "learning_rate": 5.696770385230679e-06, + "loss": 0.4498862028121948, + "step": 110 + }, + { + "epoch": 1.568888888888889, + "grad_norm": 0.0041479310020804405, + "learning_rate": 5.615241614970546e-06, + "loss": 0.44400298595428467, + "step": 111 + }, + { + "epoch": 1.5831111111111111, + "grad_norm": 0.0038208707701414824, + "learning_rate": 5.533546400411899e-06, + "loss": 0.4457319378852844, + "step": 112 + }, + { + "epoch": 1.5973333333333333, + "grad_norm": 0.0038792924024164677, + "learning_rate": 5.4517068429574215e-06, + "loss": 0.4610831141471863, + "step": 113 + }, + { + "epoch": 1.6115555555555554, + "grad_norm": 0.004012539051473141, + "learning_rate": 5.3697450830595775e-06, + "loss": 0.4425020217895508, + "step": 114 + }, + { + "epoch": 1.6257777777777778, + "grad_norm": 0.003996311221271753, + "learning_rate": 5.287683294230855e-06, + "loss": 0.44555309414863586, + "step": 115 + }, + { + "epoch": 1.6400000000000001, + "grad_norm": 0.0037504970096051693, + "learning_rate": 5.20554367704505e-06, + "loss": 0.417566180229187, + "step": 116 + }, + { + "epoch": 1.6542222222222223, + "grad_norm": 0.003828976536169648, + "learning_rate": 5.123348453131242e-06, + "loss": 0.45680171251296997, + "step": 117 + }, + { + "epoch": 1.6684444444444444, + "grad_norm": 0.0040869563817977905, + "learning_rate": 5.041119859162068e-06, + "loss": 0.4432540535926819, + "step": 118 + }, + { + "epoch": 1.6826666666666665, + "grad_norm": 0.004195951856672764, + "learning_rate": 4.958880140837934e-06, + "loss": 0.44924670457839966, + "step": 119 + }, + { + "epoch": 1.696888888888889, + "grad_norm": 0.003907010890543461, + "learning_rate": 4.876651546868759e-06, + "loss": 0.43826115131378174, + "step": 120 + }, + { + "epoch": 1.7111111111111112, + "grad_norm": 0.003893865505233407, + "learning_rate": 4.794456322954953e-06, + "loss": 0.4698702096939087, + "step": 121 + }, + { + "epoch": 1.7253333333333334, + "grad_norm": 0.003987904172390699, + "learning_rate": 4.7123167057691446e-06, + "loss": 0.4665527045726776, + "step": 122 + }, + { + "epoch": 1.7395555555555555, + "grad_norm": 0.003826166968792677, + "learning_rate": 4.630254916940424e-06, + "loss": 0.46612560749053955, + "step": 123 + }, + { + "epoch": 1.7537777777777777, + "grad_norm": 0.0037139400374144316, + "learning_rate": 4.548293157042581e-06, + "loss": 0.4559047818183899, + "step": 124 + }, + { + "epoch": 1.768, + "grad_norm": 0.0038443440571427345, + "learning_rate": 4.466453599588103e-06, + "loss": 0.4595223069190979, + "step": 125 + }, + { + "epoch": 1.7822222222222224, + "grad_norm": 0.0039060397539287806, + "learning_rate": 4.384758385029457e-06, + "loss": 0.42560791969299316, + "step": 126 + }, + { + "epoch": 1.7964444444444445, + "grad_norm": 0.0041647544130682945, + "learning_rate": 4.3032296147693225e-06, + "loss": 0.43939000368118286, + "step": 127 + }, + { + "epoch": 1.8106666666666666, + "grad_norm": 0.004102814476937056, + "learning_rate": 4.2218893451814005e-06, + "loss": 0.4399089217185974, + "step": 128 + }, + { + "epoch": 1.8248888888888888, + "grad_norm": 0.003889351151883602, + "learning_rate": 4.140759581643386e-06, + "loss": 0.4451609253883362, + "step": 129 + }, + { + "epoch": 1.8391111111111111, + "grad_norm": 0.004501548130065203, + "learning_rate": 4.059862272583755e-06, + "loss": 0.4488486051559448, + "step": 130 + }, + { + "epoch": 1.8533333333333335, + "grad_norm": 0.0038232814986258745, + "learning_rate": 3.979219303543942e-06, + "loss": 0.42662614583969116, + "step": 131 + }, + { + "epoch": 1.8675555555555556, + "grad_norm": 0.003998646512627602, + "learning_rate": 3.898852491257547e-06, + "loss": 0.4547004997730255, + "step": 132 + }, + { + "epoch": 1.8817777777777778, + "grad_norm": 0.00403416994959116, + "learning_rate": 3.818783577748138e-06, + "loss": 0.4510434567928314, + "step": 133 + }, + { + "epoch": 1.896, + "grad_norm": 0.003799683880060911, + "learning_rate": 3.7390342244472886e-06, + "loss": 0.45353883504867554, + "step": 134 + }, + { + "epoch": 1.9102222222222223, + "grad_norm": 0.003921311814337969, + "learning_rate": 3.659626006334395e-06, + "loss": 0.4302864670753479, + "step": 135 + }, + { + "epoch": 1.9244444444444444, + "grad_norm": 0.00398287083953619, + "learning_rate": 3.5805804060998926e-06, + "loss": 0.4398406744003296, + "step": 136 + }, + { + "epoch": 1.9386666666666668, + "grad_norm": 0.0037779631093144417, + "learning_rate": 3.501918808333453e-06, + "loss": 0.465728759765625, + "step": 137 + }, + { + "epoch": 1.952888888888889, + "grad_norm": 0.003967550117522478, + "learning_rate": 3.4236624937386874e-06, + "loss": 0.4686010777950287, + "step": 138 + }, + { + "epoch": 1.967111111111111, + "grad_norm": 0.00354909454472363, + "learning_rate": 3.3458326333759927e-06, + "loss": 0.43872472643852234, + "step": 139 + }, + { + "epoch": 1.9813333333333332, + "grad_norm": 0.00383565342053771, + "learning_rate": 3.268450282935026e-06, + "loss": 0.4455334544181824, + "step": 140 + }, + { + "epoch": 1.9955555555555555, + "grad_norm": 0.0037798655685037374, + "learning_rate": 3.1915363770384223e-06, + "loss": 0.43381941318511963, + "step": 141 + }, + { + "epoch": 2.0, + "grad_norm": 0.00621951324865222, + "learning_rate": 3.1151117235782346e-06, + "loss": 0.4559262990951538, + "step": 142 + }, + { + "epoch": 2.014222222222222, + "grad_norm": 0.004304260015487671, + "learning_rate": 3.0391969980866874e-06, + "loss": 0.43779897689819336, + "step": 143 + }, + { + "epoch": 2.0284444444444443, + "grad_norm": 0.003964607138186693, + "learning_rate": 2.963812738142713e-06, + "loss": 0.4199046492576599, + "step": 144 + }, + { + "epoch": 2.042666666666667, + "grad_norm": 0.004212403669953346, + "learning_rate": 2.8889793378158284e-06, + "loss": 0.43830418586730957, + "step": 145 + }, + { + "epoch": 2.056888888888889, + "grad_norm": 0.0037585427053272724, + "learning_rate": 2.814717042148827e-06, + "loss": 0.4249774217605591, + "step": 146 + }, + { + "epoch": 2.071111111111111, + "grad_norm": 0.003818828146904707, + "learning_rate": 2.7410459416807856e-06, + "loss": 0.41759681701660156, + "step": 147 + }, + { + "epoch": 2.0853333333333333, + "grad_norm": 0.0038647297769784927, + "learning_rate": 2.6679859670118785e-06, + "loss": 0.43880075216293335, + "step": 148 + }, + { + "epoch": 2.0995555555555554, + "grad_norm": 0.0037169770803302526, + "learning_rate": 2.5955568834114523e-06, + "loss": 0.4053642153739929, + "step": 149 + }, + { + "epoch": 2.113777777777778, + "grad_norm": 0.0037585999816656113, + "learning_rate": 2.523778285470835e-06, + "loss": 0.4293328523635864, + "step": 150 + }, + { + "epoch": 2.113777777777778, + "eval_loss": 0.46622127294540405, + "eval_runtime": 42.1452, + "eval_samples_per_second": 4.746, + "eval_steps_per_second": 2.373, + "step": 150 + }, + { + "epoch": 2.128, + "grad_norm": 0.004196058958768845, + "learning_rate": 2.452669591802307e-06, + "loss": 0.4054480195045471, + "step": 151 + }, + { + "epoch": 2.1422222222222222, + "grad_norm": 0.003635152243077755, + "learning_rate": 2.3822500397857016e-06, + "loss": 0.41965848207473755, + "step": 152 + }, + { + "epoch": 2.1564444444444444, + "grad_norm": 0.0042378562502563, + "learning_rate": 2.3125386803640188e-06, + "loss": 0.4354519248008728, + "step": 153 + }, + { + "epoch": 2.1706666666666665, + "grad_norm": 0.003634256776422262, + "learning_rate": 2.243554372889479e-06, + "loss": 0.4178982675075531, + "step": 154 + }, + { + "epoch": 2.1848888888888887, + "grad_norm": 0.003790440736338496, + "learning_rate": 2.1753157800214107e-06, + "loss": 0.4098745584487915, + "step": 155 + }, + { + "epoch": 2.1991111111111112, + "grad_norm": 0.0038817040622234344, + "learning_rate": 2.1078413626773547e-06, + "loss": 0.43672817945480347, + "step": 156 + }, + { + "epoch": 2.2133333333333334, + "grad_norm": 0.003961282316595316, + "learning_rate": 2.0411493750387423e-06, + "loss": 0.39173248410224915, + "step": 157 + }, + { + "epoch": 2.2275555555555555, + "grad_norm": 0.003964588046073914, + "learning_rate": 1.9752578596124955e-06, + "loss": 0.44043296575546265, + "step": 158 + }, + { + "epoch": 2.2417777777777776, + "grad_norm": 0.003892629873007536, + "learning_rate": 1.9101846423499113e-06, + "loss": 0.41284430027008057, + "step": 159 + }, + { + "epoch": 2.2560000000000002, + "grad_norm": 0.004033091943711042, + "learning_rate": 1.8459473278241125e-06, + "loss": 0.41392982006073, + "step": 160 + }, + { + "epoch": 2.2702222222222224, + "grad_norm": 0.003734015626832843, + "learning_rate": 1.7825632944674016e-06, + "loss": 0.41735541820526123, + "step": 161 + }, + { + "epoch": 2.2844444444444445, + "grad_norm": 0.003964587114751339, + "learning_rate": 1.7200496898697832e-06, + "loss": 0.43122196197509766, + "step": 162 + }, + { + "epoch": 2.2986666666666666, + "grad_norm": 0.0036877307575196028, + "learning_rate": 1.6584234261399535e-06, + "loss": 0.41986942291259766, + "step": 163 + }, + { + "epoch": 2.3128888888888888, + "grad_norm": 0.0038088378496468067, + "learning_rate": 1.5977011753299726e-06, + "loss": 0.41601645946502686, + "step": 164 + }, + { + "epoch": 2.327111111111111, + "grad_norm": 0.003943529445677996, + "learning_rate": 1.5378993649249053e-06, + "loss": 0.4430370330810547, + "step": 165 + }, + { + "epoch": 2.3413333333333335, + "grad_norm": 0.0038139254320412874, + "learning_rate": 1.4790341733986085e-06, + "loss": 0.41499751806259155, + "step": 166 + }, + { + "epoch": 2.3555555555555556, + "grad_norm": 0.003613486187532544, + "learning_rate": 1.4211215258368866e-06, + "loss": 0.40212416648864746, + "step": 167 + }, + { + "epoch": 2.3697777777777778, + "grad_norm": 0.0037240793462842703, + "learning_rate": 1.3641770896292083e-06, + "loss": 0.40819668769836426, + "step": 168 + }, + { + "epoch": 2.384, + "grad_norm": 0.003915995359420776, + "learning_rate": 1.3082162702301276e-06, + "loss": 0.4199577569961548, + "step": 169 + }, + { + "epoch": 2.398222222222222, + "grad_norm": 0.0038723198231309652, + "learning_rate": 1.2532542069915722e-06, + "loss": 0.4247972071170807, + "step": 170 + }, + { + "epoch": 2.4124444444444446, + "grad_norm": 0.0036193751730024815, + "learning_rate": 1.1993057690671174e-06, + "loss": 0.41396448016166687, + "step": 171 + }, + { + "epoch": 2.4266666666666667, + "grad_norm": 0.003564434824511409, + "learning_rate": 1.1463855513893695e-06, + "loss": 0.43425244092941284, + "step": 172 + }, + { + "epoch": 2.440888888888889, + "grad_norm": 0.003745888127014041, + "learning_rate": 1.0945078707215224e-06, + "loss": 0.42147308588027954, + "step": 173 + }, + { + "epoch": 2.455111111111111, + "grad_norm": 0.003768347902223468, + "learning_rate": 1.0436867617841768e-06, + "loss": 0.4213670492172241, + "step": 174 + }, + { + "epoch": 2.469333333333333, + "grad_norm": 0.0036286180838942528, + "learning_rate": 9.939359734584552e-07, + "loss": 0.4306361675262451, + "step": 175 + }, + { + "epoch": 2.4835555555555557, + "grad_norm": 0.0037941799964755774, + "learning_rate": 9.452689650664515e-07, + "loss": 0.4333479404449463, + "step": 176 + }, + { + "epoch": 2.497777777777778, + "grad_norm": 0.003679306246340275, + "learning_rate": 8.976989027300265e-07, + "loss": 0.4236246347427368, + "step": 177 + }, + { + "epoch": 2.512, + "grad_norm": 0.003801122773438692, + "learning_rate": 8.512386558088919e-07, + "loss": 0.4386535882949829, + "step": 178 + }, + { + "epoch": 2.526222222222222, + "grad_norm": 0.00364946317858994, + "learning_rate": 8.059007934190194e-07, + "loss": 0.42091357707977295, + "step": 179 + }, + { + "epoch": 2.5404444444444443, + "grad_norm": 0.003728381125256419, + "learning_rate": 7.61697581032243e-07, + "loss": 0.44629859924316406, + "step": 180 + }, + { + "epoch": 2.554666666666667, + "grad_norm": 0.003702825866639614, + "learning_rate": 7.186409771580355e-07, + "loss": 0.4341578185558319, + "step": 181 + }, + { + "epoch": 2.568888888888889, + "grad_norm": 0.00379473646171391, + "learning_rate": 6.76742630108298e-07, + "loss": 0.4224834442138672, + "step": 182 + }, + { + "epoch": 2.583111111111111, + "grad_norm": 0.003699215827509761, + "learning_rate": 6.360138748461015e-07, + "loss": 0.4217371344566345, + "step": 183 + }, + { + "epoch": 2.5973333333333333, + "grad_norm": 0.0036750342696905136, + "learning_rate": 5.964657299191712e-07, + "loss": 0.4117254614830017, + "step": 184 + }, + { + "epoch": 2.6115555555555554, + "grad_norm": 0.003708022413775325, + "learning_rate": 5.581088944789953e-07, + "loss": 0.4190234839916229, + "step": 185 + }, + { + "epoch": 2.6257777777777775, + "grad_norm": 0.00363738089799881, + "learning_rate": 5.20953745386329e-07, + "loss": 0.4357784390449524, + "step": 186 + }, + { + "epoch": 2.64, + "grad_norm": 0.0036364425905048847, + "learning_rate": 4.850103344038853e-07, + "loss": 0.43076783418655396, + "step": 187 + }, + { + "epoch": 2.6542222222222223, + "grad_norm": 0.003594007110223174, + "learning_rate": 4.502883854769935e-07, + "loss": 0.4010687470436096, + "step": 188 + }, + { + "epoch": 2.6684444444444444, + "grad_norm": 0.00375847308896482, + "learning_rate": 4.167972921029262e-07, + "loss": 0.4280475378036499, + "step": 189 + }, + { + "epoch": 2.6826666666666665, + "grad_norm": 0.003887615166604519, + "learning_rate": 3.8454611478963235e-07, + "loss": 0.4384361803531647, + "step": 190 + }, + { + "epoch": 2.696888888888889, + "grad_norm": 0.0036584900226444006, + "learning_rate": 3.535435786045538e-07, + "loss": 0.4107455611228943, + "step": 191 + }, + { + "epoch": 2.7111111111111112, + "grad_norm": 0.003653391497209668, + "learning_rate": 3.237980708141919e-07, + "loss": 0.42101866006851196, + "step": 192 + }, + { + "epoch": 2.7253333333333334, + "grad_norm": 0.003794454736635089, + "learning_rate": 2.9531763861505967e-07, + "loss": 0.43159422278404236, + "step": 193 + }, + { + "epoch": 2.7395555555555555, + "grad_norm": 0.003839928889647126, + "learning_rate": 2.681099869566328e-07, + "loss": 0.4079360365867615, + "step": 194 + }, + { + "epoch": 2.7537777777777777, + "grad_norm": 0.0035740381572395563, + "learning_rate": 2.4218247645689306e-07, + "loss": 0.4277493357658386, + "step": 195 + }, + { + "epoch": 2.768, + "grad_norm": 0.0035776360891759396, + "learning_rate": 2.175421214110235e-07, + "loss": 0.4117092490196228, + "step": 196 + }, + { + "epoch": 2.7822222222222224, + "grad_norm": 0.0038700539153069258, + "learning_rate": 1.941955878938029e-07, + "loss": 0.42654484510421753, + "step": 197 + }, + { + "epoch": 2.7964444444444445, + "grad_norm": 0.003715057857334614, + "learning_rate": 1.721491919561913e-07, + "loss": 0.43574994802474976, + "step": 198 + }, + { + "epoch": 2.8106666666666666, + "grad_norm": 0.0036776140332221985, + "learning_rate": 1.514088979166256e-07, + "loss": 0.41729098558425903, + "step": 199 + }, + { + "epoch": 2.824888888888889, + "grad_norm": 0.003615819849073887, + "learning_rate": 1.3198031674745814e-07, + "loss": 0.42997270822525024, + "step": 200 + }, + { + "epoch": 2.824888888888889, + "eval_loss": 0.4636807143688202, + "eval_runtime": 38.1939, + "eval_samples_per_second": 5.236, + "eval_steps_per_second": 2.618, + "step": 200 + }, + { + "epoch": 2.8391111111111114, + "grad_norm": 0.00375831825658679, + "learning_rate": 1.138687045569975e-07, + "loss": 0.42529386281967163, + "step": 201 + }, + { + "epoch": 2.8533333333333335, + "grad_norm": 0.0035394008737057447, + "learning_rate": 9.70789611675449e-08, + "loss": 0.4284723401069641, + "step": 202 + }, + { + "epoch": 2.8675555555555556, + "grad_norm": 0.0037534094881266356, + "learning_rate": 8.161562878982399e-08, + "loss": 0.41172438859939575, + "step": 203 + }, + { + "epoch": 2.8817777777777778, + "grad_norm": 0.003908019047230482, + "learning_rate": 6.74828907941516e-08, + "loss": 0.43681836128234863, + "step": 204 + }, + { + "epoch": 2.896, + "grad_norm": 0.003631765488535166, + "learning_rate": 5.468457057869358e-08, + "loss": 0.43698325753211975, + "step": 205 + }, + { + "epoch": 2.910222222222222, + "grad_norm": 0.003675674321129918, + "learning_rate": 4.322413053509944e-08, + "loss": 0.4135843515396118, + "step": 206 + }, + { + "epoch": 2.924444444444444, + "grad_norm": 0.0037287427112460136, + "learning_rate": 3.31046711118066e-08, + "loss": 0.41366809606552124, + "step": 207 + }, + { + "epoch": 2.9386666666666668, + "grad_norm": 0.0037086675874888897, + "learning_rate": 2.4328929975260262e-08, + "loss": 0.4246295094490051, + "step": 208 + }, + { + "epoch": 2.952888888888889, + "grad_norm": 0.0035672816447913647, + "learning_rate": 1.6899281269279756e-08, + "loss": 0.4110904037952423, + "step": 209 + }, + { + "epoch": 2.967111111111111, + "grad_norm": 0.0037544071674346924, + "learning_rate": 1.0817734972768946e-08, + "loss": 0.4201211929321289, + "step": 210 + }, + { + "epoch": 2.981333333333333, + "grad_norm": 0.004007895011454821, + "learning_rate": 6.085936355947897e-09, + "loss": 0.43755394220352173, + "step": 211 + }, + { + "epoch": 2.9955555555555557, + "grad_norm": 0.003630666760727763, + "learning_rate": 2.7051655352494654e-09, + "loss": 0.4097408652305603, + "step": 212 + }, + { + "epoch": 3.0, + "grad_norm": 0.00691953394562006, + "learning_rate": 6.763371270035457e-10, + "loss": 0.5003441572189331, + "step": 213 + }, + { + "epoch": 3.0, + "step": 213, + "total_flos": 5.2362750737986355e+17, + "train_loss": 0.4823716257659482, + "train_runtime": 11702.1548, + "train_samples_per_second": 4.615, + "train_steps_per_second": 0.018 + } + ], + "logging_steps": 1, + "max_steps": 213, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 5.2362750737986355e+17, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/qwen2.5-3b-instruct/training_args.bin b/qwen2.5-3b-instruct/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..078e4bb631cb6e7996cf1a54cb10c1a5978aa53a --- /dev/null +++ b/qwen2.5-3b-instruct/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c156fe01067afc4b06ba073aac841107d9e7b73798856df6f0417af272038b3d +size 5649 diff --git a/qwen2.5-3b-instruct/training_eval_loss.png b/qwen2.5-3b-instruct/training_eval_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..9899b56374d9752f775f7349d5c2131361680707 Binary files /dev/null and b/qwen2.5-3b-instruct/training_eval_loss.png differ diff --git a/qwen2.5-3b-instruct/training_loss.png b/qwen2.5-3b-instruct/training_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..d789c59d3d40969f213a8e3e364973235cb32880 Binary files /dev/null and b/qwen2.5-3b-instruct/training_loss.png differ diff --git a/qwen2.5-7b-instruct/README.md b/qwen2.5-7b-instruct/README.md new file mode 100644 index 0000000000000000000000000000000000000000..028d3e1bd9f54daa414624ee98ede99ed935c904 --- /dev/null +++ b/qwen2.5-7b-instruct/README.md @@ -0,0 +1,69 @@ +--- +library_name: transformers +license: other +base_model: Qwen/Qwen2.5-7B-Instruct +tags: +- llama-factory +- full +- generated_from_trainer +model-index: +- name: qwen2.5-7b-instruct + results: [] +--- + + + +# qwen2.5-7b-instruct + +This model is a fine-tuned version of [Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct) on the codealpaca_train_20k_trace_generation_valid dataset. +It achieves the following results on the evaluation set: +- Loss: 0.3999 + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 1e-05 +- train_batch_size: 1 +- eval_batch_size: 1 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 64 +- total_train_batch_size: 256 +- total_eval_batch_size: 4 +- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_steps: 0.1 +- num_epochs: 3 + +### Training results + +| Training Loss | Epoch | Step | Validation Loss | +|:-------------:|:------:|:----:|:---------------:| +| 0.4036 | 0.7111 | 50 | 0.4182 | +| 0.3423 | 1.4124 | 100 | 0.3991 | +| 0.2845 | 2.1138 | 150 | 0.4042 | +| 0.2866 | 2.8249 | 200 | 0.3999 | + + +### Framework versions + +- Transformers 5.2.0 +- Pytorch 2.11.0+cu128 +- Datasets 4.0.0 +- Tokenizers 0.22.2 diff --git a/qwen2.5-7b-instruct/all_results.json b/qwen2.5-7b-instruct/all_results.json new file mode 100644 index 0000000000000000000000000000000000000000..e786f8093fae95a1ecbba6783be20e6eb94e77d0 --- /dev/null +++ b/qwen2.5-7b-instruct/all_results.json @@ -0,0 +1,12 @@ +{ + "epoch": 3.0, + "eval_loss": 0.3998967707157135, + "eval_runtime": 27.0141, + "eval_samples_per_second": 7.404, + "eval_steps_per_second": 1.851, + "total_flos": 62893478699008.0, + "train_loss": 0.36721718521185326, + "train_runtime": 28478.4394, + "train_samples_per_second": 1.896, + "train_steps_per_second": 0.007 +} \ No newline at end of file diff --git a/qwen2.5-7b-instruct/chat_template.jinja b/qwen2.5-7b-instruct/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e --- /dev/null +++ b/qwen2.5-7b-instruct/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/qwen2.5-7b-instruct/checkpoint-142/chat_template.jinja b/qwen2.5-7b-instruct/checkpoint-142/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/qwen2.5-7b-instruct/checkpoint-142/config.json b/qwen2.5-7b-instruct/checkpoint-142/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2948dc473025f5f3adb7c03ee0a5609588eb5c79 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 18944, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 28, + "model_type": "qwen2", + "num_attention_heads": 28, + "num_hidden_layers": 28, + "num_key_value_heads": 4, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.2.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 152064 +} diff --git a/qwen2.5-7b-instruct/checkpoint-142/generation_config.json b/qwen2.5-7b-instruct/checkpoint-142/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..47205fbf3386a215e7c2b9d62df6576f49b77e7b --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.05, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.2.0" +} diff --git a/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_0_mp_rank_00_model_states.pt b/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_0_mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..183d1cd5a61a819397c3e705e240419995efdba2 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_0_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29c0dcb8119be5f088ab9f20aa21275d82b5a08c92977bca93547f536cbc106e +size 167072 diff --git a/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a8a19cbf01327a7fd6613c0d73e3cbbc41869a6b --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c590eb6ab031dcb818ebe1dcb88ef0a5fce04a9e01ceb15440c69a77bd14327 +size 22846856441 diff --git a/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_1_mp_rank_00_model_states.pt b/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_1_mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..43e14639df7ead683e60576f363a0098e28c9b7a --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_1_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ccc863af88153df3714c3bb6543d3a4b59eb6a1490690e412e4b83340f23eff +size 167072 diff --git a/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2544661268762cd0114c0d85f6ebc1601df47129 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:275af28a1833a2edfc366c5cafee35ef7c8f79cd73493041809a0ab53cd01fbb +size 22846856441 diff --git a/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_2_mp_rank_00_model_states.pt b/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_2_mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6479be5b9ab661d304796ad932609f83ee655135 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_2_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5722d57f849317fc943e3aeca8f2b2187f3a0de39a3edc27345871f27a1b824 +size 167072 diff --git a/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_2_mp_rank_00_optim_states.pt b/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5823992b27aad822a146f87575da75f4577972e --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f08586ef59d469555f584b77da494846c4fd8d6ccae891acd79fa8487948026 +size 22846856441 diff --git a/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_3_mp_rank_00_model_states.pt b/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_3_mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..768e1125525356afa56adb030fbae92763065214 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_3_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:298d062dc9ccef0b6dfb8f4ff524ab66ff66b59b14b6830b8a25e7deb428a9cf +size 167072 diff --git a/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_3_mp_rank_00_optim_states.pt b/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e250077a6a4cc47e45f01cb5c201449415e8a539 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/global_step142/zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5b5eebedb719475975de4c236ceba51350f561f9dd0ea0661ad3af6054a4c1e +size 22846856441 diff --git a/qwen2.5-7b-instruct/checkpoint-142/latest b/qwen2.5-7b-instruct/checkpoint-142/latest new file mode 100644 index 0000000000000000000000000000000000000000..2473290e5450bdba7c995cca9b045916ff371abc --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/latest @@ -0,0 +1 @@ +global_step142 \ No newline at end of file diff --git a/qwen2.5-7b-instruct/checkpoint-142/model.safetensors b/qwen2.5-7b-instruct/checkpoint-142/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..41fa03f6000762797fddd43eee0126bface1ddf2 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b95c689bf9d804b8bca6322fa33ae7a33bd6a0352f2431fb395461386e7269a +size 30462505080 diff --git a/qwen2.5-7b-instruct/checkpoint-142/rng_state_0.pth b/qwen2.5-7b-instruct/checkpoint-142/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..b2e0e7337f62c934e90c141589ca185ff78fae36 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fe6b873f1da64bb980afe982fc1c6b6bc461ad8c33073d856804323f64b26d9 +size 15429 diff --git a/qwen2.5-7b-instruct/checkpoint-142/rng_state_1.pth b/qwen2.5-7b-instruct/checkpoint-142/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..204fc909b58b8ae0fab9c8b0f41ea5cc8f3277f8 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13db4dd10b22b5464f5ef0b539152b46390eae6a827b355108cb54b1af2eb830 +size 15429 diff --git a/qwen2.5-7b-instruct/checkpoint-142/rng_state_2.pth b/qwen2.5-7b-instruct/checkpoint-142/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..0c3095386194e2f629b38c7957abe0b9ab74aa8b --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06e293f13fb6e873e8081636b6b07115410b397c55f1ed08b8772b059c8eb74e +size 15429 diff --git a/qwen2.5-7b-instruct/checkpoint-142/rng_state_3.pth b/qwen2.5-7b-instruct/checkpoint-142/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..fdf7f18d5d86f0d363c9ca44b9ad169d469e47f7 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c595c077664f94b7f91088561951be20e39483e01865774a83bc46f3971a3152 +size 15429 diff --git a/qwen2.5-7b-instruct/checkpoint-142/scheduler.pt b/qwen2.5-7b-instruct/checkpoint-142/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..2967e7243507a115f9424b1b386bf3657f047c75 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:922deb71175797a96c4bddc5e1cef1c097df38c0dfa52334e97c89e0f573f2bb +size 1465 diff --git a/qwen2.5-7b-instruct/checkpoint-142/tokenizer.json b/qwen2.5-7b-instruct/checkpoint-142/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..34510ff0037cd50428af467a17ead5a96140a32c --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/qwen2.5-7b-instruct/checkpoint-142/tokenizer_config.json b/qwen2.5-7b-instruct/checkpoint-142/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..145e2c7a68b109ae0253dda29fc2e66cb01b4b85 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen2.5-7b-instruct/checkpoint-142/trainer_state.json b/qwen2.5-7b-instruct/checkpoint-142/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f17edf369cf854f46a2447533fd3d1a90febe00e --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/trainer_state.json @@ -0,0 +1,1044 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 50, + "global_step": 142, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.014222222222222223, + "grad_norm": 10.21201439945407, + "learning_rate": 0.0, + "loss": 0.9819135069847107, + "step": 1 + }, + { + "epoch": 0.028444444444444446, + "grad_norm": 10.607308805754391, + "learning_rate": 4.5454545454545457e-07, + "loss": 0.992728054523468, + "step": 2 + }, + { + "epoch": 0.042666666666666665, + "grad_norm": 8.137000657744727, + "learning_rate": 9.090909090909091e-07, + "loss": 0.88365638256073, + "step": 3 + }, + { + "epoch": 0.05688888888888889, + "grad_norm": 5.015580274302158, + "learning_rate": 1.3636363636363636e-06, + "loss": 0.8155229091644287, + "step": 4 + }, + { + "epoch": 0.07111111111111111, + "grad_norm": 3.513429105458463, + "learning_rate": 1.8181818181818183e-06, + "loss": 0.7148971557617188, + "step": 5 + }, + { + "epoch": 0.08533333333333333, + "grad_norm": 2.890137665957255, + "learning_rate": 2.2727272727272728e-06, + "loss": 0.7388776540756226, + "step": 6 + }, + { + "epoch": 0.09955555555555555, + "grad_norm": 2.6650307199214267, + "learning_rate": 2.7272727272727272e-06, + "loss": 0.6954053044319153, + "step": 7 + }, + { + "epoch": 0.11377777777777778, + "grad_norm": 2.1027438786791453, + "learning_rate": 3.181818181818182e-06, + "loss": 0.6385300159454346, + "step": 8 + }, + { + "epoch": 0.128, + "grad_norm": 1.9707186974912183, + "learning_rate": 3.6363636363636366e-06, + "loss": 0.5974307060241699, + "step": 9 + }, + { + "epoch": 0.14222222222222222, + "grad_norm": 1.7892988232740288, + "learning_rate": 4.0909090909090915e-06, + "loss": 0.5823870897293091, + "step": 10 + }, + { + "epoch": 0.15644444444444444, + "grad_norm": 1.8117663888596973, + "learning_rate": 4.5454545454545455e-06, + "loss": 0.5595569610595703, + "step": 11 + }, + { + "epoch": 0.17066666666666666, + "grad_norm": 1.6892201331978909, + "learning_rate": 5e-06, + "loss": 0.5447146892547607, + "step": 12 + }, + { + "epoch": 0.18488888888888888, + "grad_norm": 1.5301037695464916, + "learning_rate": 5.4545454545454545e-06, + "loss": 0.5179443359375, + "step": 13 + }, + { + "epoch": 0.1991111111111111, + "grad_norm": 1.2533624125991738, + "learning_rate": 5.90909090909091e-06, + "loss": 0.5241127014160156, + "step": 14 + }, + { + "epoch": 0.21333333333333335, + "grad_norm": 1.2394418596122698, + "learning_rate": 6.363636363636364e-06, + "loss": 0.4842333197593689, + "step": 15 + }, + { + "epoch": 0.22755555555555557, + "grad_norm": 1.1420735285454937, + "learning_rate": 6.818181818181818e-06, + "loss": 0.5132038593292236, + "step": 16 + }, + { + "epoch": 0.24177777777777779, + "grad_norm": 1.0423494529114559, + "learning_rate": 7.272727272727273e-06, + "loss": 0.4917834401130676, + "step": 17 + }, + { + "epoch": 0.256, + "grad_norm": 0.9878207423648759, + "learning_rate": 7.727272727272727e-06, + "loss": 0.4813205897808075, + "step": 18 + }, + { + "epoch": 0.2702222222222222, + "grad_norm": 0.9711754260021513, + "learning_rate": 8.181818181818183e-06, + "loss": 0.4883510172367096, + "step": 19 + }, + { + "epoch": 0.28444444444444444, + "grad_norm": 0.9163050690645712, + "learning_rate": 8.636363636363637e-06, + "loss": 0.4871647357940674, + "step": 20 + }, + { + "epoch": 0.2986666666666667, + "grad_norm": 0.8404612592234425, + "learning_rate": 9.090909090909091e-06, + "loss": 0.46865272521972656, + "step": 21 + }, + { + "epoch": 0.3128888888888889, + "grad_norm": 0.8673877914303936, + "learning_rate": 9.545454545454547e-06, + "loss": 0.4601229727268219, + "step": 22 + }, + { + "epoch": 0.32711111111111113, + "grad_norm": 0.8211864748561771, + "learning_rate": 1e-05, + "loss": 0.45921051502227783, + "step": 23 + }, + { + "epoch": 0.3413333333333333, + "grad_norm": 0.8242615873645178, + "learning_rate": 9.999323662872998e-06, + "loss": 0.46318507194519043, + "step": 24 + }, + { + "epoch": 0.35555555555555557, + "grad_norm": 0.7839407977142685, + "learning_rate": 9.99729483446475e-06, + "loss": 0.4633931815624237, + "step": 25 + }, + { + "epoch": 0.36977777777777776, + "grad_norm": 0.7111762322751543, + "learning_rate": 9.993914063644053e-06, + "loss": 0.45212602615356445, + "step": 26 + }, + { + "epoch": 0.384, + "grad_norm": 0.718221639194032, + "learning_rate": 9.989182265027232e-06, + "loss": 0.44105663895606995, + "step": 27 + }, + { + "epoch": 0.3982222222222222, + "grad_norm": 0.7040716895578701, + "learning_rate": 9.98310071873072e-06, + "loss": 0.4427238404750824, + "step": 28 + }, + { + "epoch": 0.41244444444444445, + "grad_norm": 0.7489850512171834, + "learning_rate": 9.975671070024741e-06, + "loss": 0.443437397480011, + "step": 29 + }, + { + "epoch": 0.4266666666666667, + "grad_norm": 0.7002493913122244, + "learning_rate": 9.966895328888195e-06, + "loss": 0.4632475674152374, + "step": 30 + }, + { + "epoch": 0.4408888888888889, + "grad_norm": 0.6824040285805486, + "learning_rate": 9.956775869464901e-06, + "loss": 0.4354594349861145, + "step": 31 + }, + { + "epoch": 0.45511111111111113, + "grad_norm": 0.6597936132766806, + "learning_rate": 9.945315429421307e-06, + "loss": 0.429914653301239, + "step": 32 + }, + { + "epoch": 0.4693333333333333, + "grad_norm": 0.6352588389251546, + "learning_rate": 9.932517109205849e-06, + "loss": 0.43607252836227417, + "step": 33 + }, + { + "epoch": 0.48355555555555557, + "grad_norm": 0.6921974704448327, + "learning_rate": 9.918384371210178e-06, + "loss": 0.45671695470809937, + "step": 34 + }, + { + "epoch": 0.49777777777777776, + "grad_norm": 0.6491660902599252, + "learning_rate": 9.902921038832456e-06, + "loss": 0.4388338327407837, + "step": 35 + }, + { + "epoch": 0.512, + "grad_norm": 0.6096750069184391, + "learning_rate": 9.886131295443003e-06, + "loss": 0.4141705334186554, + "step": 36 + }, + { + "epoch": 0.5262222222222223, + "grad_norm": 0.6503340768273712, + "learning_rate": 9.868019683252543e-06, + "loss": 0.4218006432056427, + "step": 37 + }, + { + "epoch": 0.5404444444444444, + "grad_norm": 0.714483452396674, + "learning_rate": 9.848591102083375e-06, + "loss": 0.4457612931728363, + "step": 38 + }, + { + "epoch": 0.5546666666666666, + "grad_norm": 0.6231964594158875, + "learning_rate": 9.82785080804381e-06, + "loss": 0.4331759214401245, + "step": 39 + }, + { + "epoch": 0.5688888888888889, + "grad_norm": 0.6572593116495337, + "learning_rate": 9.805804412106197e-06, + "loss": 0.4507564902305603, + "step": 40 + }, + { + "epoch": 0.5831111111111111, + "grad_norm": 0.673487925488406, + "learning_rate": 9.782457878588977e-06, + "loss": 0.4254251718521118, + "step": 41 + }, + { + "epoch": 0.5973333333333334, + "grad_norm": 0.5846423543296639, + "learning_rate": 9.75781752354311e-06, + "loss": 0.4125726819038391, + "step": 42 + }, + { + "epoch": 0.6115555555555555, + "grad_norm": 0.598724301403679, + "learning_rate": 9.731890013043367e-06, + "loss": 0.4325122535228729, + "step": 43 + }, + { + "epoch": 0.6257777777777778, + "grad_norm": 0.6218044129681044, + "learning_rate": 9.704682361384941e-06, + "loss": 0.4131222367286682, + "step": 44 + }, + { + "epoch": 0.64, + "grad_norm": 0.617001419027673, + "learning_rate": 9.676201929185809e-06, + "loss": 0.42036178708076477, + "step": 45 + }, + { + "epoch": 0.6542222222222223, + "grad_norm": 0.5809155918543004, + "learning_rate": 9.646456421395447e-06, + "loss": 0.4182787537574768, + "step": 46 + }, + { + "epoch": 0.6684444444444444, + "grad_norm": 0.5850555211488232, + "learning_rate": 9.615453885210368e-06, + "loss": 0.4234634041786194, + "step": 47 + }, + { + "epoch": 0.6826666666666666, + "grad_norm": 0.5511675603941021, + "learning_rate": 9.583202707897075e-06, + "loss": 0.40563416481018066, + "step": 48 + }, + { + "epoch": 0.6968888888888889, + "grad_norm": 0.6111190985848377, + "learning_rate": 9.549711614523007e-06, + "loss": 0.3980520963668823, + "step": 49 + }, + { + "epoch": 0.7111111111111111, + "grad_norm": 0.5736293779202373, + "learning_rate": 9.514989665596114e-06, + "loss": 0.4036375880241394, + "step": 50 + }, + { + "epoch": 0.7111111111111111, + "eval_loss": 0.41823333501815796, + "eval_runtime": 28.2382, + "eval_samples_per_second": 7.083, + "eval_steps_per_second": 1.771, + "step": 50 + }, + { + "epoch": 0.7253333333333334, + "grad_norm": 0.5785272249494463, + "learning_rate": 9.479046254613673e-06, + "loss": 0.41144663095474243, + "step": 51 + }, + { + "epoch": 0.7395555555555555, + "grad_norm": 0.5517717884994647, + "learning_rate": 9.441891105521005e-06, + "loss": 0.38760095834732056, + "step": 52 + }, + { + "epoch": 0.7537777777777778, + "grad_norm": 0.5648730368507402, + "learning_rate": 9.40353427008083e-06, + "loss": 0.4036053717136383, + "step": 53 + }, + { + "epoch": 0.768, + "grad_norm": 0.6019960018903892, + "learning_rate": 9.3639861251539e-06, + "loss": 0.39992809295654297, + "step": 54 + }, + { + "epoch": 0.7822222222222223, + "grad_norm": 0.5943571857589287, + "learning_rate": 9.323257369891702e-06, + "loss": 0.420696496963501, + "step": 55 + }, + { + "epoch": 0.7964444444444444, + "grad_norm": 0.5800482521740772, + "learning_rate": 9.281359022841966e-06, + "loss": 0.41333717107772827, + "step": 56 + }, + { + "epoch": 0.8106666666666666, + "grad_norm": 0.5896438771006596, + "learning_rate": 9.238302418967757e-06, + "loss": 0.41053399443626404, + "step": 57 + }, + { + "epoch": 0.8248888888888889, + "grad_norm": 0.5828843912613246, + "learning_rate": 9.194099206580981e-06, + "loss": 0.4167237877845764, + "step": 58 + }, + { + "epoch": 0.8391111111111111, + "grad_norm": 0.636505663941606, + "learning_rate": 9.14876134419111e-06, + "loss": 0.4126594066619873, + "step": 59 + }, + { + "epoch": 0.8533333333333334, + "grad_norm": 0.5954156377152778, + "learning_rate": 9.102301097269974e-06, + "loss": 0.40986335277557373, + "step": 60 + }, + { + "epoch": 0.8675555555555555, + "grad_norm": 0.5663259824764864, + "learning_rate": 9.05473103493355e-06, + "loss": 0.39707833528518677, + "step": 61 + }, + { + "epoch": 0.8817777777777778, + "grad_norm": 0.6233708113516392, + "learning_rate": 9.006064026541549e-06, + "loss": 0.4146142601966858, + "step": 62 + }, + { + "epoch": 0.896, + "grad_norm": 0.583882066947991, + "learning_rate": 8.956313238215824e-06, + "loss": 0.3761865794658661, + "step": 63 + }, + { + "epoch": 0.9102222222222223, + "grad_norm": 0.559228243852746, + "learning_rate": 8.905492129278478e-06, + "loss": 0.397402286529541, + "step": 64 + }, + { + "epoch": 0.9244444444444444, + "grad_norm": 0.5630501047235328, + "learning_rate": 8.85361444861063e-06, + "loss": 0.40299445390701294, + "step": 65 + }, + { + "epoch": 0.9386666666666666, + "grad_norm": 0.5778346041245924, + "learning_rate": 8.800694230932885e-06, + "loss": 0.4066828489303589, + "step": 66 + }, + { + "epoch": 0.9528888888888889, + "grad_norm": 0.5735220314699014, + "learning_rate": 8.74674579300843e-06, + "loss": 0.39874544739723206, + "step": 67 + }, + { + "epoch": 0.9671111111111111, + "grad_norm": 0.5774583972046927, + "learning_rate": 8.691783729769874e-06, + "loss": 0.38473668694496155, + "step": 68 + }, + { + "epoch": 0.9813333333333333, + "grad_norm": 0.6169263233631689, + "learning_rate": 8.635822910370793e-06, + "loss": 0.4083672761917114, + "step": 69 + }, + { + "epoch": 0.9955555555555555, + "grad_norm": 0.5815111675806847, + "learning_rate": 8.578878474163115e-06, + "loss": 0.3794967234134674, + "step": 70 + }, + { + "epoch": 1.0, + "grad_norm": 0.9420572347818393, + "learning_rate": 8.520965826601394e-06, + "loss": 0.40614083409309387, + "step": 71 + }, + { + "epoch": 1.0142222222222221, + "grad_norm": 0.6281039360207515, + "learning_rate": 8.462100635075097e-06, + "loss": 0.36958712339401245, + "step": 72 + }, + { + "epoch": 1.0284444444444445, + "grad_norm": 0.6233244816927183, + "learning_rate": 8.40229882467003e-06, + "loss": 0.3494463562965393, + "step": 73 + }, + { + "epoch": 1.0426666666666666, + "grad_norm": 0.5621513032586688, + "learning_rate": 8.341576573860049e-06, + "loss": 0.3302980065345764, + "step": 74 + }, + { + "epoch": 1.056888888888889, + "grad_norm": 0.5754725580485192, + "learning_rate": 8.279950310130218e-06, + "loss": 0.35172200202941895, + "step": 75 + }, + { + "epoch": 1.0711111111111111, + "grad_norm": 0.5740681468075599, + "learning_rate": 8.2174367055326e-06, + "loss": 0.3394201993942261, + "step": 76 + }, + { + "epoch": 1.0853333333333333, + "grad_norm": 0.590460680669288, + "learning_rate": 8.154052672175888e-06, + "loss": 0.33312541246414185, + "step": 77 + }, + { + "epoch": 1.0995555555555556, + "grad_norm": 0.5842296281151235, + "learning_rate": 8.08981535765009e-06, + "loss": 0.35658037662506104, + "step": 78 + }, + { + "epoch": 1.1137777777777778, + "grad_norm": 0.6088529350096018, + "learning_rate": 8.024742140387506e-06, + "loss": 0.3449564278125763, + "step": 79 + }, + { + "epoch": 1.1280000000000001, + "grad_norm": 0.5744146541082055, + "learning_rate": 7.95885062496126e-06, + "loss": 0.3357444107532501, + "step": 80 + }, + { + "epoch": 1.1422222222222222, + "grad_norm": 0.5909574288907287, + "learning_rate": 7.892158637322647e-06, + "loss": 0.3170369267463684, + "step": 81 + }, + { + "epoch": 1.1564444444444444, + "grad_norm": 0.6136816332174048, + "learning_rate": 7.824684219978591e-06, + "loss": 0.3318628668785095, + "step": 82 + }, + { + "epoch": 1.1706666666666667, + "grad_norm": 0.5568422531541258, + "learning_rate": 7.756445627110523e-06, + "loss": 0.34082144498825073, + "step": 83 + }, + { + "epoch": 1.1848888888888889, + "grad_norm": 0.6082214692469516, + "learning_rate": 7.68746131963598e-06, + "loss": 0.3453400135040283, + "step": 84 + }, + { + "epoch": 1.199111111111111, + "grad_norm": 0.5993208838733233, + "learning_rate": 7.6177499602143e-06, + "loss": 0.3420029580593109, + "step": 85 + }, + { + "epoch": 1.2133333333333334, + "grad_norm": 0.5493880460077399, + "learning_rate": 7.547330408197695e-06, + "loss": 0.3330039381980896, + "step": 86 + }, + { + "epoch": 1.2275555555555555, + "grad_norm": 0.6151717226672565, + "learning_rate": 7.476221714529167e-06, + "loss": 0.3428732454776764, + "step": 87 + }, + { + "epoch": 1.2417777777777779, + "grad_norm": 0.6359199055571157, + "learning_rate": 7.404443116588548e-06, + "loss": 0.3575543761253357, + "step": 88 + }, + { + "epoch": 1.256, + "grad_norm": 0.5764223898323285, + "learning_rate": 7.332014032988123e-06, + "loss": 0.3414611518383026, + "step": 89 + }, + { + "epoch": 1.2702222222222221, + "grad_norm": 0.5701344890989715, + "learning_rate": 7.2589540583192165e-06, + "loss": 0.3387766480445862, + "step": 90 + }, + { + "epoch": 1.2844444444444445, + "grad_norm": 0.5603334736031269, + "learning_rate": 7.185282957851175e-06, + "loss": 0.32765448093414307, + "step": 91 + }, + { + "epoch": 1.2986666666666666, + "grad_norm": 0.5637583167829048, + "learning_rate": 7.111020662184174e-06, + "loss": 0.3432917296886444, + "step": 92 + }, + { + "epoch": 1.3128888888888888, + "grad_norm": 0.5811322458912195, + "learning_rate": 7.036187261857289e-06, + "loss": 0.3334134817123413, + "step": 93 + }, + { + "epoch": 1.3271111111111111, + "grad_norm": 0.6069998855060962, + "learning_rate": 6.960803001913315e-06, + "loss": 0.3528696894645691, + "step": 94 + }, + { + "epoch": 1.3413333333333333, + "grad_norm": 0.5846537103757758, + "learning_rate": 6.884888276421766e-06, + "loss": 0.3428453207015991, + "step": 95 + }, + { + "epoch": 1.3555555555555556, + "grad_norm": 0.5679194938574699, + "learning_rate": 6.8084636229615786e-06, + "loss": 0.33261382579803467, + "step": 96 + }, + { + "epoch": 1.3697777777777778, + "grad_norm": 0.5611361976965846, + "learning_rate": 6.731549717064975e-06, + "loss": 0.33263495564460754, + "step": 97 + }, + { + "epoch": 1.384, + "grad_norm": 0.5958752887464758, + "learning_rate": 6.654167366624009e-06, + "loss": 0.3409455418586731, + "step": 98 + }, + { + "epoch": 1.3982222222222223, + "grad_norm": 0.5821747639433966, + "learning_rate": 6.576337506261314e-06, + "loss": 0.34175097942352295, + "step": 99 + }, + { + "epoch": 1.4124444444444444, + "grad_norm": 0.5474416142301471, + "learning_rate": 6.498081191666549e-06, + "loss": 0.3422796130180359, + "step": 100 + }, + { + "epoch": 1.4124444444444444, + "eval_loss": 0.39906859397888184, + "eval_runtime": 28.0052, + "eval_samples_per_second": 7.142, + "eval_steps_per_second": 1.785, + "step": 100 + }, + { + "epoch": 1.4266666666666667, + "grad_norm": 0.5470999547878086, + "learning_rate": 6.419419593900109e-06, + "loss": 0.3311368227005005, + "step": 101 + }, + { + "epoch": 1.4408888888888889, + "grad_norm": 0.5417978810198306, + "learning_rate": 6.340373993665607e-06, + "loss": 0.32347068190574646, + "step": 102 + }, + { + "epoch": 1.455111111111111, + "grad_norm": 0.5408071080127892, + "learning_rate": 6.2609657755527135e-06, + "loss": 0.32194703817367554, + "step": 103 + }, + { + "epoch": 1.4693333333333334, + "grad_norm": 0.5610127885481652, + "learning_rate": 6.1812164222518626e-06, + "loss": 0.3477523922920227, + "step": 104 + }, + { + "epoch": 1.4835555555555555, + "grad_norm": 0.6035742435126064, + "learning_rate": 6.101147508742456e-06, + "loss": 0.33583635091781616, + "step": 105 + }, + { + "epoch": 1.4977777777777779, + "grad_norm": 0.5482179611450086, + "learning_rate": 6.020780696456059e-06, + "loss": 0.3305099904537201, + "step": 106 + }, + { + "epoch": 1.512, + "grad_norm": 0.5549975206757557, + "learning_rate": 5.940137727416247e-06, + "loss": 0.3392476439476013, + "step": 107 + }, + { + "epoch": 1.5262222222222221, + "grad_norm": 0.5452500081271687, + "learning_rate": 5.859240418356614e-06, + "loss": 0.3281456232070923, + "step": 108 + }, + { + "epoch": 1.5404444444444443, + "grad_norm": 0.569343792950831, + "learning_rate": 5.778110654818602e-06, + "loss": 0.3557584285736084, + "step": 109 + }, + { + "epoch": 1.5546666666666666, + "grad_norm": 0.5609073891735213, + "learning_rate": 5.696770385230679e-06, + "loss": 0.3375227153301239, + "step": 110 + }, + { + "epoch": 1.568888888888889, + "grad_norm": 0.5979889868516021, + "learning_rate": 5.615241614970546e-06, + "loss": 0.33440524339675903, + "step": 111 + }, + { + "epoch": 1.5831111111111111, + "grad_norm": 0.5752429374657604, + "learning_rate": 5.533546400411899e-06, + "loss": 0.3354341983795166, + "step": 112 + }, + { + "epoch": 1.5973333333333333, + "grad_norm": 0.5583834084640693, + "learning_rate": 5.4517068429574215e-06, + "loss": 0.34512901306152344, + "step": 113 + }, + { + "epoch": 1.6115555555555554, + "grad_norm": 0.5366739842708482, + "learning_rate": 5.3697450830595775e-06, + "loss": 0.3282182812690735, + "step": 114 + }, + { + "epoch": 1.6257777777777778, + "grad_norm": 0.5923091160755782, + "learning_rate": 5.287683294230855e-06, + "loss": 0.3325689136981964, + "step": 115 + }, + { + "epoch": 1.6400000000000001, + "grad_norm": 0.5603788239590529, + "learning_rate": 5.20554367704505e-06, + "loss": 0.3167092204093933, + "step": 116 + }, + { + "epoch": 1.6542222222222223, + "grad_norm": 0.5675689042120237, + "learning_rate": 5.123348453131242e-06, + "loss": 0.3419051766395569, + "step": 117 + }, + { + "epoch": 1.6684444444444444, + "grad_norm": 0.5456542923932385, + "learning_rate": 5.041119859162068e-06, + "loss": 0.333318293094635, + "step": 118 + }, + { + "epoch": 1.6826666666666665, + "grad_norm": 0.570188952777049, + "learning_rate": 4.958880140837934e-06, + "loss": 0.33905792236328125, + "step": 119 + }, + { + "epoch": 1.696888888888889, + "grad_norm": 0.5534588670481234, + "learning_rate": 4.876651546868759e-06, + "loss": 0.32569897174835205, + "step": 120 + }, + { + "epoch": 1.7111111111111112, + "grad_norm": 0.5546097096941289, + "learning_rate": 4.794456322954953e-06, + "loss": 0.3532940745353699, + "step": 121 + }, + { + "epoch": 1.7253333333333334, + "grad_norm": 0.5857002004886875, + "learning_rate": 4.7123167057691446e-06, + "loss": 0.35066288709640503, + "step": 122 + }, + { + "epoch": 1.7395555555555555, + "grad_norm": 0.5351298499201909, + "learning_rate": 4.630254916940424e-06, + "loss": 0.35048598051071167, + "step": 123 + }, + { + "epoch": 1.7537777777777777, + "grad_norm": 0.5572795301130913, + "learning_rate": 4.548293157042581e-06, + "loss": 0.3389701843261719, + "step": 124 + }, + { + "epoch": 1.768, + "grad_norm": 0.5503673971003669, + "learning_rate": 4.466453599588103e-06, + "loss": 0.3477008044719696, + "step": 125 + }, + { + "epoch": 1.7822222222222224, + "grad_norm": 0.5485965256926039, + "learning_rate": 4.384758385029457e-06, + "loss": 0.3188989758491516, + "step": 126 + }, + { + "epoch": 1.7964444444444445, + "grad_norm": 0.5566326955710196, + "learning_rate": 4.3032296147693225e-06, + "loss": 0.3296913504600525, + "step": 127 + }, + { + "epoch": 1.8106666666666666, + "grad_norm": 0.5592929839051507, + "learning_rate": 4.2218893451814005e-06, + "loss": 0.3278958797454834, + "step": 128 + }, + { + "epoch": 1.8248888888888888, + "grad_norm": 0.5372208136274655, + "learning_rate": 4.140759581643386e-06, + "loss": 0.3338126838207245, + "step": 129 + }, + { + "epoch": 1.8391111111111111, + "grad_norm": 0.5506377641318588, + "learning_rate": 4.059862272583755e-06, + "loss": 0.33511388301849365, + "step": 130 + }, + { + "epoch": 1.8533333333333335, + "grad_norm": 0.5509814672788427, + "learning_rate": 3.979219303543942e-06, + "loss": 0.3190721869468689, + "step": 131 + }, + { + "epoch": 1.8675555555555556, + "grad_norm": 0.5437024683532695, + "learning_rate": 3.898852491257547e-06, + "loss": 0.3397029638290405, + "step": 132 + }, + { + "epoch": 1.8817777777777778, + "grad_norm": 0.5577533834862016, + "learning_rate": 3.818783577748138e-06, + "loss": 0.3368285894393921, + "step": 133 + }, + { + "epoch": 1.896, + "grad_norm": 0.5654570173541145, + "learning_rate": 3.7390342244472886e-06, + "loss": 0.34375038743019104, + "step": 134 + }, + { + "epoch": 1.9102222222222223, + "grad_norm": 0.5435541441710147, + "learning_rate": 3.659626006334395e-06, + "loss": 0.32158786058425903, + "step": 135 + }, + { + "epoch": 1.9244444444444444, + "grad_norm": 0.5517597326060792, + "learning_rate": 3.5805804060998926e-06, + "loss": 0.32734596729278564, + "step": 136 + }, + { + "epoch": 1.9386666666666668, + "grad_norm": 0.5529004589065376, + "learning_rate": 3.501918808333453e-06, + "loss": 0.3495728671550751, + "step": 137 + }, + { + "epoch": 1.952888888888889, + "grad_norm": 0.5497949696155701, + "learning_rate": 3.4236624937386874e-06, + "loss": 0.3545406758785248, + "step": 138 + }, + { + "epoch": 1.967111111111111, + "grad_norm": 0.50810534397043, + "learning_rate": 3.3458326333759927e-06, + "loss": 0.3283476233482361, + "step": 139 + }, + { + "epoch": 1.9813333333333332, + "grad_norm": 0.5410931225650443, + "learning_rate": 3.268450282935026e-06, + "loss": 0.3362443447113037, + "step": 140 + }, + { + "epoch": 1.9955555555555555, + "grad_norm": 0.5526133533239165, + "learning_rate": 3.1915363770384223e-06, + "loss": 0.33009064197540283, + "step": 141 + }, + { + "epoch": 2.0, + "grad_norm": 0.8788381822845209, + "learning_rate": 3.1151117235782346e-06, + "loss": 0.34658753871917725, + "step": 142 + } + ], + "logging_steps": 1, + "max_steps": 213, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 41927044538368.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/qwen2.5-7b-instruct/checkpoint-142/training_args.bin b/qwen2.5-7b-instruct/checkpoint-142/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..e59d7187e91e3dd0138c38f45c246765301e22e6 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:238e1f7259942c35813600da4056c3d822147cad18b0d1147780f439481f9c35 +size 7377 diff --git a/qwen2.5-7b-instruct/checkpoint-142/zero_to_fp32.py b/qwen2.5-7b-instruct/checkpoint-142/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-142/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/qwen2.5-7b-instruct/checkpoint-213/chat_template.jinja b/qwen2.5-7b-instruct/checkpoint-213/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/qwen2.5-7b-instruct/checkpoint-213/config.json b/qwen2.5-7b-instruct/checkpoint-213/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2948dc473025f5f3adb7c03ee0a5609588eb5c79 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 18944, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 28, + "model_type": "qwen2", + "num_attention_heads": 28, + "num_hidden_layers": 28, + "num_key_value_heads": 4, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.2.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 152064 +} diff --git a/qwen2.5-7b-instruct/checkpoint-213/generation_config.json b/qwen2.5-7b-instruct/checkpoint-213/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..47205fbf3386a215e7c2b9d62df6576f49b77e7b --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.05, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.2.0" +} diff --git a/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_0_mp_rank_00_model_states.pt b/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_0_mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a2a61964a052855d74fa23389e8244e86e97bea2 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_0_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97254c4e98a3a0ab2e539f19505e2e55fd761f2d8c7666e8527ce879a91e1673 +size 167072 diff --git a/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c07af30f9fc51829f582a2870711c348e2bcab3a --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32bd0b042b0015c092e6a46f3668c7f20fc3361456bae555cf42848b0fd169fd +size 22846856441 diff --git a/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_1_mp_rank_00_model_states.pt b/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_1_mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a0ceb23487ca202c8f08a7fa14cf659f3422ed25 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_1_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9b45bf11a2150d9bdc72c5ef5424c0831e39f5e0fe9e0ea01b1e27357caccc2 +size 167072 diff --git a/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..633a6889cbdce5a6c94658ca1bdbd2d32271c860 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a83c865e253b2879b88483cac20e27f651c4f48fac2e792ab3ad3be4273c414 +size 22846856441 diff --git a/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_2_mp_rank_00_model_states.pt b/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_2_mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a2ab6d412458f1e3b3e580e4964d6a819d14ebaf --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_2_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3760eeb383b472c7dcb493dc0a14c0f2da4a02f9087c2d71d1677d7d62e9e7af +size 167072 diff --git a/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_2_mp_rank_00_optim_states.pt b/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b3206646cf1dbe87766c96a3ae372c86dcded723 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b806699334da9e6472c5e383bee91c91433c63a8ee2e0471c33821135b4a302 +size 22846856441 diff --git a/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_3_mp_rank_00_model_states.pt b/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_3_mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..626f6ceddf885fbef04516c602bd27b38ed7c849 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_3_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6309af8128886c428c3ec8e2952c1cff0087e95cde36264d8b0921a1d871c115 +size 167072 diff --git a/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_3_mp_rank_00_optim_states.pt b/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd8cf41d21c0995f9c0710f728d24359a6ad6b76 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/global_step213/zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b04588ee4fa5a79a3f3118c863eeb464b3b50f05028b1dc4ab6137949d15964 +size 22846856441 diff --git a/qwen2.5-7b-instruct/checkpoint-213/latest b/qwen2.5-7b-instruct/checkpoint-213/latest new file mode 100644 index 0000000000000000000000000000000000000000..1e31e99efe64a3a8ae4660c05e9b97baa2c58524 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/latest @@ -0,0 +1 @@ +global_step213 \ No newline at end of file diff --git a/qwen2.5-7b-instruct/checkpoint-213/model.safetensors b/qwen2.5-7b-instruct/checkpoint-213/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..49af64a80870e764d28758163b444b449cac9ab6 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7978e4a287a8f0a1329531793688c46cd76bc7f8069c951f5e3b8dc9227caf30 +size 30462505080 diff --git a/qwen2.5-7b-instruct/checkpoint-213/rng_state_0.pth b/qwen2.5-7b-instruct/checkpoint-213/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..b2e0e7337f62c934e90c141589ca185ff78fae36 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fe6b873f1da64bb980afe982fc1c6b6bc461ad8c33073d856804323f64b26d9 +size 15429 diff --git a/qwen2.5-7b-instruct/checkpoint-213/rng_state_1.pth b/qwen2.5-7b-instruct/checkpoint-213/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..204fc909b58b8ae0fab9c8b0f41ea5cc8f3277f8 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13db4dd10b22b5464f5ef0b539152b46390eae6a827b355108cb54b1af2eb830 +size 15429 diff --git a/qwen2.5-7b-instruct/checkpoint-213/rng_state_2.pth b/qwen2.5-7b-instruct/checkpoint-213/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..0c3095386194e2f629b38c7957abe0b9ab74aa8b --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06e293f13fb6e873e8081636b6b07115410b397c55f1ed08b8772b059c8eb74e +size 15429 diff --git a/qwen2.5-7b-instruct/checkpoint-213/rng_state_3.pth b/qwen2.5-7b-instruct/checkpoint-213/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..fdf7f18d5d86f0d363c9ca44b9ad169d469e47f7 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c595c077664f94b7f91088561951be20e39483e01865774a83bc46f3971a3152 +size 15429 diff --git a/qwen2.5-7b-instruct/checkpoint-213/scheduler.pt b/qwen2.5-7b-instruct/checkpoint-213/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..39cf87775c3a71d10e7311e0cab769e112f357ba --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c38a67b98c17be81d4c6d00b60029cdedd43e9aba35ca28ec28ef946c400f268 +size 1465 diff --git a/qwen2.5-7b-instruct/checkpoint-213/tokenizer.json b/qwen2.5-7b-instruct/checkpoint-213/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..34510ff0037cd50428af467a17ead5a96140a32c --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/qwen2.5-7b-instruct/checkpoint-213/tokenizer_config.json b/qwen2.5-7b-instruct/checkpoint-213/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..145e2c7a68b109ae0253dda29fc2e66cb01b4b85 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen2.5-7b-instruct/checkpoint-213/trainer_state.json b/qwen2.5-7b-instruct/checkpoint-213/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..577d87270c631942770fdb5efefe161f346c6dec --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/trainer_state.json @@ -0,0 +1,1557 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 50, + "global_step": 213, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.014222222222222223, + "grad_norm": 10.21201439945407, + "learning_rate": 0.0, + "loss": 0.9819135069847107, + "step": 1 + }, + { + "epoch": 0.028444444444444446, + "grad_norm": 10.607308805754391, + "learning_rate": 4.5454545454545457e-07, + "loss": 0.992728054523468, + "step": 2 + }, + { + "epoch": 0.042666666666666665, + "grad_norm": 8.137000657744727, + "learning_rate": 9.090909090909091e-07, + "loss": 0.88365638256073, + "step": 3 + }, + { + "epoch": 0.05688888888888889, + "grad_norm": 5.015580274302158, + "learning_rate": 1.3636363636363636e-06, + "loss": 0.8155229091644287, + "step": 4 + }, + { + "epoch": 0.07111111111111111, + "grad_norm": 3.513429105458463, + "learning_rate": 1.8181818181818183e-06, + "loss": 0.7148971557617188, + "step": 5 + }, + { + "epoch": 0.08533333333333333, + "grad_norm": 2.890137665957255, + "learning_rate": 2.2727272727272728e-06, + "loss": 0.7388776540756226, + "step": 6 + }, + { + "epoch": 0.09955555555555555, + "grad_norm": 2.6650307199214267, + "learning_rate": 2.7272727272727272e-06, + "loss": 0.6954053044319153, + "step": 7 + }, + { + "epoch": 0.11377777777777778, + "grad_norm": 2.1027438786791453, + "learning_rate": 3.181818181818182e-06, + "loss": 0.6385300159454346, + "step": 8 + }, + { + "epoch": 0.128, + "grad_norm": 1.9707186974912183, + "learning_rate": 3.6363636363636366e-06, + "loss": 0.5974307060241699, + "step": 9 + }, + { + "epoch": 0.14222222222222222, + "grad_norm": 1.7892988232740288, + "learning_rate": 4.0909090909090915e-06, + "loss": 0.5823870897293091, + "step": 10 + }, + { + "epoch": 0.15644444444444444, + "grad_norm": 1.8117663888596973, + "learning_rate": 4.5454545454545455e-06, + "loss": 0.5595569610595703, + "step": 11 + }, + { + "epoch": 0.17066666666666666, + "grad_norm": 1.6892201331978909, + "learning_rate": 5e-06, + "loss": 0.5447146892547607, + "step": 12 + }, + { + "epoch": 0.18488888888888888, + "grad_norm": 1.5301037695464916, + "learning_rate": 5.4545454545454545e-06, + "loss": 0.5179443359375, + "step": 13 + }, + { + "epoch": 0.1991111111111111, + "grad_norm": 1.2533624125991738, + "learning_rate": 5.90909090909091e-06, + "loss": 0.5241127014160156, + "step": 14 + }, + { + "epoch": 0.21333333333333335, + "grad_norm": 1.2394418596122698, + "learning_rate": 6.363636363636364e-06, + "loss": 0.4842333197593689, + "step": 15 + }, + { + "epoch": 0.22755555555555557, + "grad_norm": 1.1420735285454937, + "learning_rate": 6.818181818181818e-06, + "loss": 0.5132038593292236, + "step": 16 + }, + { + "epoch": 0.24177777777777779, + "grad_norm": 1.0423494529114559, + "learning_rate": 7.272727272727273e-06, + "loss": 0.4917834401130676, + "step": 17 + }, + { + "epoch": 0.256, + "grad_norm": 0.9878207423648759, + "learning_rate": 7.727272727272727e-06, + "loss": 0.4813205897808075, + "step": 18 + }, + { + "epoch": 0.2702222222222222, + "grad_norm": 0.9711754260021513, + "learning_rate": 8.181818181818183e-06, + "loss": 0.4883510172367096, + "step": 19 + }, + { + "epoch": 0.28444444444444444, + "grad_norm": 0.9163050690645712, + "learning_rate": 8.636363636363637e-06, + "loss": 0.4871647357940674, + "step": 20 + }, + { + "epoch": 0.2986666666666667, + "grad_norm": 0.8404612592234425, + "learning_rate": 9.090909090909091e-06, + "loss": 0.46865272521972656, + "step": 21 + }, + { + "epoch": 0.3128888888888889, + "grad_norm": 0.8673877914303936, + "learning_rate": 9.545454545454547e-06, + "loss": 0.4601229727268219, + "step": 22 + }, + { + "epoch": 0.32711111111111113, + "grad_norm": 0.8211864748561771, + "learning_rate": 1e-05, + "loss": 0.45921051502227783, + "step": 23 + }, + { + "epoch": 0.3413333333333333, + "grad_norm": 0.8242615873645178, + "learning_rate": 9.999323662872998e-06, + "loss": 0.46318507194519043, + "step": 24 + }, + { + "epoch": 0.35555555555555557, + "grad_norm": 0.7839407977142685, + "learning_rate": 9.99729483446475e-06, + "loss": 0.4633931815624237, + "step": 25 + }, + { + "epoch": 0.36977777777777776, + "grad_norm": 0.7111762322751543, + "learning_rate": 9.993914063644053e-06, + "loss": 0.45212602615356445, + "step": 26 + }, + { + "epoch": 0.384, + "grad_norm": 0.718221639194032, + "learning_rate": 9.989182265027232e-06, + "loss": 0.44105663895606995, + "step": 27 + }, + { + "epoch": 0.3982222222222222, + "grad_norm": 0.7040716895578701, + "learning_rate": 9.98310071873072e-06, + "loss": 0.4427238404750824, + "step": 28 + }, + { + "epoch": 0.41244444444444445, + "grad_norm": 0.7489850512171834, + "learning_rate": 9.975671070024741e-06, + "loss": 0.443437397480011, + "step": 29 + }, + { + "epoch": 0.4266666666666667, + "grad_norm": 0.7002493913122244, + "learning_rate": 9.966895328888195e-06, + "loss": 0.4632475674152374, + "step": 30 + }, + { + "epoch": 0.4408888888888889, + "grad_norm": 0.6824040285805486, + "learning_rate": 9.956775869464901e-06, + "loss": 0.4354594349861145, + "step": 31 + }, + { + "epoch": 0.45511111111111113, + "grad_norm": 0.6597936132766806, + "learning_rate": 9.945315429421307e-06, + "loss": 0.429914653301239, + "step": 32 + }, + { + "epoch": 0.4693333333333333, + "grad_norm": 0.6352588389251546, + "learning_rate": 9.932517109205849e-06, + "loss": 0.43607252836227417, + "step": 33 + }, + { + "epoch": 0.48355555555555557, + "grad_norm": 0.6921974704448327, + "learning_rate": 9.918384371210178e-06, + "loss": 0.45671695470809937, + "step": 34 + }, + { + "epoch": 0.49777777777777776, + "grad_norm": 0.6491660902599252, + "learning_rate": 9.902921038832456e-06, + "loss": 0.4388338327407837, + "step": 35 + }, + { + "epoch": 0.512, + "grad_norm": 0.6096750069184391, + "learning_rate": 9.886131295443003e-06, + "loss": 0.4141705334186554, + "step": 36 + }, + { + "epoch": 0.5262222222222223, + "grad_norm": 0.6503340768273712, + "learning_rate": 9.868019683252543e-06, + "loss": 0.4218006432056427, + "step": 37 + }, + { + "epoch": 0.5404444444444444, + "grad_norm": 0.714483452396674, + "learning_rate": 9.848591102083375e-06, + "loss": 0.4457612931728363, + "step": 38 + }, + { + "epoch": 0.5546666666666666, + "grad_norm": 0.6231964594158875, + "learning_rate": 9.82785080804381e-06, + "loss": 0.4331759214401245, + "step": 39 + }, + { + "epoch": 0.5688888888888889, + "grad_norm": 0.6572593116495337, + "learning_rate": 9.805804412106197e-06, + "loss": 0.4507564902305603, + "step": 40 + }, + { + "epoch": 0.5831111111111111, + "grad_norm": 0.673487925488406, + "learning_rate": 9.782457878588977e-06, + "loss": 0.4254251718521118, + "step": 41 + }, + { + "epoch": 0.5973333333333334, + "grad_norm": 0.5846423543296639, + "learning_rate": 9.75781752354311e-06, + "loss": 0.4125726819038391, + "step": 42 + }, + { + "epoch": 0.6115555555555555, + "grad_norm": 0.598724301403679, + "learning_rate": 9.731890013043367e-06, + "loss": 0.4325122535228729, + "step": 43 + }, + { + "epoch": 0.6257777777777778, + "grad_norm": 0.6218044129681044, + "learning_rate": 9.704682361384941e-06, + "loss": 0.4131222367286682, + "step": 44 + }, + { + "epoch": 0.64, + "grad_norm": 0.617001419027673, + "learning_rate": 9.676201929185809e-06, + "loss": 0.42036178708076477, + "step": 45 + }, + { + "epoch": 0.6542222222222223, + "grad_norm": 0.5809155918543004, + "learning_rate": 9.646456421395447e-06, + "loss": 0.4182787537574768, + "step": 46 + }, + { + "epoch": 0.6684444444444444, + "grad_norm": 0.5850555211488232, + "learning_rate": 9.615453885210368e-06, + "loss": 0.4234634041786194, + "step": 47 + }, + { + "epoch": 0.6826666666666666, + "grad_norm": 0.5511675603941021, + "learning_rate": 9.583202707897075e-06, + "loss": 0.40563416481018066, + "step": 48 + }, + { + "epoch": 0.6968888888888889, + "grad_norm": 0.6111190985848377, + "learning_rate": 9.549711614523007e-06, + "loss": 0.3980520963668823, + "step": 49 + }, + { + "epoch": 0.7111111111111111, + "grad_norm": 0.5736293779202373, + "learning_rate": 9.514989665596114e-06, + "loss": 0.4036375880241394, + "step": 50 + }, + { + "epoch": 0.7111111111111111, + "eval_loss": 0.41823333501815796, + "eval_runtime": 28.2382, + "eval_samples_per_second": 7.083, + "eval_steps_per_second": 1.771, + "step": 50 + }, + { + "epoch": 0.7253333333333334, + "grad_norm": 0.5785272249494463, + "learning_rate": 9.479046254613673e-06, + "loss": 0.41144663095474243, + "step": 51 + }, + { + "epoch": 0.7395555555555555, + "grad_norm": 0.5517717884994647, + "learning_rate": 9.441891105521005e-06, + "loss": 0.38760095834732056, + "step": 52 + }, + { + "epoch": 0.7537777777777778, + "grad_norm": 0.5648730368507402, + "learning_rate": 9.40353427008083e-06, + "loss": 0.4036053717136383, + "step": 53 + }, + { + "epoch": 0.768, + "grad_norm": 0.6019960018903892, + "learning_rate": 9.3639861251539e-06, + "loss": 0.39992809295654297, + "step": 54 + }, + { + "epoch": 0.7822222222222223, + "grad_norm": 0.5943571857589287, + "learning_rate": 9.323257369891702e-06, + "loss": 0.420696496963501, + "step": 55 + }, + { + "epoch": 0.7964444444444444, + "grad_norm": 0.5800482521740772, + "learning_rate": 9.281359022841966e-06, + "loss": 0.41333717107772827, + "step": 56 + }, + { + "epoch": 0.8106666666666666, + "grad_norm": 0.5896438771006596, + "learning_rate": 9.238302418967757e-06, + "loss": 0.41053399443626404, + "step": 57 + }, + { + "epoch": 0.8248888888888889, + "grad_norm": 0.5828843912613246, + "learning_rate": 9.194099206580981e-06, + "loss": 0.4167237877845764, + "step": 58 + }, + { + "epoch": 0.8391111111111111, + "grad_norm": 0.636505663941606, + "learning_rate": 9.14876134419111e-06, + "loss": 0.4126594066619873, + "step": 59 + }, + { + "epoch": 0.8533333333333334, + "grad_norm": 0.5954156377152778, + "learning_rate": 9.102301097269974e-06, + "loss": 0.40986335277557373, + "step": 60 + }, + { + "epoch": 0.8675555555555555, + "grad_norm": 0.5663259824764864, + "learning_rate": 9.05473103493355e-06, + "loss": 0.39707833528518677, + "step": 61 + }, + { + "epoch": 0.8817777777777778, + "grad_norm": 0.6233708113516392, + "learning_rate": 9.006064026541549e-06, + "loss": 0.4146142601966858, + "step": 62 + }, + { + "epoch": 0.896, + "grad_norm": 0.583882066947991, + "learning_rate": 8.956313238215824e-06, + "loss": 0.3761865794658661, + "step": 63 + }, + { + "epoch": 0.9102222222222223, + "grad_norm": 0.559228243852746, + "learning_rate": 8.905492129278478e-06, + "loss": 0.397402286529541, + "step": 64 + }, + { + "epoch": 0.9244444444444444, + "grad_norm": 0.5630501047235328, + "learning_rate": 8.85361444861063e-06, + "loss": 0.40299445390701294, + "step": 65 + }, + { + "epoch": 0.9386666666666666, + "grad_norm": 0.5778346041245924, + "learning_rate": 8.800694230932885e-06, + "loss": 0.4066828489303589, + "step": 66 + }, + { + "epoch": 0.9528888888888889, + "grad_norm": 0.5735220314699014, + "learning_rate": 8.74674579300843e-06, + "loss": 0.39874544739723206, + "step": 67 + }, + { + "epoch": 0.9671111111111111, + "grad_norm": 0.5774583972046927, + "learning_rate": 8.691783729769874e-06, + "loss": 0.38473668694496155, + "step": 68 + }, + { + "epoch": 0.9813333333333333, + "grad_norm": 0.6169263233631689, + "learning_rate": 8.635822910370793e-06, + "loss": 0.4083672761917114, + "step": 69 + }, + { + "epoch": 0.9955555555555555, + "grad_norm": 0.5815111675806847, + "learning_rate": 8.578878474163115e-06, + "loss": 0.3794967234134674, + "step": 70 + }, + { + "epoch": 1.0, + "grad_norm": 0.9420572347818393, + "learning_rate": 8.520965826601394e-06, + "loss": 0.40614083409309387, + "step": 71 + }, + { + "epoch": 1.0142222222222221, + "grad_norm": 0.6281039360207515, + "learning_rate": 8.462100635075097e-06, + "loss": 0.36958712339401245, + "step": 72 + }, + { + "epoch": 1.0284444444444445, + "grad_norm": 0.6233244816927183, + "learning_rate": 8.40229882467003e-06, + "loss": 0.3494463562965393, + "step": 73 + }, + { + "epoch": 1.0426666666666666, + "grad_norm": 0.5621513032586688, + "learning_rate": 8.341576573860049e-06, + "loss": 0.3302980065345764, + "step": 74 + }, + { + "epoch": 1.056888888888889, + "grad_norm": 0.5754725580485192, + "learning_rate": 8.279950310130218e-06, + "loss": 0.35172200202941895, + "step": 75 + }, + { + "epoch": 1.0711111111111111, + "grad_norm": 0.5740681468075599, + "learning_rate": 8.2174367055326e-06, + "loss": 0.3394201993942261, + "step": 76 + }, + { + "epoch": 1.0853333333333333, + "grad_norm": 0.590460680669288, + "learning_rate": 8.154052672175888e-06, + "loss": 0.33312541246414185, + "step": 77 + }, + { + "epoch": 1.0995555555555556, + "grad_norm": 0.5842296281151235, + "learning_rate": 8.08981535765009e-06, + "loss": 0.35658037662506104, + "step": 78 + }, + { + "epoch": 1.1137777777777778, + "grad_norm": 0.6088529350096018, + "learning_rate": 8.024742140387506e-06, + "loss": 0.3449564278125763, + "step": 79 + }, + { + "epoch": 1.1280000000000001, + "grad_norm": 0.5744146541082055, + "learning_rate": 7.95885062496126e-06, + "loss": 0.3357444107532501, + "step": 80 + }, + { + "epoch": 1.1422222222222222, + "grad_norm": 0.5909574288907287, + "learning_rate": 7.892158637322647e-06, + "loss": 0.3170369267463684, + "step": 81 + }, + { + "epoch": 1.1564444444444444, + "grad_norm": 0.6136816332174048, + "learning_rate": 7.824684219978591e-06, + "loss": 0.3318628668785095, + "step": 82 + }, + { + "epoch": 1.1706666666666667, + "grad_norm": 0.5568422531541258, + "learning_rate": 7.756445627110523e-06, + "loss": 0.34082144498825073, + "step": 83 + }, + { + "epoch": 1.1848888888888889, + "grad_norm": 0.6082214692469516, + "learning_rate": 7.68746131963598e-06, + "loss": 0.3453400135040283, + "step": 84 + }, + { + "epoch": 1.199111111111111, + "grad_norm": 0.5993208838733233, + "learning_rate": 7.6177499602143e-06, + "loss": 0.3420029580593109, + "step": 85 + }, + { + "epoch": 1.2133333333333334, + "grad_norm": 0.5493880460077399, + "learning_rate": 7.547330408197695e-06, + "loss": 0.3330039381980896, + "step": 86 + }, + { + "epoch": 1.2275555555555555, + "grad_norm": 0.6151717226672565, + "learning_rate": 7.476221714529167e-06, + "loss": 0.3428732454776764, + "step": 87 + }, + { + "epoch": 1.2417777777777779, + "grad_norm": 0.6359199055571157, + "learning_rate": 7.404443116588548e-06, + "loss": 0.3575543761253357, + "step": 88 + }, + { + "epoch": 1.256, + "grad_norm": 0.5764223898323285, + "learning_rate": 7.332014032988123e-06, + "loss": 0.3414611518383026, + "step": 89 + }, + { + "epoch": 1.2702222222222221, + "grad_norm": 0.5701344890989715, + "learning_rate": 7.2589540583192165e-06, + "loss": 0.3387766480445862, + "step": 90 + }, + { + "epoch": 1.2844444444444445, + "grad_norm": 0.5603334736031269, + "learning_rate": 7.185282957851175e-06, + "loss": 0.32765448093414307, + "step": 91 + }, + { + "epoch": 1.2986666666666666, + "grad_norm": 0.5637583167829048, + "learning_rate": 7.111020662184174e-06, + "loss": 0.3432917296886444, + "step": 92 + }, + { + "epoch": 1.3128888888888888, + "grad_norm": 0.5811322458912195, + "learning_rate": 7.036187261857289e-06, + "loss": 0.3334134817123413, + "step": 93 + }, + { + "epoch": 1.3271111111111111, + "grad_norm": 0.6069998855060962, + "learning_rate": 6.960803001913315e-06, + "loss": 0.3528696894645691, + "step": 94 + }, + { + "epoch": 1.3413333333333333, + "grad_norm": 0.5846537103757758, + "learning_rate": 6.884888276421766e-06, + "loss": 0.3428453207015991, + "step": 95 + }, + { + "epoch": 1.3555555555555556, + "grad_norm": 0.5679194938574699, + "learning_rate": 6.8084636229615786e-06, + "loss": 0.33261382579803467, + "step": 96 + }, + { + "epoch": 1.3697777777777778, + "grad_norm": 0.5611361976965846, + "learning_rate": 6.731549717064975e-06, + "loss": 0.33263495564460754, + "step": 97 + }, + { + "epoch": 1.384, + "grad_norm": 0.5958752887464758, + "learning_rate": 6.654167366624009e-06, + "loss": 0.3409455418586731, + "step": 98 + }, + { + "epoch": 1.3982222222222223, + "grad_norm": 0.5821747639433966, + "learning_rate": 6.576337506261314e-06, + "loss": 0.34175097942352295, + "step": 99 + }, + { + "epoch": 1.4124444444444444, + "grad_norm": 0.5474416142301471, + "learning_rate": 6.498081191666549e-06, + "loss": 0.3422796130180359, + "step": 100 + }, + { + "epoch": 1.4124444444444444, + "eval_loss": 0.39906859397888184, + "eval_runtime": 28.0052, + "eval_samples_per_second": 7.142, + "eval_steps_per_second": 1.785, + "step": 100 + }, + { + "epoch": 1.4266666666666667, + "grad_norm": 0.5470999547878086, + "learning_rate": 6.419419593900109e-06, + "loss": 0.3311368227005005, + "step": 101 + }, + { + "epoch": 1.4408888888888889, + "grad_norm": 0.5417978810198306, + "learning_rate": 6.340373993665607e-06, + "loss": 0.32347068190574646, + "step": 102 + }, + { + "epoch": 1.455111111111111, + "grad_norm": 0.5408071080127892, + "learning_rate": 6.2609657755527135e-06, + "loss": 0.32194703817367554, + "step": 103 + }, + { + "epoch": 1.4693333333333334, + "grad_norm": 0.5610127885481652, + "learning_rate": 6.1812164222518626e-06, + "loss": 0.3477523922920227, + "step": 104 + }, + { + "epoch": 1.4835555555555555, + "grad_norm": 0.6035742435126064, + "learning_rate": 6.101147508742456e-06, + "loss": 0.33583635091781616, + "step": 105 + }, + { + "epoch": 1.4977777777777779, + "grad_norm": 0.5482179611450086, + "learning_rate": 6.020780696456059e-06, + "loss": 0.3305099904537201, + "step": 106 + }, + { + "epoch": 1.512, + "grad_norm": 0.5549975206757557, + "learning_rate": 5.940137727416247e-06, + "loss": 0.3392476439476013, + "step": 107 + }, + { + "epoch": 1.5262222222222221, + "grad_norm": 0.5452500081271687, + "learning_rate": 5.859240418356614e-06, + "loss": 0.3281456232070923, + "step": 108 + }, + { + "epoch": 1.5404444444444443, + "grad_norm": 0.569343792950831, + "learning_rate": 5.778110654818602e-06, + "loss": 0.3557584285736084, + "step": 109 + }, + { + "epoch": 1.5546666666666666, + "grad_norm": 0.5609073891735213, + "learning_rate": 5.696770385230679e-06, + "loss": 0.3375227153301239, + "step": 110 + }, + { + "epoch": 1.568888888888889, + "grad_norm": 0.5979889868516021, + "learning_rate": 5.615241614970546e-06, + "loss": 0.33440524339675903, + "step": 111 + }, + { + "epoch": 1.5831111111111111, + "grad_norm": 0.5752429374657604, + "learning_rate": 5.533546400411899e-06, + "loss": 0.3354341983795166, + "step": 112 + }, + { + "epoch": 1.5973333333333333, + "grad_norm": 0.5583834084640693, + "learning_rate": 5.4517068429574215e-06, + "loss": 0.34512901306152344, + "step": 113 + }, + { + "epoch": 1.6115555555555554, + "grad_norm": 0.5366739842708482, + "learning_rate": 5.3697450830595775e-06, + "loss": 0.3282182812690735, + "step": 114 + }, + { + "epoch": 1.6257777777777778, + "grad_norm": 0.5923091160755782, + "learning_rate": 5.287683294230855e-06, + "loss": 0.3325689136981964, + "step": 115 + }, + { + "epoch": 1.6400000000000001, + "grad_norm": 0.5603788239590529, + "learning_rate": 5.20554367704505e-06, + "loss": 0.3167092204093933, + "step": 116 + }, + { + "epoch": 1.6542222222222223, + "grad_norm": 0.5675689042120237, + "learning_rate": 5.123348453131242e-06, + "loss": 0.3419051766395569, + "step": 117 + }, + { + "epoch": 1.6684444444444444, + "grad_norm": 0.5456542923932385, + "learning_rate": 5.041119859162068e-06, + "loss": 0.333318293094635, + "step": 118 + }, + { + "epoch": 1.6826666666666665, + "grad_norm": 0.570188952777049, + "learning_rate": 4.958880140837934e-06, + "loss": 0.33905792236328125, + "step": 119 + }, + { + "epoch": 1.696888888888889, + "grad_norm": 0.5534588670481234, + "learning_rate": 4.876651546868759e-06, + "loss": 0.32569897174835205, + "step": 120 + }, + { + "epoch": 1.7111111111111112, + "grad_norm": 0.5546097096941289, + "learning_rate": 4.794456322954953e-06, + "loss": 0.3532940745353699, + "step": 121 + }, + { + "epoch": 1.7253333333333334, + "grad_norm": 0.5857002004886875, + "learning_rate": 4.7123167057691446e-06, + "loss": 0.35066288709640503, + "step": 122 + }, + { + "epoch": 1.7395555555555555, + "grad_norm": 0.5351298499201909, + "learning_rate": 4.630254916940424e-06, + "loss": 0.35048598051071167, + "step": 123 + }, + { + "epoch": 1.7537777777777777, + "grad_norm": 0.5572795301130913, + "learning_rate": 4.548293157042581e-06, + "loss": 0.3389701843261719, + "step": 124 + }, + { + "epoch": 1.768, + "grad_norm": 0.5503673971003669, + "learning_rate": 4.466453599588103e-06, + "loss": 0.3477008044719696, + "step": 125 + }, + { + "epoch": 1.7822222222222224, + "grad_norm": 0.5485965256926039, + "learning_rate": 4.384758385029457e-06, + "loss": 0.3188989758491516, + "step": 126 + }, + { + "epoch": 1.7964444444444445, + "grad_norm": 0.5566326955710196, + "learning_rate": 4.3032296147693225e-06, + "loss": 0.3296913504600525, + "step": 127 + }, + { + "epoch": 1.8106666666666666, + "grad_norm": 0.5592929839051507, + "learning_rate": 4.2218893451814005e-06, + "loss": 0.3278958797454834, + "step": 128 + }, + { + "epoch": 1.8248888888888888, + "grad_norm": 0.5372208136274655, + "learning_rate": 4.140759581643386e-06, + "loss": 0.3338126838207245, + "step": 129 + }, + { + "epoch": 1.8391111111111111, + "grad_norm": 0.5506377641318588, + "learning_rate": 4.059862272583755e-06, + "loss": 0.33511388301849365, + "step": 130 + }, + { + "epoch": 1.8533333333333335, + "grad_norm": 0.5509814672788427, + "learning_rate": 3.979219303543942e-06, + "loss": 0.3190721869468689, + "step": 131 + }, + { + "epoch": 1.8675555555555556, + "grad_norm": 0.5437024683532695, + "learning_rate": 3.898852491257547e-06, + "loss": 0.3397029638290405, + "step": 132 + }, + { + "epoch": 1.8817777777777778, + "grad_norm": 0.5577533834862016, + "learning_rate": 3.818783577748138e-06, + "loss": 0.3368285894393921, + "step": 133 + }, + { + "epoch": 1.896, + "grad_norm": 0.5654570173541145, + "learning_rate": 3.7390342244472886e-06, + "loss": 0.34375038743019104, + "step": 134 + }, + { + "epoch": 1.9102222222222223, + "grad_norm": 0.5435541441710147, + "learning_rate": 3.659626006334395e-06, + "loss": 0.32158786058425903, + "step": 135 + }, + { + "epoch": 1.9244444444444444, + "grad_norm": 0.5517597326060792, + "learning_rate": 3.5805804060998926e-06, + "loss": 0.32734596729278564, + "step": 136 + }, + { + "epoch": 1.9386666666666668, + "grad_norm": 0.5529004589065376, + "learning_rate": 3.501918808333453e-06, + "loss": 0.3495728671550751, + "step": 137 + }, + { + "epoch": 1.952888888888889, + "grad_norm": 0.5497949696155701, + "learning_rate": 3.4236624937386874e-06, + "loss": 0.3545406758785248, + "step": 138 + }, + { + "epoch": 1.967111111111111, + "grad_norm": 0.50810534397043, + "learning_rate": 3.3458326333759927e-06, + "loss": 0.3283476233482361, + "step": 139 + }, + { + "epoch": 1.9813333333333332, + "grad_norm": 0.5410931225650443, + "learning_rate": 3.268450282935026e-06, + "loss": 0.3362443447113037, + "step": 140 + }, + { + "epoch": 1.9955555555555555, + "grad_norm": 0.5526133533239165, + "learning_rate": 3.1915363770384223e-06, + "loss": 0.33009064197540283, + "step": 141 + }, + { + "epoch": 2.0, + "grad_norm": 0.8788381822845209, + "learning_rate": 3.1151117235782346e-06, + "loss": 0.34658753871917725, + "step": 142 + }, + { + "epoch": 2.014222222222222, + "grad_norm": 0.6321149682877057, + "learning_rate": 3.0391969980866874e-06, + "loss": 0.2953590750694275, + "step": 143 + }, + { + "epoch": 2.0284444444444443, + "grad_norm": 0.5718010363312446, + "learning_rate": 2.963812738142713e-06, + "loss": 0.28190308809280396, + "step": 144 + }, + { + "epoch": 2.042666666666667, + "grad_norm": 0.5860488240075866, + "learning_rate": 2.8889793378158284e-06, + "loss": 0.29579877853393555, + "step": 145 + }, + { + "epoch": 2.056888888888889, + "grad_norm": 0.5087999304214683, + "learning_rate": 2.814717042148827e-06, + "loss": 0.2849843502044678, + "step": 146 + }, + { + "epoch": 2.071111111111111, + "grad_norm": 0.5097817184428313, + "learning_rate": 2.7410459416807856e-06, + "loss": 0.2797221541404724, + "step": 147 + }, + { + "epoch": 2.0853333333333333, + "grad_norm": 0.5476441083873779, + "learning_rate": 2.6679859670118785e-06, + "loss": 0.2898455262184143, + "step": 148 + }, + { + "epoch": 2.0995555555555554, + "grad_norm": 0.5438939155777798, + "learning_rate": 2.5955568834114523e-06, + "loss": 0.27163612842559814, + "step": 149 + }, + { + "epoch": 2.113777777777778, + "grad_norm": 0.5562699539267528, + "learning_rate": 2.523778285470835e-06, + "loss": 0.28451889753341675, + "step": 150 + }, + { + "epoch": 2.113777777777778, + "eval_loss": 0.4041806757450104, + "eval_runtime": 27.9066, + "eval_samples_per_second": 7.167, + "eval_steps_per_second": 1.792, + "step": 150 + }, + { + "epoch": 2.128, + "grad_norm": 0.5511756829107596, + "learning_rate": 2.452669591802307e-06, + "loss": 0.2696727514266968, + "step": 151 + }, + { + "epoch": 2.1422222222222222, + "grad_norm": 0.5230762023952198, + "learning_rate": 2.3822500397857016e-06, + "loss": 0.2786106467247009, + "step": 152 + }, + { + "epoch": 2.1564444444444444, + "grad_norm": 0.5376649924382556, + "learning_rate": 2.3125386803640188e-06, + "loss": 0.29131805896759033, + "step": 153 + }, + { + "epoch": 2.1706666666666665, + "grad_norm": 0.5176483788211893, + "learning_rate": 2.243554372889479e-06, + "loss": 0.2755666971206665, + "step": 154 + }, + { + "epoch": 2.1848888888888887, + "grad_norm": 0.5045947338221831, + "learning_rate": 2.1753157800214107e-06, + "loss": 0.27331122756004333, + "step": 155 + }, + { + "epoch": 2.1991111111111112, + "grad_norm": 0.5193109465079893, + "learning_rate": 2.1078413626773547e-06, + "loss": 0.28715455532073975, + "step": 156 + }, + { + "epoch": 2.2133333333333334, + "grad_norm": 0.5212127411265028, + "learning_rate": 2.0411493750387423e-06, + "loss": 0.26138654351234436, + "step": 157 + }, + { + "epoch": 2.2275555555555555, + "grad_norm": 0.5448628823500382, + "learning_rate": 1.9752578596124955e-06, + "loss": 0.29015523195266724, + "step": 158 + }, + { + "epoch": 2.2417777777777776, + "grad_norm": 0.5501585558078743, + "learning_rate": 1.9101846423499113e-06, + "loss": 0.2733340561389923, + "step": 159 + }, + { + "epoch": 2.2560000000000002, + "grad_norm": 0.5315657420322433, + "learning_rate": 1.8459473278241125e-06, + "loss": 0.2742033302783966, + "step": 160 + }, + { + "epoch": 2.2702222222222224, + "grad_norm": 0.4902572304429622, + "learning_rate": 1.7825632944674016e-06, + "loss": 0.2752198874950409, + "step": 161 + }, + { + "epoch": 2.2844444444444445, + "grad_norm": 0.534591959936222, + "learning_rate": 1.7200496898697832e-06, + "loss": 0.28458085656166077, + "step": 162 + }, + { + "epoch": 2.2986666666666666, + "grad_norm": 0.5106425495762753, + "learning_rate": 1.6584234261399535e-06, + "loss": 0.277709424495697, + "step": 163 + }, + { + "epoch": 2.3128888888888888, + "grad_norm": 0.5539379653737809, + "learning_rate": 1.5977011753299726e-06, + "loss": 0.2714066505432129, + "step": 164 + }, + { + "epoch": 2.327111111111111, + "grad_norm": 0.5579822508546088, + "learning_rate": 1.5378993649249053e-06, + "loss": 0.2929142117500305, + "step": 165 + }, + { + "epoch": 2.3413333333333335, + "grad_norm": 0.5128512129047608, + "learning_rate": 1.4790341733986085e-06, + "loss": 0.274400919675827, + "step": 166 + }, + { + "epoch": 2.3555555555555556, + "grad_norm": 0.5180316098734932, + "learning_rate": 1.4211215258368866e-06, + "loss": 0.2633841633796692, + "step": 167 + }, + { + "epoch": 2.3697777777777778, + "grad_norm": 0.5226866839878248, + "learning_rate": 1.3641770896292083e-06, + "loss": 0.271608829498291, + "step": 168 + }, + { + "epoch": 2.384, + "grad_norm": 0.5213295087948763, + "learning_rate": 1.3082162702301276e-06, + "loss": 0.2772142291069031, + "step": 169 + }, + { + "epoch": 2.398222222222222, + "grad_norm": 0.5200525913463248, + "learning_rate": 1.2532542069915722e-06, + "loss": 0.2833596467971802, + "step": 170 + }, + { + "epoch": 2.4124444444444446, + "grad_norm": 0.5158405462335311, + "learning_rate": 1.1993057690671174e-06, + "loss": 0.27554380893707275, + "step": 171 + }, + { + "epoch": 2.4266666666666667, + "grad_norm": 0.5070004563853152, + "learning_rate": 1.1463855513893695e-06, + "loss": 0.291002094745636, + "step": 172 + }, + { + "epoch": 2.440888888888889, + "grad_norm": 0.5227423053894303, + "learning_rate": 1.0945078707215224e-06, + "loss": 0.2779809832572937, + "step": 173 + }, + { + "epoch": 2.455111111111111, + "grad_norm": 0.530364411528684, + "learning_rate": 1.0436867617841768e-06, + "loss": 0.2763213515281677, + "step": 174 + }, + { + "epoch": 2.469333333333333, + "grad_norm": 0.5210228993927588, + "learning_rate": 9.939359734584552e-07, + "loss": 0.28753769397735596, + "step": 175 + }, + { + "epoch": 2.4835555555555557, + "grad_norm": 0.5214828400537481, + "learning_rate": 9.452689650664515e-07, + "loss": 0.2837107181549072, + "step": 176 + }, + { + "epoch": 2.497777777777778, + "grad_norm": 0.5036225380576258, + "learning_rate": 8.976989027300265e-07, + "loss": 0.2816150188446045, + "step": 177 + }, + { + "epoch": 2.512, + "grad_norm": 0.5488666037756793, + "learning_rate": 8.512386558088919e-07, + "loss": 0.2889126241207123, + "step": 178 + }, + { + "epoch": 2.526222222222222, + "grad_norm": 0.5181437270813625, + "learning_rate": 8.059007934190194e-07, + "loss": 0.2788556218147278, + "step": 179 + }, + { + "epoch": 2.5404444444444443, + "grad_norm": 0.5160804538602589, + "learning_rate": 7.61697581032243e-07, + "loss": 0.29226237535476685, + "step": 180 + }, + { + "epoch": 2.554666666666667, + "grad_norm": 0.5105467594147919, + "learning_rate": 7.186409771580355e-07, + "loss": 0.2915460765361786, + "step": 181 + }, + { + "epoch": 2.568888888888889, + "grad_norm": 0.49398572364663706, + "learning_rate": 6.76742630108298e-07, + "loss": 0.2790933847427368, + "step": 182 + }, + { + "epoch": 2.583111111111111, + "grad_norm": 1.14219599430935e+31, + "learning_rate": 6.360138748461015e-07, + "loss": 0.27808326482772827, + "step": 183 + }, + { + "epoch": 2.5973333333333333, + "grad_norm": 0.4983345243585802, + "learning_rate": 5.964657299191712e-07, + "loss": 0.2690117359161377, + "step": 184 + }, + { + "epoch": 2.6115555555555554, + "grad_norm": 0.5069008538717243, + "learning_rate": 5.581088944789953e-07, + "loss": 0.27599871158599854, + "step": 185 + }, + { + "epoch": 2.6257777777777775, + "grad_norm": 0.521476062094092, + "learning_rate": 5.20953745386329e-07, + "loss": 0.2919716238975525, + "step": 186 + }, + { + "epoch": 2.64, + "grad_norm": 0.510790594456331, + "learning_rate": 4.850103344038853e-07, + "loss": 0.28502631187438965, + "step": 187 + }, + { + "epoch": 2.6542222222222223, + "grad_norm": 0.49201617380837526, + "learning_rate": 4.502883854769935e-07, + "loss": 0.26374831795692444, + "step": 188 + }, + { + "epoch": 2.6684444444444444, + "grad_norm": 0.5211970358874106, + "learning_rate": 4.167972921029262e-07, + "loss": 0.28197193145751953, + "step": 189 + }, + { + "epoch": 2.6826666666666665, + "grad_norm": 0.5370264524554923, + "learning_rate": 3.8454611478963235e-07, + "loss": 0.2938516139984131, + "step": 190 + }, + { + "epoch": 2.696888888888889, + "grad_norm": 0.5232192741343314, + "learning_rate": 3.535435786045538e-07, + "loss": 0.27452725172042847, + "step": 191 + }, + { + "epoch": 2.7111111111111112, + "grad_norm": 0.5024016728659283, + "learning_rate": 3.237980708141919e-07, + "loss": 0.2790898084640503, + "step": 192 + }, + { + "epoch": 2.7253333333333334, + "grad_norm": 0.5309491933991259, + "learning_rate": 2.9531763861505967e-07, + "loss": 0.2802097201347351, + "step": 193 + }, + { + "epoch": 2.7395555555555555, + "grad_norm": 0.5117668670583797, + "learning_rate": 2.681099869566328e-07, + "loss": 0.2703718841075897, + "step": 194 + }, + { + "epoch": 2.7537777777777777, + "grad_norm": 0.5093979232137983, + "learning_rate": 2.4218247645689306e-07, + "loss": 0.2852793335914612, + "step": 195 + }, + { + "epoch": 2.768, + "grad_norm": 0.49789356969379145, + "learning_rate": 2.175421214110235e-07, + "loss": 0.27227064967155457, + "step": 196 + }, + { + "epoch": 2.7822222222222224, + "grad_norm": 0.5255831952638869, + "learning_rate": 1.941955878938029e-07, + "loss": 0.28222954273223877, + "step": 197 + }, + { + "epoch": 2.7964444444444445, + "grad_norm": 0.5060979808451155, + "learning_rate": 1.721491919561913e-07, + "loss": 0.2887798249721527, + "step": 198 + }, + { + "epoch": 2.8106666666666666, + "grad_norm": 0.5024415391860155, + "learning_rate": 1.514088979166256e-07, + "loss": 0.2730322480201721, + "step": 199 + }, + { + "epoch": 2.824888888888889, + "grad_norm": 0.5071775702854565, + "learning_rate": 1.3198031674745814e-07, + "loss": 0.28655844926834106, + "step": 200 + }, + { + "epoch": 2.824888888888889, + "eval_loss": 0.3998975455760956, + "eval_runtime": 27.4547, + "eval_samples_per_second": 7.285, + "eval_steps_per_second": 1.821, + "step": 200 + }, + { + "epoch": 2.8391111111111114, + "grad_norm": 0.5098506733931571, + "learning_rate": 1.138687045569975e-07, + "loss": 0.28616365790367126, + "step": 201 + }, + { + "epoch": 2.8533333333333335, + "grad_norm": 0.5014028056158423, + "learning_rate": 9.70789611675449e-08, + "loss": 0.28517472743988037, + "step": 202 + }, + { + "epoch": 2.8675555555555556, + "grad_norm": 0.5046489746973516, + "learning_rate": 8.161562878982399e-08, + "loss": 0.27302050590515137, + "step": 203 + }, + { + "epoch": 2.8817777777777778, + "grad_norm": 0.5244675639408302, + "learning_rate": 6.74828907941516e-08, + "loss": 0.28563594818115234, + "step": 204 + }, + { + "epoch": 2.896, + "grad_norm": 0.5147937769432819, + "learning_rate": 5.468457057869358e-08, + "loss": 0.2887130379676819, + "step": 205 + }, + { + "epoch": 2.910222222222222, + "grad_norm": 0.49008819658339253, + "learning_rate": 4.322413053509944e-08, + "loss": 0.2721640467643738, + "step": 206 + }, + { + "epoch": 2.924444444444444, + "grad_norm": 0.5051230135404199, + "learning_rate": 3.31046711118066e-08, + "loss": 0.27188998460769653, + "step": 207 + }, + { + "epoch": 2.9386666666666668, + "grad_norm": 0.5046747206900266, + "learning_rate": 2.4328929975260262e-08, + "loss": 0.2795502245426178, + "step": 208 + }, + { + "epoch": 2.952888888888889, + "grad_norm": 0.5035617947795644, + "learning_rate": 1.6899281269279756e-08, + "loss": 0.2727120816707611, + "step": 209 + }, + { + "epoch": 2.967111111111111, + "grad_norm": 0.5078812650660502, + "learning_rate": 1.0817734972768946e-08, + "loss": 0.2795644998550415, + "step": 210 + }, + { + "epoch": 2.981333333333333, + "grad_norm": 0.5269813851443549, + "learning_rate": 6.085936355947897e-09, + "loss": 0.28946471214294434, + "step": 211 + }, + { + "epoch": 2.9955555555555557, + "grad_norm": 0.503969003244228, + "learning_rate": 2.7051655352494654e-09, + "loss": 0.26888906955718994, + "step": 212 + }, + { + "epoch": 3.0, + "grad_norm": 0.9730892144000087, + "learning_rate": 6.763371270035457e-10, + "loss": 0.33630549907684326, + "step": 213 + } + ], + "logging_steps": 1, + "max_steps": 213, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 62893478699008.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/qwen2.5-7b-instruct/checkpoint-213/training_args.bin b/qwen2.5-7b-instruct/checkpoint-213/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..e59d7187e91e3dd0138c38f45c246765301e22e6 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:238e1f7259942c35813600da4056c3d822147cad18b0d1147780f439481f9c35 +size 7377 diff --git a/qwen2.5-7b-instruct/checkpoint-213/zero_to_fp32.py b/qwen2.5-7b-instruct/checkpoint-213/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-213/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/qwen2.5-7b-instruct/checkpoint-71/chat_template.jinja b/qwen2.5-7b-instruct/checkpoint-71/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/qwen2.5-7b-instruct/checkpoint-71/config.json b/qwen2.5-7b-instruct/checkpoint-71/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2948dc473025f5f3adb7c03ee0a5609588eb5c79 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 18944, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 28, + "model_type": "qwen2", + "num_attention_heads": 28, + "num_hidden_layers": 28, + "num_key_value_heads": 4, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.2.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 152064 +} diff --git a/qwen2.5-7b-instruct/checkpoint-71/generation_config.json b/qwen2.5-7b-instruct/checkpoint-71/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..47205fbf3386a215e7c2b9d62df6576f49b77e7b --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.05, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.2.0" +} diff --git a/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_0_mp_rank_00_model_states.pt b/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_0_mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4081d9db287c73ed2188e5956ca1e28cd50dc4ca --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_0_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3d3498303bb4bed5fb4425a4a67f0989f9f5e02ca6c8908f91570b3381486fb +size 167072 diff --git a/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e70c19c66c8e2ffbd8039e2eba0ae52dccd303ef --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5598926945c1c4fafe12563887531758060617d5338ef79b15616d2b1deaefdf +size 22846856441 diff --git a/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_1_mp_rank_00_model_states.pt b/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_1_mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..09b07bf18c1fc43162b7314d001c7b0185bc3b37 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_1_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad882420a980b56c5b187d5632cfee8062791ce7c4f6d8fd55b182b5733cef7d +size 167072 diff --git a/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2816f8024e0d5dc9873e9a1d900652848a1a5088 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e427601c11d53a4b94a56e5c65bb37d1630d92170801839c5752368cc28bd681 +size 22846856441 diff --git a/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_2_mp_rank_00_model_states.pt b/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_2_mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a7c153b9cf2dd5fa3d227c5a1bb13cd43b6ddedb --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_2_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49038e93d0cba5a941440173dad71221d177b50281b14e9db4098a021a23b6c5 +size 167072 diff --git a/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_2_mp_rank_00_optim_states.pt b/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a4483ec9392a5274ce9d86dbfb782ea4e2aa7514 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:718f2823439466b93d8de79348d5d6ce32731f9797f92140673d84d4768a7030 +size 22846856441 diff --git a/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_3_mp_rank_00_model_states.pt b/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_3_mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d95e02e2354564c41bc0d72b0c239b86689837a8 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_3_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06c6ecac9984c2f05df2dad94f300e4ae686c5d336cee696ce58d27c3940dad1 +size 167072 diff --git a/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_3_mp_rank_00_optim_states.pt b/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..24b61114b7a7d475f2bc2de4321cd7d7eac7edbf --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/global_step71/zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9691b08fe4efef57d3f206f1ba9450fd7f317f5a6a40965eae4be47c006f3050 +size 22846856441 diff --git a/qwen2.5-7b-instruct/checkpoint-71/latest b/qwen2.5-7b-instruct/checkpoint-71/latest new file mode 100644 index 0000000000000000000000000000000000000000..bbeadc7466d2728e3046120a012ebc37c29267cb --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/latest @@ -0,0 +1 @@ +global_step71 \ No newline at end of file diff --git a/qwen2.5-7b-instruct/checkpoint-71/model.safetensors b/qwen2.5-7b-instruct/checkpoint-71/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1f7f3cb0c6b27805dcdb386b376a2f74a26d0481 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:521c8db18ad282a6033b394ffd0c655c11ecf3b5e7a85e0a9e1ee07960c254c9 +size 30462505080 diff --git a/qwen2.5-7b-instruct/checkpoint-71/rng_state_0.pth b/qwen2.5-7b-instruct/checkpoint-71/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..b2e0e7337f62c934e90c141589ca185ff78fae36 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fe6b873f1da64bb980afe982fc1c6b6bc461ad8c33073d856804323f64b26d9 +size 15429 diff --git a/qwen2.5-7b-instruct/checkpoint-71/rng_state_1.pth b/qwen2.5-7b-instruct/checkpoint-71/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..204fc909b58b8ae0fab9c8b0f41ea5cc8f3277f8 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13db4dd10b22b5464f5ef0b539152b46390eae6a827b355108cb54b1af2eb830 +size 15429 diff --git a/qwen2.5-7b-instruct/checkpoint-71/rng_state_2.pth b/qwen2.5-7b-instruct/checkpoint-71/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..0c3095386194e2f629b38c7957abe0b9ab74aa8b --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06e293f13fb6e873e8081636b6b07115410b397c55f1ed08b8772b059c8eb74e +size 15429 diff --git a/qwen2.5-7b-instruct/checkpoint-71/rng_state_3.pth b/qwen2.5-7b-instruct/checkpoint-71/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..fdf7f18d5d86f0d363c9ca44b9ad169d469e47f7 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c595c077664f94b7f91088561951be20e39483e01865774a83bc46f3971a3152 +size 15429 diff --git a/qwen2.5-7b-instruct/checkpoint-71/scheduler.pt b/qwen2.5-7b-instruct/checkpoint-71/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3be1d29bde40f225713ff5084d3d9d69858fcfd --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c96c28f12671e2f5db493417daefb3ff0b1bc36602843e0d7ef6df3649c27b3 +size 1465 diff --git a/qwen2.5-7b-instruct/checkpoint-71/tokenizer.json b/qwen2.5-7b-instruct/checkpoint-71/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..34510ff0037cd50428af467a17ead5a96140a32c --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/qwen2.5-7b-instruct/checkpoint-71/tokenizer_config.json b/qwen2.5-7b-instruct/checkpoint-71/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..145e2c7a68b109ae0253dda29fc2e66cb01b4b85 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen2.5-7b-instruct/checkpoint-71/trainer_state.json b/qwen2.5-7b-instruct/checkpoint-71/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..72870fa98cb7d83bcca6136f6b2044a9fdc3ffa0 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/trainer_state.json @@ -0,0 +1,539 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 50, + "global_step": 71, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.014222222222222223, + "grad_norm": 10.21201439945407, + "learning_rate": 0.0, + "loss": 0.9819135069847107, + "step": 1 + }, + { + "epoch": 0.028444444444444446, + "grad_norm": 10.607308805754391, + "learning_rate": 4.5454545454545457e-07, + "loss": 0.992728054523468, + "step": 2 + }, + { + "epoch": 0.042666666666666665, + "grad_norm": 8.137000657744727, + "learning_rate": 9.090909090909091e-07, + "loss": 0.88365638256073, + "step": 3 + }, + { + "epoch": 0.05688888888888889, + "grad_norm": 5.015580274302158, + "learning_rate": 1.3636363636363636e-06, + "loss": 0.8155229091644287, + "step": 4 + }, + { + "epoch": 0.07111111111111111, + "grad_norm": 3.513429105458463, + "learning_rate": 1.8181818181818183e-06, + "loss": 0.7148971557617188, + "step": 5 + }, + { + "epoch": 0.08533333333333333, + "grad_norm": 2.890137665957255, + "learning_rate": 2.2727272727272728e-06, + "loss": 0.7388776540756226, + "step": 6 + }, + { + "epoch": 0.09955555555555555, + "grad_norm": 2.6650307199214267, + "learning_rate": 2.7272727272727272e-06, + "loss": 0.6954053044319153, + "step": 7 + }, + { + "epoch": 0.11377777777777778, + "grad_norm": 2.1027438786791453, + "learning_rate": 3.181818181818182e-06, + "loss": 0.6385300159454346, + "step": 8 + }, + { + "epoch": 0.128, + "grad_norm": 1.9707186974912183, + "learning_rate": 3.6363636363636366e-06, + "loss": 0.5974307060241699, + "step": 9 + }, + { + "epoch": 0.14222222222222222, + "grad_norm": 1.7892988232740288, + "learning_rate": 4.0909090909090915e-06, + "loss": 0.5823870897293091, + "step": 10 + }, + { + "epoch": 0.15644444444444444, + "grad_norm": 1.8117663888596973, + "learning_rate": 4.5454545454545455e-06, + "loss": 0.5595569610595703, + "step": 11 + }, + { + "epoch": 0.17066666666666666, + "grad_norm": 1.6892201331978909, + "learning_rate": 5e-06, + "loss": 0.5447146892547607, + "step": 12 + }, + { + "epoch": 0.18488888888888888, + "grad_norm": 1.5301037695464916, + "learning_rate": 5.4545454545454545e-06, + "loss": 0.5179443359375, + "step": 13 + }, + { + "epoch": 0.1991111111111111, + "grad_norm": 1.2533624125991738, + "learning_rate": 5.90909090909091e-06, + "loss": 0.5241127014160156, + "step": 14 + }, + { + "epoch": 0.21333333333333335, + "grad_norm": 1.2394418596122698, + "learning_rate": 6.363636363636364e-06, + "loss": 0.4842333197593689, + "step": 15 + }, + { + "epoch": 0.22755555555555557, + "grad_norm": 1.1420735285454937, + "learning_rate": 6.818181818181818e-06, + "loss": 0.5132038593292236, + "step": 16 + }, + { + "epoch": 0.24177777777777779, + "grad_norm": 1.0423494529114559, + "learning_rate": 7.272727272727273e-06, + "loss": 0.4917834401130676, + "step": 17 + }, + { + "epoch": 0.256, + "grad_norm": 0.9878207423648759, + "learning_rate": 7.727272727272727e-06, + "loss": 0.4813205897808075, + "step": 18 + }, + { + "epoch": 0.2702222222222222, + "grad_norm": 0.9711754260021513, + "learning_rate": 8.181818181818183e-06, + "loss": 0.4883510172367096, + "step": 19 + }, + { + "epoch": 0.28444444444444444, + "grad_norm": 0.9163050690645712, + "learning_rate": 8.636363636363637e-06, + "loss": 0.4871647357940674, + "step": 20 + }, + { + "epoch": 0.2986666666666667, + "grad_norm": 0.8404612592234425, + "learning_rate": 9.090909090909091e-06, + "loss": 0.46865272521972656, + "step": 21 + }, + { + "epoch": 0.3128888888888889, + "grad_norm": 0.8673877914303936, + "learning_rate": 9.545454545454547e-06, + "loss": 0.4601229727268219, + "step": 22 + }, + { + "epoch": 0.32711111111111113, + "grad_norm": 0.8211864748561771, + "learning_rate": 1e-05, + "loss": 0.45921051502227783, + "step": 23 + }, + { + "epoch": 0.3413333333333333, + "grad_norm": 0.8242615873645178, + "learning_rate": 9.999323662872998e-06, + "loss": 0.46318507194519043, + "step": 24 + }, + { + "epoch": 0.35555555555555557, + "grad_norm": 0.7839407977142685, + "learning_rate": 9.99729483446475e-06, + "loss": 0.4633931815624237, + "step": 25 + }, + { + "epoch": 0.36977777777777776, + "grad_norm": 0.7111762322751543, + "learning_rate": 9.993914063644053e-06, + "loss": 0.45212602615356445, + "step": 26 + }, + { + "epoch": 0.384, + "grad_norm": 0.718221639194032, + "learning_rate": 9.989182265027232e-06, + "loss": 0.44105663895606995, + "step": 27 + }, + { + "epoch": 0.3982222222222222, + "grad_norm": 0.7040716895578701, + "learning_rate": 9.98310071873072e-06, + "loss": 0.4427238404750824, + "step": 28 + }, + { + "epoch": 0.41244444444444445, + "grad_norm": 0.7489850512171834, + "learning_rate": 9.975671070024741e-06, + "loss": 0.443437397480011, + "step": 29 + }, + { + "epoch": 0.4266666666666667, + "grad_norm": 0.7002493913122244, + "learning_rate": 9.966895328888195e-06, + "loss": 0.4632475674152374, + "step": 30 + }, + { + "epoch": 0.4408888888888889, + "grad_norm": 0.6824040285805486, + "learning_rate": 9.956775869464901e-06, + "loss": 0.4354594349861145, + "step": 31 + }, + { + "epoch": 0.45511111111111113, + "grad_norm": 0.6597936132766806, + "learning_rate": 9.945315429421307e-06, + "loss": 0.429914653301239, + "step": 32 + }, + { + "epoch": 0.4693333333333333, + "grad_norm": 0.6352588389251546, + "learning_rate": 9.932517109205849e-06, + "loss": 0.43607252836227417, + "step": 33 + }, + { + "epoch": 0.48355555555555557, + "grad_norm": 0.6921974704448327, + "learning_rate": 9.918384371210178e-06, + "loss": 0.45671695470809937, + "step": 34 + }, + { + "epoch": 0.49777777777777776, + "grad_norm": 0.6491660902599252, + "learning_rate": 9.902921038832456e-06, + "loss": 0.4388338327407837, + "step": 35 + }, + { + "epoch": 0.512, + "grad_norm": 0.6096750069184391, + "learning_rate": 9.886131295443003e-06, + "loss": 0.4141705334186554, + "step": 36 + }, + { + "epoch": 0.5262222222222223, + "grad_norm": 0.6503340768273712, + "learning_rate": 9.868019683252543e-06, + "loss": 0.4218006432056427, + "step": 37 + }, + { + "epoch": 0.5404444444444444, + "grad_norm": 0.714483452396674, + "learning_rate": 9.848591102083375e-06, + "loss": 0.4457612931728363, + "step": 38 + }, + { + "epoch": 0.5546666666666666, + "grad_norm": 0.6231964594158875, + "learning_rate": 9.82785080804381e-06, + "loss": 0.4331759214401245, + "step": 39 + }, + { + "epoch": 0.5688888888888889, + "grad_norm": 0.6572593116495337, + "learning_rate": 9.805804412106197e-06, + "loss": 0.4507564902305603, + "step": 40 + }, + { + "epoch": 0.5831111111111111, + "grad_norm": 0.673487925488406, + "learning_rate": 9.782457878588977e-06, + "loss": 0.4254251718521118, + "step": 41 + }, + { + "epoch": 0.5973333333333334, + "grad_norm": 0.5846423543296639, + "learning_rate": 9.75781752354311e-06, + "loss": 0.4125726819038391, + "step": 42 + }, + { + "epoch": 0.6115555555555555, + "grad_norm": 0.598724301403679, + "learning_rate": 9.731890013043367e-06, + "loss": 0.4325122535228729, + "step": 43 + }, + { + "epoch": 0.6257777777777778, + "grad_norm": 0.6218044129681044, + "learning_rate": 9.704682361384941e-06, + "loss": 0.4131222367286682, + "step": 44 + }, + { + "epoch": 0.64, + "grad_norm": 0.617001419027673, + "learning_rate": 9.676201929185809e-06, + "loss": 0.42036178708076477, + "step": 45 + }, + { + "epoch": 0.6542222222222223, + "grad_norm": 0.5809155918543004, + "learning_rate": 9.646456421395447e-06, + "loss": 0.4182787537574768, + "step": 46 + }, + { + "epoch": 0.6684444444444444, + "grad_norm": 0.5850555211488232, + "learning_rate": 9.615453885210368e-06, + "loss": 0.4234634041786194, + "step": 47 + }, + { + "epoch": 0.6826666666666666, + "grad_norm": 0.5511675603941021, + "learning_rate": 9.583202707897075e-06, + "loss": 0.40563416481018066, + "step": 48 + }, + { + "epoch": 0.6968888888888889, + "grad_norm": 0.6111190985848377, + "learning_rate": 9.549711614523007e-06, + "loss": 0.3980520963668823, + "step": 49 + }, + { + "epoch": 0.7111111111111111, + "grad_norm": 0.5736293779202373, + "learning_rate": 9.514989665596114e-06, + "loss": 0.4036375880241394, + "step": 50 + }, + { + "epoch": 0.7111111111111111, + "eval_loss": 0.41823333501815796, + "eval_runtime": 28.2382, + "eval_samples_per_second": 7.083, + "eval_steps_per_second": 1.771, + "step": 50 + }, + { + "epoch": 0.7253333333333334, + "grad_norm": 0.5785272249494463, + "learning_rate": 9.479046254613673e-06, + "loss": 0.41144663095474243, + "step": 51 + }, + { + "epoch": 0.7395555555555555, + "grad_norm": 0.5517717884994647, + "learning_rate": 9.441891105521005e-06, + "loss": 0.38760095834732056, + "step": 52 + }, + { + "epoch": 0.7537777777777778, + "grad_norm": 0.5648730368507402, + "learning_rate": 9.40353427008083e-06, + "loss": 0.4036053717136383, + "step": 53 + }, + { + "epoch": 0.768, + "grad_norm": 0.6019960018903892, + "learning_rate": 9.3639861251539e-06, + "loss": 0.39992809295654297, + "step": 54 + }, + { + "epoch": 0.7822222222222223, + "grad_norm": 0.5943571857589287, + "learning_rate": 9.323257369891702e-06, + "loss": 0.420696496963501, + "step": 55 + }, + { + "epoch": 0.7964444444444444, + "grad_norm": 0.5800482521740772, + "learning_rate": 9.281359022841966e-06, + "loss": 0.41333717107772827, + "step": 56 + }, + { + "epoch": 0.8106666666666666, + "grad_norm": 0.5896438771006596, + "learning_rate": 9.238302418967757e-06, + "loss": 0.41053399443626404, + "step": 57 + }, + { + "epoch": 0.8248888888888889, + "grad_norm": 0.5828843912613246, + "learning_rate": 9.194099206580981e-06, + "loss": 0.4167237877845764, + "step": 58 + }, + { + "epoch": 0.8391111111111111, + "grad_norm": 0.636505663941606, + "learning_rate": 9.14876134419111e-06, + "loss": 0.4126594066619873, + "step": 59 + }, + { + "epoch": 0.8533333333333334, + "grad_norm": 0.5954156377152778, + "learning_rate": 9.102301097269974e-06, + "loss": 0.40986335277557373, + "step": 60 + }, + { + "epoch": 0.8675555555555555, + "grad_norm": 0.5663259824764864, + "learning_rate": 9.05473103493355e-06, + "loss": 0.39707833528518677, + "step": 61 + }, + { + "epoch": 0.8817777777777778, + "grad_norm": 0.6233708113516392, + "learning_rate": 9.006064026541549e-06, + "loss": 0.4146142601966858, + "step": 62 + }, + { + "epoch": 0.896, + "grad_norm": 0.583882066947991, + "learning_rate": 8.956313238215824e-06, + "loss": 0.3761865794658661, + "step": 63 + }, + { + "epoch": 0.9102222222222223, + "grad_norm": 0.559228243852746, + "learning_rate": 8.905492129278478e-06, + "loss": 0.397402286529541, + "step": 64 + }, + { + "epoch": 0.9244444444444444, + "grad_norm": 0.5630501047235328, + "learning_rate": 8.85361444861063e-06, + "loss": 0.40299445390701294, + "step": 65 + }, + { + "epoch": 0.9386666666666666, + "grad_norm": 0.5778346041245924, + "learning_rate": 8.800694230932885e-06, + "loss": 0.4066828489303589, + "step": 66 + }, + { + "epoch": 0.9528888888888889, + "grad_norm": 0.5735220314699014, + "learning_rate": 8.74674579300843e-06, + "loss": 0.39874544739723206, + "step": 67 + }, + { + "epoch": 0.9671111111111111, + "grad_norm": 0.5774583972046927, + "learning_rate": 8.691783729769874e-06, + "loss": 0.38473668694496155, + "step": 68 + }, + { + "epoch": 0.9813333333333333, + "grad_norm": 0.6169263233631689, + "learning_rate": 8.635822910370793e-06, + "loss": 0.4083672761917114, + "step": 69 + }, + { + "epoch": 0.9955555555555555, + "grad_norm": 0.5815111675806847, + "learning_rate": 8.578878474163115e-06, + "loss": 0.3794967234134674, + "step": 70 + }, + { + "epoch": 1.0, + "grad_norm": 0.9420572347818393, + "learning_rate": 8.520965826601394e-06, + "loss": 0.40614083409309387, + "step": 71 + } + ], + "logging_steps": 1, + "max_steps": 213, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 20960610426880.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/qwen2.5-7b-instruct/checkpoint-71/training_args.bin b/qwen2.5-7b-instruct/checkpoint-71/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..e59d7187e91e3dd0138c38f45c246765301e22e6 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:238e1f7259942c35813600da4056c3d822147cad18b0d1147780f439481f9c35 +size 7377 diff --git a/qwen2.5-7b-instruct/checkpoint-71/zero_to_fp32.py b/qwen2.5-7b-instruct/checkpoint-71/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..5995d6e6f04e43b989587aa9022a3aef0c66d694 --- /dev/null +++ b/qwen2.5-7b-instruct/checkpoint-71/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/qwen2.5-7b-instruct/config.json b/qwen2.5-7b-instruct/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2948dc473025f5f3adb7c03ee0a5609588eb5c79 --- /dev/null +++ b/qwen2.5-7b-instruct/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 18944, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 28, + "model_type": "qwen2", + "num_attention_heads": 28, + "num_hidden_layers": 28, + "num_key_value_heads": 4, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.2.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 152064 +} diff --git a/qwen2.5-7b-instruct/eval_results.json b/qwen2.5-7b-instruct/eval_results.json new file mode 100644 index 0000000000000000000000000000000000000000..a7760208a5ef95f6d36d2309daec7f9dc317f860 --- /dev/null +++ b/qwen2.5-7b-instruct/eval_results.json @@ -0,0 +1,7 @@ +{ + "epoch": 3.0, + "eval_loss": 0.3998967707157135, + "eval_runtime": 27.0141, + "eval_samples_per_second": 7.404, + "eval_steps_per_second": 1.851 +} \ No newline at end of file diff --git a/qwen2.5-7b-instruct/generation_config.json b/qwen2.5-7b-instruct/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..47205fbf3386a215e7c2b9d62df6576f49b77e7b --- /dev/null +++ b/qwen2.5-7b-instruct/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.05, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.2.0" +} diff --git a/qwen2.5-7b-instruct/model.safetensors b/qwen2.5-7b-instruct/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ff8873127d880ec4b8a52babcc991396f57dd2fa --- /dev/null +++ b/qwen2.5-7b-instruct/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24c4947d6e26f390855ee3e85691f481c8392903f6cfa31be423754e4552afc8 +size 30462505080 diff --git a/qwen2.5-7b-instruct/tokenizer.json b/qwen2.5-7b-instruct/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..34510ff0037cd50428af467a17ead5a96140a32c --- /dev/null +++ b/qwen2.5-7b-instruct/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/qwen2.5-7b-instruct/tokenizer_config.json b/qwen2.5-7b-instruct/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..145e2c7a68b109ae0253dda29fc2e66cb01b4b85 --- /dev/null +++ b/qwen2.5-7b-instruct/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen2.5-7b-instruct/train_results.json b/qwen2.5-7b-instruct/train_results.json new file mode 100644 index 0000000000000000000000000000000000000000..723638f6fbee8dc813f819acbd91a9639384287b --- /dev/null +++ b/qwen2.5-7b-instruct/train_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 3.0, + "total_flos": 62893478699008.0, + "train_loss": 0.36721718521185326, + "train_runtime": 28478.4394, + "train_samples_per_second": 1.896, + "train_steps_per_second": 0.007 +} \ No newline at end of file diff --git a/qwen2.5-7b-instruct/trainer_log.jsonl b/qwen2.5-7b-instruct/trainer_log.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c69ce22b2dcfa2dc0477a17bd7c098384cab2985 --- /dev/null +++ b/qwen2.5-7b-instruct/trainer_log.jsonl @@ -0,0 +1,218 @@ +{"current_steps": 1, "total_steps": 213, "loss": 0.9819135069847107, "lr": 0.0, "epoch": 0.014222222222222223, "percentage": 0.47, "elapsed_time": "0:02:56", "remaining_time": "10:24:15"} +{"current_steps": 2, "total_steps": 213, "loss": 0.992728054523468, "lr": 4.5454545454545457e-07, "epoch": 0.028444444444444446, "percentage": 0.94, "elapsed_time": "0:05:45", "remaining_time": "10:07:50"} +{"current_steps": 3, "total_steps": 213, "loss": 0.88365638256073, "lr": 9.090909090909091e-07, "epoch": 0.042666666666666665, "percentage": 1.41, "elapsed_time": "0:08:37", "remaining_time": "10:03:30"} +{"current_steps": 4, "total_steps": 213, "loss": 0.8155229091644287, "lr": 1.3636363636363636e-06, "epoch": 0.05688888888888889, "percentage": 1.88, "elapsed_time": "0:10:47", "remaining_time": "9:24:08"} +{"current_steps": 5, "total_steps": 213, "loss": 0.7148971557617188, "lr": 1.8181818181818183e-06, "epoch": 0.07111111111111111, "percentage": 2.35, "elapsed_time": "0:12:56", "remaining_time": "8:58:38"} +{"current_steps": 6, "total_steps": 213, "loss": 0.7388776540756226, "lr": 2.2727272727272728e-06, "epoch": 0.08533333333333333, "percentage": 2.82, "elapsed_time": "0:15:03", "remaining_time": "8:39:31"} +{"current_steps": 7, "total_steps": 213, "loss": 0.6954053044319153, "lr": 2.7272727272727272e-06, "epoch": 0.09955555555555555, "percentage": 3.29, "elapsed_time": "0:17:08", "remaining_time": "8:24:23"} +{"current_steps": 8, "total_steps": 213, "loss": 0.6385300159454346, "lr": 3.181818181818182e-06, "epoch": 0.11377777777777778, "percentage": 3.76, "elapsed_time": "0:19:16", "remaining_time": "8:14:05"} +{"current_steps": 9, "total_steps": 213, "loss": 0.5974307060241699, "lr": 3.6363636363636366e-06, "epoch": 0.128, "percentage": 4.23, "elapsed_time": "0:21:25", "remaining_time": "8:05:36"} +{"current_steps": 10, "total_steps": 213, "loss": 0.5823870897293091, "lr": 4.0909090909090915e-06, "epoch": 0.14222222222222222, "percentage": 4.69, "elapsed_time": "0:23:34", "remaining_time": "7:58:40"} +{"current_steps": 11, "total_steps": 213, "loss": 0.5595569610595703, "lr": 4.5454545454545455e-06, "epoch": 0.15644444444444444, "percentage": 5.16, "elapsed_time": "0:25:44", "remaining_time": "7:52:48"} +{"current_steps": 12, "total_steps": 213, "loss": 0.5447146892547607, "lr": 5e-06, "epoch": 0.17066666666666666, "percentage": 5.63, "elapsed_time": "0:27:54", "remaining_time": "7:47:26"} +{"current_steps": 13, "total_steps": 213, "loss": 0.5179443359375, "lr": 5.4545454545454545e-06, "epoch": 0.18488888888888888, "percentage": 6.1, "elapsed_time": "0:30:02", "remaining_time": "7:42:05"} +{"current_steps": 14, "total_steps": 213, "loss": 0.5241127014160156, "lr": 5.90909090909091e-06, "epoch": 0.1991111111111111, "percentage": 6.57, "elapsed_time": "0:32:14", "remaining_time": "7:38:21"} +{"current_steps": 15, "total_steps": 213, "loss": 0.4842333197593689, "lr": 6.363636363636364e-06, "epoch": 0.21333333333333335, "percentage": 7.04, "elapsed_time": "0:34:21", "remaining_time": "7:33:31"} +{"current_steps": 16, "total_steps": 213, "loss": 0.5132038593292236, "lr": 6.818181818181818e-06, "epoch": 0.22755555555555557, "percentage": 7.51, "elapsed_time": "0:36:28", "remaining_time": "7:29:11"} +{"current_steps": 17, "total_steps": 213, "loss": 0.4917834401130676, "lr": 7.272727272727273e-06, "epoch": 0.24177777777777779, "percentage": 7.98, "elapsed_time": "0:38:36", "remaining_time": "7:25:05"} +{"current_steps": 18, "total_steps": 213, "loss": 0.4813205897808075, "lr": 7.727272727272727e-06, "epoch": 0.256, "percentage": 8.45, "elapsed_time": "0:40:42", "remaining_time": "7:21:02"} +{"current_steps": 19, "total_steps": 213, "loss": 0.4883510172367096, "lr": 8.181818181818183e-06, "epoch": 0.2702222222222222, "percentage": 8.92, "elapsed_time": "0:42:54", "remaining_time": "7:18:05"} +{"current_steps": 20, "total_steps": 213, "loss": 0.4871647357940674, "lr": 8.636363636363637e-06, "epoch": 0.28444444444444444, "percentage": 9.39, "elapsed_time": "0:45:04", "remaining_time": "7:14:55"} +{"current_steps": 21, "total_steps": 213, "loss": 0.46865272521972656, "lr": 9.090909090909091e-06, "epoch": 0.2986666666666667, "percentage": 9.86, "elapsed_time": "0:47:15", "remaining_time": "7:12:01"} +{"current_steps": 22, "total_steps": 213, "loss": 0.4601229727268219, "lr": 9.545454545454547e-06, "epoch": 0.3128888888888889, "percentage": 10.33, "elapsed_time": "0:49:24", "remaining_time": "7:08:56"} +{"current_steps": 23, "total_steps": 213, "loss": 0.45921051502227783, "lr": 1e-05, "epoch": 0.32711111111111113, "percentage": 10.8, "elapsed_time": "0:51:35", "remaining_time": "7:06:10"} +{"current_steps": 24, "total_steps": 213, "loss": 0.46318507194519043, "lr": 9.999323662872998e-06, "epoch": 0.3413333333333333, "percentage": 11.27, "elapsed_time": "0:53:41", "remaining_time": "7:02:52"} +{"current_steps": 25, "total_steps": 213, "loss": 0.4633931815624237, "lr": 9.99729483446475e-06, "epoch": 0.35555555555555557, "percentage": 11.74, "elapsed_time": "0:55:52", "remaining_time": "7:00:09"} +{"current_steps": 26, "total_steps": 213, "loss": 0.45212602615356445, "lr": 9.993914063644053e-06, "epoch": 0.36977777777777776, "percentage": 12.21, "elapsed_time": "0:57:58", "remaining_time": "6:56:57"} +{"current_steps": 27, "total_steps": 213, "loss": 0.44105663895606995, "lr": 9.989182265027232e-06, "epoch": 0.384, "percentage": 12.68, "elapsed_time": "1:00:08", "remaining_time": "6:54:18"} +{"current_steps": 28, "total_steps": 213, "loss": 0.4427238404750824, "lr": 9.98310071873072e-06, "epoch": 0.3982222222222222, "percentage": 13.15, "elapsed_time": "1:02:17", "remaining_time": "6:51:34"} +{"current_steps": 29, "total_steps": 213, "loss": 0.443437397480011, "lr": 9.975671070024741e-06, "epoch": 0.41244444444444445, "percentage": 13.62, "elapsed_time": "1:04:24", "remaining_time": "6:48:37"} +{"current_steps": 30, "total_steps": 213, "loss": 0.4632475674152374, "lr": 9.966895328888195e-06, "epoch": 0.4266666666666667, "percentage": 14.08, "elapsed_time": "1:06:34", "remaining_time": "6:46:04"} +{"current_steps": 31, "total_steps": 213, "loss": 0.4354594349861145, "lr": 9.956775869464901e-06, "epoch": 0.4408888888888889, "percentage": 14.55, "elapsed_time": "1:08:42", "remaining_time": "6:43:23"} +{"current_steps": 32, "total_steps": 213, "loss": 0.429914653301239, "lr": 9.945315429421307e-06, "epoch": 0.45511111111111113, "percentage": 15.02, "elapsed_time": "1:10:49", "remaining_time": "6:40:38"} +{"current_steps": 33, "total_steps": 213, "loss": 0.43607252836227417, "lr": 9.932517109205849e-06, "epoch": 0.4693333333333333, "percentage": 15.49, "elapsed_time": "1:12:57", "remaining_time": "6:37:56"} +{"current_steps": 34, "total_steps": 213, "loss": 0.45671695470809937, "lr": 9.918384371210178e-06, "epoch": 0.48355555555555557, "percentage": 15.96, "elapsed_time": "1:15:04", "remaining_time": "6:35:12"} +{"current_steps": 35, "total_steps": 213, "loss": 0.4388338327407837, "lr": 9.902921038832456e-06, "epoch": 0.49777777777777776, "percentage": 16.43, "elapsed_time": "1:17:11", "remaining_time": "6:32:36"} +{"current_steps": 36, "total_steps": 213, "loss": 0.4141705334186554, "lr": 9.886131295443003e-06, "epoch": 0.512, "percentage": 16.9, "elapsed_time": "1:19:20", "remaining_time": "6:30:07"} +{"current_steps": 37, "total_steps": 213, "loss": 0.4218006432056427, "lr": 9.868019683252543e-06, "epoch": 0.5262222222222223, "percentage": 17.37, "elapsed_time": "1:21:30", "remaining_time": "6:27:41"} +{"current_steps": 38, "total_steps": 213, "loss": 0.4457612931728363, "lr": 9.848591102083375e-06, "epoch": 0.5404444444444444, "percentage": 17.84, "elapsed_time": "1:23:37", "remaining_time": "6:25:06"} +{"current_steps": 39, "total_steps": 213, "loss": 0.4331759214401245, "lr": 9.82785080804381e-06, "epoch": 0.5546666666666666, "percentage": 18.31, "elapsed_time": "1:25:46", "remaining_time": "6:22:40"} +{"current_steps": 40, "total_steps": 213, "loss": 0.4507564902305603, "lr": 9.805804412106197e-06, "epoch": 0.5688888888888889, "percentage": 18.78, "elapsed_time": "1:27:51", "remaining_time": "6:20:01"} +{"current_steps": 41, "total_steps": 213, "loss": 0.4254251718521118, "lr": 9.782457878588977e-06, "epoch": 0.5831111111111111, "percentage": 19.25, "elapsed_time": "1:29:58", "remaining_time": "6:17:29"} +{"current_steps": 42, "total_steps": 213, "loss": 0.4125726819038391, "lr": 9.75781752354311e-06, "epoch": 0.5973333333333334, "percentage": 19.72, "elapsed_time": "1:32:07", "remaining_time": "6:15:06"} +{"current_steps": 43, "total_steps": 213, "loss": 0.4325122535228729, "lr": 9.731890013043367e-06, "epoch": 0.6115555555555555, "percentage": 20.19, "elapsed_time": "1:34:14", "remaining_time": "6:12:35"} +{"current_steps": 44, "total_steps": 213, "loss": 0.4131222367286682, "lr": 9.704682361384941e-06, "epoch": 0.6257777777777778, "percentage": 20.66, "elapsed_time": "1:36:25", "remaining_time": "6:10:21"} +{"current_steps": 45, "total_steps": 213, "loss": 0.42036178708076477, "lr": 9.676201929185809e-06, "epoch": 0.64, "percentage": 21.13, "elapsed_time": "1:38:31", "remaining_time": "6:07:49"} +{"current_steps": 46, "total_steps": 213, "loss": 0.4182787537574768, "lr": 9.646456421395447e-06, "epoch": 0.6542222222222223, "percentage": 21.6, "elapsed_time": "1:40:41", "remaining_time": "6:05:34"} +{"current_steps": 47, "total_steps": 213, "loss": 0.4234634041786194, "lr": 9.615453885210368e-06, "epoch": 0.6684444444444444, "percentage": 22.07, "elapsed_time": "1:42:50", "remaining_time": "6:03:12"} +{"current_steps": 48, "total_steps": 213, "loss": 0.40563416481018066, "lr": 9.583202707897075e-06, "epoch": 0.6826666666666666, "percentage": 22.54, "elapsed_time": "1:45:01", "remaining_time": "6:01:00"} +{"current_steps": 49, "total_steps": 213, "loss": 0.3980520963668823, "lr": 9.549711614523007e-06, "epoch": 0.6968888888888889, "percentage": 23.0, "elapsed_time": "1:47:07", "remaining_time": "5:58:31"} +{"current_steps": 50, "total_steps": 213, "loss": 0.4036375880241394, "lr": 9.514989665596114e-06, "epoch": 0.7111111111111111, "percentage": 23.47, "elapsed_time": "1:49:17", "remaining_time": "5:56:17"} +{"current_steps": 50, "total_steps": 213, "eval_loss": 0.41823333501815796, "epoch": 0.7111111111111111, "percentage": 23.47, "elapsed_time": "1:49:45", "remaining_time": "5:57:49"} +{"current_steps": 51, "total_steps": 213, "loss": 0.41144663095474243, "lr": 9.479046254613673e-06, "epoch": 0.7253333333333334, "percentage": 23.94, "elapsed_time": "1:51:54", "remaining_time": "5:55:29"} +{"current_steps": 52, "total_steps": 213, "loss": 0.38760095834732056, "lr": 9.441891105521005e-06, "epoch": 0.7395555555555555, "percentage": 24.41, "elapsed_time": "1:54:06", "remaining_time": "5:53:18"} +{"current_steps": 53, "total_steps": 213, "loss": 0.4036053717136383, "lr": 9.40353427008083e-06, "epoch": 0.7537777777777778, "percentage": 24.88, "elapsed_time": "1:56:16", "remaining_time": "5:51:00"} +{"current_steps": 54, "total_steps": 213, "loss": 0.39992809295654297, "lr": 9.3639861251539e-06, "epoch": 0.768, "percentage": 25.35, "elapsed_time": "1:58:26", "remaining_time": "5:48:45"} +{"current_steps": 55, "total_steps": 213, "loss": 0.420696496963501, "lr": 9.323257369891702e-06, "epoch": 0.7822222222222223, "percentage": 25.82, "elapsed_time": "2:00:34", "remaining_time": "5:46:23"} +{"current_steps": 56, "total_steps": 213, "loss": 0.41333717107772827, "lr": 9.281359022841966e-06, "epoch": 0.7964444444444444, "percentage": 26.29, "elapsed_time": "2:02:41", "remaining_time": "5:43:58"} +{"current_steps": 57, "total_steps": 213, "loss": 0.41053399443626404, "lr": 9.238302418967757e-06, "epoch": 0.8106666666666666, "percentage": 26.76, "elapsed_time": "2:04:48", "remaining_time": "5:41:35"} +{"current_steps": 58, "total_steps": 213, "loss": 0.4167237877845764, "lr": 9.194099206580981e-06, "epoch": 0.8248888888888889, "percentage": 27.23, "elapsed_time": "2:06:57", "remaining_time": "5:39:16"} +{"current_steps": 59, "total_steps": 213, "loss": 0.4126594066619873, "lr": 9.14876134419111e-06, "epoch": 0.8391111111111111, "percentage": 27.7, "elapsed_time": "2:09:04", "remaining_time": "5:36:54"} +{"current_steps": 60, "total_steps": 213, "loss": 0.40986335277557373, "lr": 9.102301097269974e-06, "epoch": 0.8533333333333334, "percentage": 28.17, "elapsed_time": "2:11:13", "remaining_time": "5:34:37"} +{"current_steps": 61, "total_steps": 213, "loss": 0.39707833528518677, "lr": 9.05473103493355e-06, "epoch": 0.8675555555555555, "percentage": 28.64, "elapsed_time": "2:13:21", "remaining_time": "5:32:18"} +{"current_steps": 62, "total_steps": 213, "loss": 0.4146142601966858, "lr": 9.006064026541549e-06, "epoch": 0.8817777777777778, "percentage": 29.11, "elapsed_time": "2:15:28", "remaining_time": "5:29:57"} +{"current_steps": 63, "total_steps": 213, "loss": 0.3761865794658661, "lr": 8.956313238215824e-06, "epoch": 0.896, "percentage": 29.58, "elapsed_time": "2:17:43", "remaining_time": "5:27:56"} +{"current_steps": 64, "total_steps": 213, "loss": 0.397402286529541, "lr": 8.905492129278478e-06, "epoch": 0.9102222222222223, "percentage": 30.05, "elapsed_time": "2:19:52", "remaining_time": "5:25:39"} +{"current_steps": 65, "total_steps": 213, "loss": 0.40299445390701294, "lr": 8.85361444861063e-06, "epoch": 0.9244444444444444, "percentage": 30.52, "elapsed_time": "2:22:06", "remaining_time": "5:23:34"} +{"current_steps": 66, "total_steps": 213, "loss": 0.4066828489303589, "lr": 8.800694230932885e-06, "epoch": 0.9386666666666666, "percentage": 30.99, "elapsed_time": "2:24:15", "remaining_time": "5:21:17"} +{"current_steps": 67, "total_steps": 213, "loss": 0.39874544739723206, "lr": 8.74674579300843e-06, "epoch": 0.9528888888888889, "percentage": 31.46, "elapsed_time": "2:26:25", "remaining_time": "5:19:04"} +{"current_steps": 68, "total_steps": 213, "loss": 0.38473668694496155, "lr": 8.691783729769874e-06, "epoch": 0.9671111111111111, "percentage": 31.92, "elapsed_time": "2:28:34", "remaining_time": "5:16:47"} +{"current_steps": 69, "total_steps": 213, "loss": 0.4083672761917114, "lr": 8.635822910370793e-06, "epoch": 0.9813333333333333, "percentage": 32.39, "elapsed_time": "2:30:41", "remaining_time": "5:14:29"} +{"current_steps": 70, "total_steps": 213, "loss": 0.3794967234134674, "lr": 8.578878474163115e-06, "epoch": 0.9955555555555555, "percentage": 32.86, "elapsed_time": "2:32:50", "remaining_time": "5:12:14"} +{"current_steps": 71, "total_steps": 213, "loss": 0.40614083409309387, "lr": 8.520965826601394e-06, "epoch": 1.0, "percentage": 33.33, "elapsed_time": "2:33:31", "remaining_time": "5:07:02"} +{"current_steps": 72, "total_steps": 213, "loss": 0.36958712339401245, "lr": 8.462100635075097e-06, "epoch": 1.0142222222222221, "percentage": 33.8, "elapsed_time": "2:41:20", "remaining_time": "5:15:57"} +{"current_steps": 73, "total_steps": 213, "loss": 0.3494463562965393, "lr": 8.40229882467003e-06, "epoch": 1.0284444444444445, "percentage": 34.27, "elapsed_time": "2:43:30", "remaining_time": "5:13:34"} +{"current_steps": 74, "total_steps": 213, "loss": 0.3302980065345764, "lr": 8.341576573860049e-06, "epoch": 1.0426666666666666, "percentage": 34.74, "elapsed_time": "2:45:42", "remaining_time": "5:11:15"} +{"current_steps": 75, "total_steps": 213, "loss": 0.35172200202941895, "lr": 8.279950310130218e-06, "epoch": 1.056888888888889, "percentage": 35.21, "elapsed_time": "2:47:50", "remaining_time": "5:08:50"} +{"current_steps": 76, "total_steps": 213, "loss": 0.3394201993942261, "lr": 8.2174367055326e-06, "epoch": 1.0711111111111111, "percentage": 35.68, "elapsed_time": "2:49:59", "remaining_time": "5:06:26"} +{"current_steps": 77, "total_steps": 213, "loss": 0.33312541246414185, "lr": 8.154052672175888e-06, "epoch": 1.0853333333333333, "percentage": 36.15, "elapsed_time": "2:52:08", "remaining_time": "5:04:03"} +{"current_steps": 78, "total_steps": 213, "loss": 0.35658037662506104, "lr": 8.08981535765009e-06, "epoch": 1.0995555555555556, "percentage": 36.62, "elapsed_time": "2:54:18", "remaining_time": "5:01:41"} +{"current_steps": 79, "total_steps": 213, "loss": 0.3449564278125763, "lr": 8.024742140387506e-06, "epoch": 1.1137777777777778, "percentage": 37.09, "elapsed_time": "2:56:24", "remaining_time": "4:59:13"} +{"current_steps": 80, "total_steps": 213, "loss": 0.3357444107532501, "lr": 7.95885062496126e-06, "epoch": 1.1280000000000001, "percentage": 37.56, "elapsed_time": "2:58:31", "remaining_time": "4:56:47"} +{"current_steps": 81, "total_steps": 213, "loss": 0.3170369267463684, "lr": 7.892158637322647e-06, "epoch": 1.1422222222222222, "percentage": 38.03, "elapsed_time": "3:00:39", "remaining_time": "4:54:23"} +{"current_steps": 82, "total_steps": 213, "loss": 0.3318628668785095, "lr": 7.824684219978591e-06, "epoch": 1.1564444444444444, "percentage": 38.5, "elapsed_time": "3:02:48", "remaining_time": "4:52:02"} +{"current_steps": 83, "total_steps": 213, "loss": 0.34082144498825073, "lr": 7.756445627110523e-06, "epoch": 1.1706666666666667, "percentage": 38.97, "elapsed_time": "3:05:00", "remaining_time": "4:49:46"} +{"current_steps": 84, "total_steps": 213, "loss": 0.3453400135040283, "lr": 7.68746131963598e-06, "epoch": 1.1848888888888889, "percentage": 39.44, "elapsed_time": "3:07:07", "remaining_time": "4:47:22"} +{"current_steps": 85, "total_steps": 213, "loss": 0.3420029580593109, "lr": 7.6177499602143e-06, "epoch": 1.199111111111111, "percentage": 39.91, "elapsed_time": "3:09:15", "remaining_time": "4:45:00"} +{"current_steps": 86, "total_steps": 213, "loss": 0.3330039381980896, "lr": 7.547330408197695e-06, "epoch": 1.2133333333333334, "percentage": 40.38, "elapsed_time": "3:11:26", "remaining_time": "4:42:42"} +{"current_steps": 87, "total_steps": 213, "loss": 0.3428732454776764, "lr": 7.476221714529167e-06, "epoch": 1.2275555555555555, "percentage": 40.85, "elapsed_time": "3:13:35", "remaining_time": "4:40:22"} +{"current_steps": 88, "total_steps": 213, "loss": 0.3575543761253357, "lr": 7.404443116588548e-06, "epoch": 1.2417777777777779, "percentage": 41.31, "elapsed_time": "3:15:42", "remaining_time": "4:37:59"} +{"current_steps": 89, "total_steps": 213, "loss": 0.3414611518383026, "lr": 7.332014032988123e-06, "epoch": 1.256, "percentage": 41.78, "elapsed_time": "3:17:48", "remaining_time": "4:35:36"} +{"current_steps": 90, "total_steps": 213, "loss": 0.3387766480445862, "lr": 7.2589540583192165e-06, "epoch": 1.2702222222222221, "percentage": 42.25, "elapsed_time": "3:19:57", "remaining_time": "4:33:16"} +{"current_steps": 91, "total_steps": 213, "loss": 0.32765448093414307, "lr": 7.185282957851175e-06, "epoch": 1.2844444444444445, "percentage": 42.72, "elapsed_time": "3:22:07", "remaining_time": "4:30:59"} +{"current_steps": 92, "total_steps": 213, "loss": 0.3432917296886444, "lr": 7.111020662184174e-06, "epoch": 1.2986666666666666, "percentage": 43.19, "elapsed_time": "3:24:17", "remaining_time": "4:28:41"} +{"current_steps": 93, "total_steps": 213, "loss": 0.3334134817123413, "lr": 7.036187261857289e-06, "epoch": 1.3128888888888888, "percentage": 43.66, "elapsed_time": "3:26:25", "remaining_time": "4:26:21"} +{"current_steps": 94, "total_steps": 213, "loss": 0.3528696894645691, "lr": 6.960803001913315e-06, "epoch": 1.3271111111111111, "percentage": 44.13, "elapsed_time": "3:28:31", "remaining_time": "4:23:59"} +{"current_steps": 95, "total_steps": 213, "loss": 0.3428453207015991, "lr": 6.884888276421766e-06, "epoch": 1.3413333333333333, "percentage": 44.6, "elapsed_time": "3:30:42", "remaining_time": "4:21:42"} +{"current_steps": 96, "total_steps": 213, "loss": 0.33261382579803467, "lr": 6.8084636229615786e-06, "epoch": 1.3555555555555556, "percentage": 45.07, "elapsed_time": "3:32:50", "remaining_time": "4:19:24"} +{"current_steps": 97, "total_steps": 213, "loss": 0.33263495564460754, "lr": 6.731549717064975e-06, "epoch": 1.3697777777777778, "percentage": 45.54, "elapsed_time": "3:35:00", "remaining_time": "4:17:06"} +{"current_steps": 98, "total_steps": 213, "loss": 0.3409455418586731, "lr": 6.654167366624009e-06, "epoch": 1.384, "percentage": 46.01, "elapsed_time": "3:37:06", "remaining_time": "4:14:46"} +{"current_steps": 99, "total_steps": 213, "loss": 0.34175097942352295, "lr": 6.576337506261314e-06, "epoch": 1.3982222222222223, "percentage": 46.48, "elapsed_time": "3:39:14", "remaining_time": "4:12:27"} +{"current_steps": 100, "total_steps": 213, "loss": 0.3422796130180359, "lr": 6.498081191666549e-06, "epoch": 1.4124444444444444, "percentage": 46.95, "elapsed_time": "3:41:24", "remaining_time": "4:10:11"} +{"current_steps": 100, "total_steps": 213, "eval_loss": 0.39906859397888184, "epoch": 1.4124444444444444, "percentage": 46.95, "elapsed_time": "3:41:52", "remaining_time": "4:10:42"} +{"current_steps": 101, "total_steps": 213, "loss": 0.3311368227005005, "lr": 6.419419593900109e-06, "epoch": 1.4266666666666667, "percentage": 47.42, "elapsed_time": "3:43:59", "remaining_time": "4:08:22"} +{"current_steps": 102, "total_steps": 213, "loss": 0.32347068190574646, "lr": 6.340373993665607e-06, "epoch": 1.4408888888888889, "percentage": 47.89, "elapsed_time": "3:46:07", "remaining_time": "4:06:04"} +{"current_steps": 103, "total_steps": 213, "loss": 0.32194703817367554, "lr": 6.2609657755527135e-06, "epoch": 1.455111111111111, "percentage": 48.36, "elapsed_time": "3:48:16", "remaining_time": "4:03:46"} +{"current_steps": 104, "total_steps": 213, "loss": 0.3477523922920227, "lr": 6.1812164222518626e-06, "epoch": 1.4693333333333334, "percentage": 48.83, "elapsed_time": "3:50:23", "remaining_time": "4:01:28"} +{"current_steps": 105, "total_steps": 213, "loss": 0.33583635091781616, "lr": 6.101147508742456e-06, "epoch": 1.4835555555555555, "percentage": 49.3, "elapsed_time": "3:52:30", "remaining_time": "3:59:09"} +{"current_steps": 106, "total_steps": 213, "loss": 0.3305099904537201, "lr": 6.020780696456059e-06, "epoch": 1.4977777777777779, "percentage": 49.77, "elapsed_time": "3:54:42", "remaining_time": "3:56:55"} +{"current_steps": 107, "total_steps": 213, "loss": 0.3392476439476013, "lr": 5.940137727416247e-06, "epoch": 1.512, "percentage": 50.23, "elapsed_time": "3:56:53", "remaining_time": "3:54:40"} +{"current_steps": 108, "total_steps": 213, "loss": 0.3281456232070923, "lr": 5.859240418356614e-06, "epoch": 1.5262222222222221, "percentage": 50.7, "elapsed_time": "3:59:04", "remaining_time": "3:52:25"} +{"current_steps": 109, "total_steps": 213, "loss": 0.3557584285736084, "lr": 5.778110654818602e-06, "epoch": 1.5404444444444443, "percentage": 51.17, "elapsed_time": "4:01:10", "remaining_time": "3:50:06"} +{"current_steps": 110, "total_steps": 213, "loss": 0.3375227153301239, "lr": 5.696770385230679e-06, "epoch": 1.5546666666666666, "percentage": 51.64, "elapsed_time": "4:03:20", "remaining_time": "3:47:51"} +{"current_steps": 111, "total_steps": 213, "loss": 0.33440524339675903, "lr": 5.615241614970546e-06, "epoch": 1.568888888888889, "percentage": 52.11, "elapsed_time": "4:05:28", "remaining_time": "3:45:34"} +{"current_steps": 112, "total_steps": 213, "loss": 0.3354341983795166, "lr": 5.533546400411899e-06, "epoch": 1.5831111111111111, "percentage": 52.58, "elapsed_time": "4:07:38", "remaining_time": "3:43:19"} +{"current_steps": 113, "total_steps": 213, "loss": 0.34512901306152344, "lr": 5.4517068429574215e-06, "epoch": 1.5973333333333333, "percentage": 53.05, "elapsed_time": "4:09:49", "remaining_time": "3:41:04"} +{"current_steps": 114, "total_steps": 213, "loss": 0.3282182812690735, "lr": 5.3697450830595775e-06, "epoch": 1.6115555555555554, "percentage": 53.52, "elapsed_time": "4:12:00", "remaining_time": "3:38:50"} +{"current_steps": 115, "total_steps": 213, "loss": 0.3325689136981964, "lr": 5.287683294230855e-06, "epoch": 1.6257777777777778, "percentage": 53.99, "elapsed_time": "4:14:06", "remaining_time": "3:36:32"} +{"current_steps": 116, "total_steps": 213, "loss": 0.3167092204093933, "lr": 5.20554367704505e-06, "epoch": 1.6400000000000001, "percentage": 54.46, "elapsed_time": "4:16:17", "remaining_time": "3:34:18"} +{"current_steps": 117, "total_steps": 213, "loss": 0.3419051766395569, "lr": 5.123348453131242e-06, "epoch": 1.6542222222222223, "percentage": 54.93, "elapsed_time": "4:18:25", "remaining_time": "3:32:02"} +{"current_steps": 118, "total_steps": 213, "loss": 0.333318293094635, "lr": 5.041119859162068e-06, "epoch": 1.6684444444444444, "percentage": 55.4, "elapsed_time": "4:20:35", "remaining_time": "3:29:48"} +{"current_steps": 119, "total_steps": 213, "loss": 0.33905792236328125, "lr": 4.958880140837934e-06, "epoch": 1.6826666666666665, "percentage": 55.87, "elapsed_time": "4:22:43", "remaining_time": "3:27:31"} +{"current_steps": 120, "total_steps": 213, "loss": 0.32569897174835205, "lr": 4.876651546868759e-06, "epoch": 1.696888888888889, "percentage": 56.34, "elapsed_time": "4:24:53", "remaining_time": "3:25:17"} +{"current_steps": 121, "total_steps": 213, "loss": 0.3532940745353699, "lr": 4.794456322954953e-06, "epoch": 1.7111111111111112, "percentage": 56.81, "elapsed_time": "4:27:03", "remaining_time": "3:23:02"} +{"current_steps": 122, "total_steps": 213, "loss": 0.35066288709640503, "lr": 4.7123167057691446e-06, "epoch": 1.7253333333333334, "percentage": 57.28, "elapsed_time": "4:29:12", "remaining_time": "3:20:48"} +{"current_steps": 123, "total_steps": 213, "loss": 0.35048598051071167, "lr": 4.630254916940424e-06, "epoch": 1.7395555555555555, "percentage": 57.75, "elapsed_time": "4:31:24", "remaining_time": "3:18:35"} +{"current_steps": 124, "total_steps": 213, "loss": 0.3389701843261719, "lr": 4.548293157042581e-06, "epoch": 1.7537777777777777, "percentage": 58.22, "elapsed_time": "4:33:35", "remaining_time": "3:16:22"} +{"current_steps": 125, "total_steps": 213, "loss": 0.3477008044719696, "lr": 4.466453599588103e-06, "epoch": 1.768, "percentage": 58.69, "elapsed_time": "4:35:44", "remaining_time": "3:14:07"} +{"current_steps": 126, "total_steps": 213, "loss": 0.3188989758491516, "lr": 4.384758385029457e-06, "epoch": 1.7822222222222224, "percentage": 59.15, "elapsed_time": "4:37:53", "remaining_time": "3:11:52"} +{"current_steps": 127, "total_steps": 213, "loss": 0.3296913504600525, "lr": 4.3032296147693225e-06, "epoch": 1.7964444444444445, "percentage": 59.62, "elapsed_time": "4:40:00", "remaining_time": "3:09:36"} +{"current_steps": 128, "total_steps": 213, "loss": 0.3278958797454834, "lr": 4.2218893451814005e-06, "epoch": 1.8106666666666666, "percentage": 60.09, "elapsed_time": "4:42:11", "remaining_time": "3:07:23"} +{"current_steps": 129, "total_steps": 213, "loss": 0.3338126838207245, "lr": 4.140759581643386e-06, "epoch": 1.8248888888888888, "percentage": 60.56, "elapsed_time": "4:44:23", "remaining_time": "3:05:10"} +{"current_steps": 130, "total_steps": 213, "loss": 0.33511388301849365, "lr": 4.059862272583755e-06, "epoch": 1.8391111111111111, "percentage": 61.03, "elapsed_time": "4:46:31", "remaining_time": "3:02:56"} +{"current_steps": 131, "total_steps": 213, "loss": 0.3190721869468689, "lr": 3.979219303543942e-06, "epoch": 1.8533333333333335, "percentage": 61.5, "elapsed_time": "4:48:43", "remaining_time": "3:00:43"} +{"current_steps": 132, "total_steps": 213, "loss": 0.3397029638290405, "lr": 3.898852491257547e-06, "epoch": 1.8675555555555556, "percentage": 61.97, "elapsed_time": "4:50:53", "remaining_time": "2:58:29"} +{"current_steps": 133, "total_steps": 213, "loss": 0.3368285894393921, "lr": 3.818783577748138e-06, "epoch": 1.8817777777777778, "percentage": 62.44, "elapsed_time": "4:53:04", "remaining_time": "2:56:16"} +{"current_steps": 134, "total_steps": 213, "loss": 0.34375038743019104, "lr": 3.7390342244472886e-06, "epoch": 1.896, "percentage": 62.91, "elapsed_time": "4:55:09", "remaining_time": "2:54:00"} +{"current_steps": 135, "total_steps": 213, "loss": 0.32158786058425903, "lr": 3.659626006334395e-06, "epoch": 1.9102222222222223, "percentage": 63.38, "elapsed_time": "4:57:19", "remaining_time": "2:51:47"} +{"current_steps": 136, "total_steps": 213, "loss": 0.32734596729278564, "lr": 3.5805804060998926e-06, "epoch": 1.9244444444444444, "percentage": 63.85, "elapsed_time": "4:59:26", "remaining_time": "2:49:31"} +{"current_steps": 137, "total_steps": 213, "loss": 0.3495728671550751, "lr": 3.501918808333453e-06, "epoch": 1.9386666666666668, "percentage": 64.32, "elapsed_time": "5:01:35", "remaining_time": "2:47:18"} +{"current_steps": 138, "total_steps": 213, "loss": 0.3545406758785248, "lr": 3.4236624937386874e-06, "epoch": 1.952888888888889, "percentage": 64.79, "elapsed_time": "5:03:43", "remaining_time": "2:45:03"} +{"current_steps": 139, "total_steps": 213, "loss": 0.3283476233482361, "lr": 3.3458326333759927e-06, "epoch": 1.967111111111111, "percentage": 65.26, "elapsed_time": "5:05:54", "remaining_time": "2:42:51"} +{"current_steps": 140, "total_steps": 213, "loss": 0.3362443447113037, "lr": 3.268450282935026e-06, "epoch": 1.9813333333333332, "percentage": 65.73, "elapsed_time": "5:08:01", "remaining_time": "2:40:36"} +{"current_steps": 141, "total_steps": 213, "loss": 0.33009064197540283, "lr": 3.1915363770384223e-06, "epoch": 1.9955555555555555, "percentage": 66.2, "elapsed_time": "5:10:13", "remaining_time": "2:38:24"} +{"current_steps": 142, "total_steps": 213, "loss": 0.34658753871917725, "lr": 3.1151117235782346e-06, "epoch": 2.0, "percentage": 66.67, "elapsed_time": "5:10:56", "remaining_time": "2:35:28"} +{"current_steps": 143, "total_steps": 213, "loss": 0.2953590750694275, "lr": 3.0391969980866874e-06, "epoch": 2.014222222222222, "percentage": 67.14, "elapsed_time": "5:18:29", "remaining_time": "2:35:54"} +{"current_steps": 144, "total_steps": 213, "loss": 0.28190308809280396, "lr": 2.963812738142713e-06, "epoch": 2.0284444444444443, "percentage": 67.61, "elapsed_time": "5:20:40", "remaining_time": "2:33:39"} +{"current_steps": 145, "total_steps": 213, "loss": 0.29579877853393555, "lr": 2.8889793378158284e-06, "epoch": 2.042666666666667, "percentage": 68.08, "elapsed_time": "5:22:49", "remaining_time": "2:31:23"} +{"current_steps": 146, "total_steps": 213, "loss": 0.2849843502044678, "lr": 2.814717042148827e-06, "epoch": 2.056888888888889, "percentage": 68.54, "elapsed_time": "5:24:59", "remaining_time": "2:29:08"} +{"current_steps": 147, "total_steps": 213, "loss": 0.2797221541404724, "lr": 2.7410459416807856e-06, "epoch": 2.071111111111111, "percentage": 69.01, "elapsed_time": "5:27:10", "remaining_time": "2:26:53"} +{"current_steps": 148, "total_steps": 213, "loss": 0.2898455262184143, "lr": 2.6679859670118785e-06, "epoch": 2.0853333333333333, "percentage": 69.48, "elapsed_time": "5:29:20", "remaining_time": "2:24:38"} +{"current_steps": 149, "total_steps": 213, "loss": 0.27163612842559814, "lr": 2.5955568834114523e-06, "epoch": 2.0995555555555554, "percentage": 69.95, "elapsed_time": "5:31:28", "remaining_time": "2:22:22"} +{"current_steps": 150, "total_steps": 213, "loss": 0.28451889753341675, "lr": 2.523778285470835e-06, "epoch": 2.113777777777778, "percentage": 70.42, "elapsed_time": "5:33:37", "remaining_time": "2:20:07"} +{"current_steps": 150, "total_steps": 213, "eval_loss": 0.4041806757450104, "epoch": 2.113777777777778, "percentage": 70.42, "elapsed_time": "5:34:05", "remaining_time": "2:20:19"} +{"current_steps": 151, "total_steps": 213, "loss": 0.2696727514266968, "lr": 2.452669591802307e-06, "epoch": 2.128, "percentage": 70.89, "elapsed_time": "5:36:11", "remaining_time": "2:18:02"} +{"current_steps": 152, "total_steps": 213, "loss": 0.2786106467247009, "lr": 2.3822500397857016e-06, "epoch": 2.1422222222222222, "percentage": 71.36, "elapsed_time": "5:38:21", "remaining_time": "2:15:47"} +{"current_steps": 153, "total_steps": 213, "loss": 0.29131805896759033, "lr": 2.3125386803640188e-06, "epoch": 2.1564444444444444, "percentage": 71.83, "elapsed_time": "5:40:32", "remaining_time": "2:13:32"} +{"current_steps": 154, "total_steps": 213, "loss": 0.2755666971206665, "lr": 2.243554372889479e-06, "epoch": 2.1706666666666665, "percentage": 72.3, "elapsed_time": "5:42:44", "remaining_time": "2:11:18"} +{"current_steps": 155, "total_steps": 213, "loss": 0.27331122756004333, "lr": 2.1753157800214107e-06, "epoch": 2.1848888888888887, "percentage": 72.77, "elapsed_time": "5:44:54", "remaining_time": "2:09:03"} +{"current_steps": 156, "total_steps": 213, "loss": 0.28715455532073975, "lr": 2.1078413626773547e-06, "epoch": 2.1991111111111112, "percentage": 73.24, "elapsed_time": "5:47:03", "remaining_time": "2:06:48"} +{"current_steps": 157, "total_steps": 213, "loss": 0.26138654351234436, "lr": 2.0411493750387423e-06, "epoch": 2.2133333333333334, "percentage": 73.71, "elapsed_time": "5:49:12", "remaining_time": "2:04:33"} +{"current_steps": 158, "total_steps": 213, "loss": 0.29015523195266724, "lr": 1.9752578596124955e-06, "epoch": 2.2275555555555555, "percentage": 74.18, "elapsed_time": "5:51:21", "remaining_time": "2:02:18"} +{"current_steps": 159, "total_steps": 213, "loss": 0.2733340561389923, "lr": 1.9101846423499113e-06, "epoch": 2.2417777777777776, "percentage": 74.65, "elapsed_time": "5:53:29", "remaining_time": "2:00:03"} +{"current_steps": 160, "total_steps": 213, "loss": 0.2742033302783966, "lr": 1.8459473278241125e-06, "epoch": 2.2560000000000002, "percentage": 75.12, "elapsed_time": "5:55:37", "remaining_time": "1:57:48"} +{"current_steps": 161, "total_steps": 213, "loss": 0.2752198874950409, "lr": 1.7825632944674016e-06, "epoch": 2.2702222222222224, "percentage": 75.59, "elapsed_time": "5:57:48", "remaining_time": "1:55:33"} +{"current_steps": 162, "total_steps": 213, "loss": 0.28458085656166077, "lr": 1.7200496898697832e-06, "epoch": 2.2844444444444445, "percentage": 76.06, "elapsed_time": "5:59:55", "remaining_time": "1:53:18"} +{"current_steps": 163, "total_steps": 213, "loss": 0.277709424495697, "lr": 1.6584234261399535e-06, "epoch": 2.2986666666666666, "percentage": 76.53, "elapsed_time": "6:02:07", "remaining_time": "1:51:04"} +{"current_steps": 164, "total_steps": 213, "loss": 0.2714066505432129, "lr": 1.5977011753299726e-06, "epoch": 2.3128888888888888, "percentage": 77.0, "elapsed_time": "6:04:15", "remaining_time": "1:48:49"} +{"current_steps": 165, "total_steps": 213, "loss": 0.2929142117500305, "lr": 1.5378993649249053e-06, "epoch": 2.327111111111111, "percentage": 77.46, "elapsed_time": "6:06:22", "remaining_time": "1:46:34"} +{"current_steps": 166, "total_steps": 213, "loss": 0.274400919675827, "lr": 1.4790341733986085e-06, "epoch": 2.3413333333333335, "percentage": 77.93, "elapsed_time": "6:08:32", "remaining_time": "1:44:20"} +{"current_steps": 167, "total_steps": 213, "loss": 0.2633841633796692, "lr": 1.4211215258368866e-06, "epoch": 2.3555555555555556, "percentage": 78.4, "elapsed_time": "6:10:42", "remaining_time": "1:42:06"} +{"current_steps": 168, "total_steps": 213, "loss": 0.271608829498291, "lr": 1.3641770896292083e-06, "epoch": 2.3697777777777778, "percentage": 78.87, "elapsed_time": "6:12:52", "remaining_time": "1:39:52"} +{"current_steps": 169, "total_steps": 213, "loss": 0.2772142291069031, "lr": 1.3082162702301276e-06, "epoch": 2.384, "percentage": 79.34, "elapsed_time": "6:15:02", "remaining_time": "1:37:38"} +{"current_steps": 170, "total_steps": 213, "loss": 0.2833596467971802, "lr": 1.2532542069915722e-06, "epoch": 2.398222222222222, "percentage": 79.81, "elapsed_time": "6:17:10", "remaining_time": "1:35:24"} +{"current_steps": 171, "total_steps": 213, "loss": 0.27554380893707275, "lr": 1.1993057690671174e-06, "epoch": 2.4124444444444446, "percentage": 80.28, "elapsed_time": "6:19:19", "remaining_time": "1:33:10"} +{"current_steps": 172, "total_steps": 213, "loss": 0.291002094745636, "lr": 1.1463855513893695e-06, "epoch": 2.4266666666666667, "percentage": 80.75, "elapsed_time": "6:21:30", "remaining_time": "1:30:56"} +{"current_steps": 173, "total_steps": 213, "loss": 0.2779809832572937, "lr": 1.0945078707215224e-06, "epoch": 2.440888888888889, "percentage": 81.22, "elapsed_time": "6:23:38", "remaining_time": "1:28:42"} +{"current_steps": 174, "total_steps": 213, "loss": 0.2763213515281677, "lr": 1.0436867617841768e-06, "epoch": 2.455111111111111, "percentage": 81.69, "elapsed_time": "6:25:44", "remaining_time": "1:26:27"} +{"current_steps": 175, "total_steps": 213, "loss": 0.28753769397735596, "lr": 9.939359734584552e-07, "epoch": 2.469333333333333, "percentage": 82.16, "elapsed_time": "6:27:54", "remaining_time": "1:24:13"} +{"current_steps": 176, "total_steps": 213, "loss": 0.2837107181549072, "lr": 9.452689650664515e-07, "epoch": 2.4835555555555557, "percentage": 82.63, "elapsed_time": "6:30:05", "remaining_time": "1:22:00"} +{"current_steps": 177, "total_steps": 213, "loss": 0.2816150188446045, "lr": 8.976989027300265e-07, "epoch": 2.497777777777778, "percentage": 83.1, "elapsed_time": "6:32:16", "remaining_time": "1:19:47"} +{"current_steps": 178, "total_steps": 213, "loss": 0.2889126241207123, "lr": 8.512386558088919e-07, "epoch": 2.512, "percentage": 83.57, "elapsed_time": "6:34:25", "remaining_time": "1:17:33"} +{"current_steps": 179, "total_steps": 213, "loss": 0.2788556218147278, "lr": 8.059007934190194e-07, "epoch": 2.526222222222222, "percentage": 84.04, "elapsed_time": "6:36:35", "remaining_time": "1:15:19"} +{"current_steps": 180, "total_steps": 213, "loss": 0.29226237535476685, "lr": 7.61697581032243e-07, "epoch": 2.5404444444444443, "percentage": 84.51, "elapsed_time": "6:38:42", "remaining_time": "1:13:05"} +{"current_steps": 181, "total_steps": 213, "loss": 0.2915460765361786, "lr": 7.186409771580355e-07, "epoch": 2.554666666666667, "percentage": 84.98, "elapsed_time": "6:40:52", "remaining_time": "1:10:52"} +{"current_steps": 182, "total_steps": 213, "loss": 0.2790933847427368, "lr": 6.76742630108298e-07, "epoch": 2.568888888888889, "percentage": 85.45, "elapsed_time": "6:43:03", "remaining_time": "1:08:39"} +{"current_steps": 183, "total_steps": 213, "loss": 0.27808326482772827, "lr": 6.360138748461015e-07, "epoch": 2.583111111111111, "percentage": 85.92, "elapsed_time": "6:45:12", "remaining_time": "1:06:25"} +{"current_steps": 184, "total_steps": 213, "loss": 0.2690117359161377, "lr": 5.964657299191712e-07, "epoch": 2.5973333333333333, "percentage": 86.38, "elapsed_time": "6:47:22", "remaining_time": "1:04:12"} +{"current_steps": 185, "total_steps": 213, "loss": 0.27599871158599854, "lr": 5.581088944789953e-07, "epoch": 2.6115555555555554, "percentage": 86.85, "elapsed_time": "6:49:32", "remaining_time": "1:01:59"} +{"current_steps": 186, "total_steps": 213, "loss": 0.2919716238975525, "lr": 5.20953745386329e-07, "epoch": 2.6257777777777775, "percentage": 87.32, "elapsed_time": "6:51:41", "remaining_time": "0:59:45"} +{"current_steps": 187, "total_steps": 213, "loss": 0.28502631187438965, "lr": 4.850103344038853e-07, "epoch": 2.64, "percentage": 87.79, "elapsed_time": "6:53:48", "remaining_time": "0:57:32"} +{"current_steps": 188, "total_steps": 213, "loss": 0.26374831795692444, "lr": 4.502883854769935e-07, "epoch": 2.6542222222222223, "percentage": 88.26, "elapsed_time": "6:55:57", "remaining_time": "0:55:18"} +{"current_steps": 189, "total_steps": 213, "loss": 0.28197193145751953, "lr": 4.167972921029262e-07, "epoch": 2.6684444444444444, "percentage": 88.73, "elapsed_time": "6:58:05", "remaining_time": "0:53:05"} +{"current_steps": 190, "total_steps": 213, "loss": 0.2938516139984131, "lr": 3.8454611478963235e-07, "epoch": 2.6826666666666665, "percentage": 89.2, "elapsed_time": "7:00:13", "remaining_time": "0:50:52"} +{"current_steps": 191, "total_steps": 213, "loss": 0.27452725172042847, "lr": 3.535435786045538e-07, "epoch": 2.696888888888889, "percentage": 89.67, "elapsed_time": "7:02:20", "remaining_time": "0:48:38"} +{"current_steps": 192, "total_steps": 213, "loss": 0.2790898084640503, "lr": 3.237980708141919e-07, "epoch": 2.7111111111111112, "percentage": 90.14, "elapsed_time": "7:04:28", "remaining_time": "0:46:25"} +{"current_steps": 193, "total_steps": 213, "loss": 0.2802097201347351, "lr": 2.9531763861505967e-07, "epoch": 2.7253333333333334, "percentage": 90.61, "elapsed_time": "7:06:39", "remaining_time": "0:44:12"} +{"current_steps": 194, "total_steps": 213, "loss": 0.2703718841075897, "lr": 2.681099869566328e-07, "epoch": 2.7395555555555555, "percentage": 91.08, "elapsed_time": "7:08:47", "remaining_time": "0:41:59"} +{"current_steps": 195, "total_steps": 213, "loss": 0.2852793335914612, "lr": 2.4218247645689306e-07, "epoch": 2.7537777777777777, "percentage": 91.55, "elapsed_time": "7:10:57", "remaining_time": "0:39:46"} +{"current_steps": 196, "total_steps": 213, "loss": 0.27227064967155457, "lr": 2.175421214110235e-07, "epoch": 2.768, "percentage": 92.02, "elapsed_time": "7:13:05", "remaining_time": "0:37:33"} +{"current_steps": 197, "total_steps": 213, "loss": 0.28222954273223877, "lr": 1.941955878938029e-07, "epoch": 2.7822222222222224, "percentage": 92.49, "elapsed_time": "7:15:14", "remaining_time": "0:35:20"} +{"current_steps": 198, "total_steps": 213, "loss": 0.2887798249721527, "lr": 1.721491919561913e-07, "epoch": 2.7964444444444445, "percentage": 92.96, "elapsed_time": "7:17:24", "remaining_time": "0:33:08"} +{"current_steps": 199, "total_steps": 213, "loss": 0.2730322480201721, "lr": 1.514088979166256e-07, "epoch": 2.8106666666666666, "percentage": 93.43, "elapsed_time": "7:19:34", "remaining_time": "0:30:55"} +{"current_steps": 200, "total_steps": 213, "loss": 0.28655844926834106, "lr": 1.3198031674745814e-07, "epoch": 2.824888888888889, "percentage": 93.9, "elapsed_time": "7:21:43", "remaining_time": "0:28:42"} +{"current_steps": 200, "total_steps": 213, "eval_loss": 0.3998975455760956, "epoch": 2.824888888888889, "percentage": 93.9, "elapsed_time": "7:22:11", "remaining_time": "0:28:44"} +{"current_steps": 201, "total_steps": 213, "loss": 0.28616365790367126, "lr": 1.138687045569975e-07, "epoch": 2.8391111111111114, "percentage": 94.37, "elapsed_time": "7:24:22", "remaining_time": "0:26:31"} +{"current_steps": 202, "total_steps": 213, "loss": 0.28517472743988037, "lr": 9.70789611675449e-08, "epoch": 2.8533333333333335, "percentage": 94.84, "elapsed_time": "7:26:30", "remaining_time": "0:24:18"} +{"current_steps": 203, "total_steps": 213, "loss": 0.27302050590515137, "lr": 8.161562878982399e-08, "epoch": 2.8675555555555556, "percentage": 95.31, "elapsed_time": "7:28:38", "remaining_time": "0:22:06"} +{"current_steps": 204, "total_steps": 213, "loss": 0.28563594818115234, "lr": 6.74828907941516e-08, "epoch": 2.8817777777777778, "percentage": 95.77, "elapsed_time": "7:30:45", "remaining_time": "0:19:53"} +{"current_steps": 205, "total_steps": 213, "loss": 0.2887130379676819, "lr": 5.468457057869358e-08, "epoch": 2.896, "percentage": 96.24, "elapsed_time": "7:32:56", "remaining_time": "0:17:40"} +{"current_steps": 206, "total_steps": 213, "loss": 0.2721640467643738, "lr": 4.322413053509944e-08, "epoch": 2.910222222222222, "percentage": 96.71, "elapsed_time": "7:35:08", "remaining_time": "0:15:27"} +{"current_steps": 207, "total_steps": 213, "loss": 0.27188998460769653, "lr": 3.31046711118066e-08, "epoch": 2.924444444444444, "percentage": 97.18, "elapsed_time": "7:37:16", "remaining_time": "0:13:15"} +{"current_steps": 208, "total_steps": 213, "loss": 0.2795502245426178, "lr": 2.4328929975260262e-08, "epoch": 2.9386666666666668, "percentage": 97.65, "elapsed_time": "7:39:27", "remaining_time": "0:11:02"} +{"current_steps": 209, "total_steps": 213, "loss": 0.2727120816707611, "lr": 1.6899281269279756e-08, "epoch": 2.952888888888889, "percentage": 98.12, "elapsed_time": "7:41:35", "remaining_time": "0:08:50"} +{"current_steps": 210, "total_steps": 213, "loss": 0.2795644998550415, "lr": 1.0817734972768946e-08, "epoch": 2.967111111111111, "percentage": 98.59, "elapsed_time": "7:43:42", "remaining_time": "0:06:37"} +{"current_steps": 211, "total_steps": 213, "loss": 0.28946471214294434, "lr": 6.085936355947897e-09, "epoch": 2.981333333333333, "percentage": 99.06, "elapsed_time": "7:45:50", "remaining_time": "0:04:24"} +{"current_steps": 212, "total_steps": 213, "loss": 0.26888906955718994, "lr": 2.7051655352494654e-09, "epoch": 2.9955555555555557, "percentage": 99.53, "elapsed_time": "7:48:00", "remaining_time": "0:02:12"} +{"current_steps": 213, "total_steps": 213, "loss": 0.33630549907684326, "lr": 6.763371270035457e-10, "epoch": 3.0, "percentage": 100.0, "elapsed_time": "7:48:40", "remaining_time": "0:00:00"} +{"current_steps": 213, "total_steps": 213, "epoch": 3.0, "percentage": 100.0, "elapsed_time": "7:54:35", "remaining_time": "0:00:00"} diff --git a/qwen2.5-7b-instruct/trainer_state.json b/qwen2.5-7b-instruct/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..af27f207240511151509e98329cb88e7bf2c5557 --- /dev/null +++ b/qwen2.5-7b-instruct/trainer_state.json @@ -0,0 +1,1566 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 50, + "global_step": 213, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.014222222222222223, + "grad_norm": 10.21201439945407, + "learning_rate": 0.0, + "loss": 0.9819135069847107, + "step": 1 + }, + { + "epoch": 0.028444444444444446, + "grad_norm": 10.607308805754391, + "learning_rate": 4.5454545454545457e-07, + "loss": 0.992728054523468, + "step": 2 + }, + { + "epoch": 0.042666666666666665, + "grad_norm": 8.137000657744727, + "learning_rate": 9.090909090909091e-07, + "loss": 0.88365638256073, + "step": 3 + }, + { + "epoch": 0.05688888888888889, + "grad_norm": 5.015580274302158, + "learning_rate": 1.3636363636363636e-06, + "loss": 0.8155229091644287, + "step": 4 + }, + { + "epoch": 0.07111111111111111, + "grad_norm": 3.513429105458463, + "learning_rate": 1.8181818181818183e-06, + "loss": 0.7148971557617188, + "step": 5 + }, + { + "epoch": 0.08533333333333333, + "grad_norm": 2.890137665957255, + "learning_rate": 2.2727272727272728e-06, + "loss": 0.7388776540756226, + "step": 6 + }, + { + "epoch": 0.09955555555555555, + "grad_norm": 2.6650307199214267, + "learning_rate": 2.7272727272727272e-06, + "loss": 0.6954053044319153, + "step": 7 + }, + { + "epoch": 0.11377777777777778, + "grad_norm": 2.1027438786791453, + "learning_rate": 3.181818181818182e-06, + "loss": 0.6385300159454346, + "step": 8 + }, + { + "epoch": 0.128, + "grad_norm": 1.9707186974912183, + "learning_rate": 3.6363636363636366e-06, + "loss": 0.5974307060241699, + "step": 9 + }, + { + "epoch": 0.14222222222222222, + "grad_norm": 1.7892988232740288, + "learning_rate": 4.0909090909090915e-06, + "loss": 0.5823870897293091, + "step": 10 + }, + { + "epoch": 0.15644444444444444, + "grad_norm": 1.8117663888596973, + "learning_rate": 4.5454545454545455e-06, + "loss": 0.5595569610595703, + "step": 11 + }, + { + "epoch": 0.17066666666666666, + "grad_norm": 1.6892201331978909, + "learning_rate": 5e-06, + "loss": 0.5447146892547607, + "step": 12 + }, + { + "epoch": 0.18488888888888888, + "grad_norm": 1.5301037695464916, + "learning_rate": 5.4545454545454545e-06, + "loss": 0.5179443359375, + "step": 13 + }, + { + "epoch": 0.1991111111111111, + "grad_norm": 1.2533624125991738, + "learning_rate": 5.90909090909091e-06, + "loss": 0.5241127014160156, + "step": 14 + }, + { + "epoch": 0.21333333333333335, + "grad_norm": 1.2394418596122698, + "learning_rate": 6.363636363636364e-06, + "loss": 0.4842333197593689, + "step": 15 + }, + { + "epoch": 0.22755555555555557, + "grad_norm": 1.1420735285454937, + "learning_rate": 6.818181818181818e-06, + "loss": 0.5132038593292236, + "step": 16 + }, + { + "epoch": 0.24177777777777779, + "grad_norm": 1.0423494529114559, + "learning_rate": 7.272727272727273e-06, + "loss": 0.4917834401130676, + "step": 17 + }, + { + "epoch": 0.256, + "grad_norm": 0.9878207423648759, + "learning_rate": 7.727272727272727e-06, + "loss": 0.4813205897808075, + "step": 18 + }, + { + "epoch": 0.2702222222222222, + "grad_norm": 0.9711754260021513, + "learning_rate": 8.181818181818183e-06, + "loss": 0.4883510172367096, + "step": 19 + }, + { + "epoch": 0.28444444444444444, + "grad_norm": 0.9163050690645712, + "learning_rate": 8.636363636363637e-06, + "loss": 0.4871647357940674, + "step": 20 + }, + { + "epoch": 0.2986666666666667, + "grad_norm": 0.8404612592234425, + "learning_rate": 9.090909090909091e-06, + "loss": 0.46865272521972656, + "step": 21 + }, + { + "epoch": 0.3128888888888889, + "grad_norm": 0.8673877914303936, + "learning_rate": 9.545454545454547e-06, + "loss": 0.4601229727268219, + "step": 22 + }, + { + "epoch": 0.32711111111111113, + "grad_norm": 0.8211864748561771, + "learning_rate": 1e-05, + "loss": 0.45921051502227783, + "step": 23 + }, + { + "epoch": 0.3413333333333333, + "grad_norm": 0.8242615873645178, + "learning_rate": 9.999323662872998e-06, + "loss": 0.46318507194519043, + "step": 24 + }, + { + "epoch": 0.35555555555555557, + "grad_norm": 0.7839407977142685, + "learning_rate": 9.99729483446475e-06, + "loss": 0.4633931815624237, + "step": 25 + }, + { + "epoch": 0.36977777777777776, + "grad_norm": 0.7111762322751543, + "learning_rate": 9.993914063644053e-06, + "loss": 0.45212602615356445, + "step": 26 + }, + { + "epoch": 0.384, + "grad_norm": 0.718221639194032, + "learning_rate": 9.989182265027232e-06, + "loss": 0.44105663895606995, + "step": 27 + }, + { + "epoch": 0.3982222222222222, + "grad_norm": 0.7040716895578701, + "learning_rate": 9.98310071873072e-06, + "loss": 0.4427238404750824, + "step": 28 + }, + { + "epoch": 0.41244444444444445, + "grad_norm": 0.7489850512171834, + "learning_rate": 9.975671070024741e-06, + "loss": 0.443437397480011, + "step": 29 + }, + { + "epoch": 0.4266666666666667, + "grad_norm": 0.7002493913122244, + "learning_rate": 9.966895328888195e-06, + "loss": 0.4632475674152374, + "step": 30 + }, + { + "epoch": 0.4408888888888889, + "grad_norm": 0.6824040285805486, + "learning_rate": 9.956775869464901e-06, + "loss": 0.4354594349861145, + "step": 31 + }, + { + "epoch": 0.45511111111111113, + "grad_norm": 0.6597936132766806, + "learning_rate": 9.945315429421307e-06, + "loss": 0.429914653301239, + "step": 32 + }, + { + "epoch": 0.4693333333333333, + "grad_norm": 0.6352588389251546, + "learning_rate": 9.932517109205849e-06, + "loss": 0.43607252836227417, + "step": 33 + }, + { + "epoch": 0.48355555555555557, + "grad_norm": 0.6921974704448327, + "learning_rate": 9.918384371210178e-06, + "loss": 0.45671695470809937, + "step": 34 + }, + { + "epoch": 0.49777777777777776, + "grad_norm": 0.6491660902599252, + "learning_rate": 9.902921038832456e-06, + "loss": 0.4388338327407837, + "step": 35 + }, + { + "epoch": 0.512, + "grad_norm": 0.6096750069184391, + "learning_rate": 9.886131295443003e-06, + "loss": 0.4141705334186554, + "step": 36 + }, + { + "epoch": 0.5262222222222223, + "grad_norm": 0.6503340768273712, + "learning_rate": 9.868019683252543e-06, + "loss": 0.4218006432056427, + "step": 37 + }, + { + "epoch": 0.5404444444444444, + "grad_norm": 0.714483452396674, + "learning_rate": 9.848591102083375e-06, + "loss": 0.4457612931728363, + "step": 38 + }, + { + "epoch": 0.5546666666666666, + "grad_norm": 0.6231964594158875, + "learning_rate": 9.82785080804381e-06, + "loss": 0.4331759214401245, + "step": 39 + }, + { + "epoch": 0.5688888888888889, + "grad_norm": 0.6572593116495337, + "learning_rate": 9.805804412106197e-06, + "loss": 0.4507564902305603, + "step": 40 + }, + { + "epoch": 0.5831111111111111, + "grad_norm": 0.673487925488406, + "learning_rate": 9.782457878588977e-06, + "loss": 0.4254251718521118, + "step": 41 + }, + { + "epoch": 0.5973333333333334, + "grad_norm": 0.5846423543296639, + "learning_rate": 9.75781752354311e-06, + "loss": 0.4125726819038391, + "step": 42 + }, + { + "epoch": 0.6115555555555555, + "grad_norm": 0.598724301403679, + "learning_rate": 9.731890013043367e-06, + "loss": 0.4325122535228729, + "step": 43 + }, + { + "epoch": 0.6257777777777778, + "grad_norm": 0.6218044129681044, + "learning_rate": 9.704682361384941e-06, + "loss": 0.4131222367286682, + "step": 44 + }, + { + "epoch": 0.64, + "grad_norm": 0.617001419027673, + "learning_rate": 9.676201929185809e-06, + "loss": 0.42036178708076477, + "step": 45 + }, + { + "epoch": 0.6542222222222223, + "grad_norm": 0.5809155918543004, + "learning_rate": 9.646456421395447e-06, + "loss": 0.4182787537574768, + "step": 46 + }, + { + "epoch": 0.6684444444444444, + "grad_norm": 0.5850555211488232, + "learning_rate": 9.615453885210368e-06, + "loss": 0.4234634041786194, + "step": 47 + }, + { + "epoch": 0.6826666666666666, + "grad_norm": 0.5511675603941021, + "learning_rate": 9.583202707897075e-06, + "loss": 0.40563416481018066, + "step": 48 + }, + { + "epoch": 0.6968888888888889, + "grad_norm": 0.6111190985848377, + "learning_rate": 9.549711614523007e-06, + "loss": 0.3980520963668823, + "step": 49 + }, + { + "epoch": 0.7111111111111111, + "grad_norm": 0.5736293779202373, + "learning_rate": 9.514989665596114e-06, + "loss": 0.4036375880241394, + "step": 50 + }, + { + "epoch": 0.7111111111111111, + "eval_loss": 0.41823333501815796, + "eval_runtime": 28.2382, + "eval_samples_per_second": 7.083, + "eval_steps_per_second": 1.771, + "step": 50 + }, + { + "epoch": 0.7253333333333334, + "grad_norm": 0.5785272249494463, + "learning_rate": 9.479046254613673e-06, + "loss": 0.41144663095474243, + "step": 51 + }, + { + "epoch": 0.7395555555555555, + "grad_norm": 0.5517717884994647, + "learning_rate": 9.441891105521005e-06, + "loss": 0.38760095834732056, + "step": 52 + }, + { + "epoch": 0.7537777777777778, + "grad_norm": 0.5648730368507402, + "learning_rate": 9.40353427008083e-06, + "loss": 0.4036053717136383, + "step": 53 + }, + { + "epoch": 0.768, + "grad_norm": 0.6019960018903892, + "learning_rate": 9.3639861251539e-06, + "loss": 0.39992809295654297, + "step": 54 + }, + { + "epoch": 0.7822222222222223, + "grad_norm": 0.5943571857589287, + "learning_rate": 9.323257369891702e-06, + "loss": 0.420696496963501, + "step": 55 + }, + { + "epoch": 0.7964444444444444, + "grad_norm": 0.5800482521740772, + "learning_rate": 9.281359022841966e-06, + "loss": 0.41333717107772827, + "step": 56 + }, + { + "epoch": 0.8106666666666666, + "grad_norm": 0.5896438771006596, + "learning_rate": 9.238302418967757e-06, + "loss": 0.41053399443626404, + "step": 57 + }, + { + "epoch": 0.8248888888888889, + "grad_norm": 0.5828843912613246, + "learning_rate": 9.194099206580981e-06, + "loss": 0.4167237877845764, + "step": 58 + }, + { + "epoch": 0.8391111111111111, + "grad_norm": 0.636505663941606, + "learning_rate": 9.14876134419111e-06, + "loss": 0.4126594066619873, + "step": 59 + }, + { + "epoch": 0.8533333333333334, + "grad_norm": 0.5954156377152778, + "learning_rate": 9.102301097269974e-06, + "loss": 0.40986335277557373, + "step": 60 + }, + { + "epoch": 0.8675555555555555, + "grad_norm": 0.5663259824764864, + "learning_rate": 9.05473103493355e-06, + "loss": 0.39707833528518677, + "step": 61 + }, + { + "epoch": 0.8817777777777778, + "grad_norm": 0.6233708113516392, + "learning_rate": 9.006064026541549e-06, + "loss": 0.4146142601966858, + "step": 62 + }, + { + "epoch": 0.896, + "grad_norm": 0.583882066947991, + "learning_rate": 8.956313238215824e-06, + "loss": 0.3761865794658661, + "step": 63 + }, + { + "epoch": 0.9102222222222223, + "grad_norm": 0.559228243852746, + "learning_rate": 8.905492129278478e-06, + "loss": 0.397402286529541, + "step": 64 + }, + { + "epoch": 0.9244444444444444, + "grad_norm": 0.5630501047235328, + "learning_rate": 8.85361444861063e-06, + "loss": 0.40299445390701294, + "step": 65 + }, + { + "epoch": 0.9386666666666666, + "grad_norm": 0.5778346041245924, + "learning_rate": 8.800694230932885e-06, + "loss": 0.4066828489303589, + "step": 66 + }, + { + "epoch": 0.9528888888888889, + "grad_norm": 0.5735220314699014, + "learning_rate": 8.74674579300843e-06, + "loss": 0.39874544739723206, + "step": 67 + }, + { + "epoch": 0.9671111111111111, + "grad_norm": 0.5774583972046927, + "learning_rate": 8.691783729769874e-06, + "loss": 0.38473668694496155, + "step": 68 + }, + { + "epoch": 0.9813333333333333, + "grad_norm": 0.6169263233631689, + "learning_rate": 8.635822910370793e-06, + "loss": 0.4083672761917114, + "step": 69 + }, + { + "epoch": 0.9955555555555555, + "grad_norm": 0.5815111675806847, + "learning_rate": 8.578878474163115e-06, + "loss": 0.3794967234134674, + "step": 70 + }, + { + "epoch": 1.0, + "grad_norm": 0.9420572347818393, + "learning_rate": 8.520965826601394e-06, + "loss": 0.40614083409309387, + "step": 71 + }, + { + "epoch": 1.0142222222222221, + "grad_norm": 0.6281039360207515, + "learning_rate": 8.462100635075097e-06, + "loss": 0.36958712339401245, + "step": 72 + }, + { + "epoch": 1.0284444444444445, + "grad_norm": 0.6233244816927183, + "learning_rate": 8.40229882467003e-06, + "loss": 0.3494463562965393, + "step": 73 + }, + { + "epoch": 1.0426666666666666, + "grad_norm": 0.5621513032586688, + "learning_rate": 8.341576573860049e-06, + "loss": 0.3302980065345764, + "step": 74 + }, + { + "epoch": 1.056888888888889, + "grad_norm": 0.5754725580485192, + "learning_rate": 8.279950310130218e-06, + "loss": 0.35172200202941895, + "step": 75 + }, + { + "epoch": 1.0711111111111111, + "grad_norm": 0.5740681468075599, + "learning_rate": 8.2174367055326e-06, + "loss": 0.3394201993942261, + "step": 76 + }, + { + "epoch": 1.0853333333333333, + "grad_norm": 0.590460680669288, + "learning_rate": 8.154052672175888e-06, + "loss": 0.33312541246414185, + "step": 77 + }, + { + "epoch": 1.0995555555555556, + "grad_norm": 0.5842296281151235, + "learning_rate": 8.08981535765009e-06, + "loss": 0.35658037662506104, + "step": 78 + }, + { + "epoch": 1.1137777777777778, + "grad_norm": 0.6088529350096018, + "learning_rate": 8.024742140387506e-06, + "loss": 0.3449564278125763, + "step": 79 + }, + { + "epoch": 1.1280000000000001, + "grad_norm": 0.5744146541082055, + "learning_rate": 7.95885062496126e-06, + "loss": 0.3357444107532501, + "step": 80 + }, + { + "epoch": 1.1422222222222222, + "grad_norm": 0.5909574288907287, + "learning_rate": 7.892158637322647e-06, + "loss": 0.3170369267463684, + "step": 81 + }, + { + "epoch": 1.1564444444444444, + "grad_norm": 0.6136816332174048, + "learning_rate": 7.824684219978591e-06, + "loss": 0.3318628668785095, + "step": 82 + }, + { + "epoch": 1.1706666666666667, + "grad_norm": 0.5568422531541258, + "learning_rate": 7.756445627110523e-06, + "loss": 0.34082144498825073, + "step": 83 + }, + { + "epoch": 1.1848888888888889, + "grad_norm": 0.6082214692469516, + "learning_rate": 7.68746131963598e-06, + "loss": 0.3453400135040283, + "step": 84 + }, + { + "epoch": 1.199111111111111, + "grad_norm": 0.5993208838733233, + "learning_rate": 7.6177499602143e-06, + "loss": 0.3420029580593109, + "step": 85 + }, + { + "epoch": 1.2133333333333334, + "grad_norm": 0.5493880460077399, + "learning_rate": 7.547330408197695e-06, + "loss": 0.3330039381980896, + "step": 86 + }, + { + "epoch": 1.2275555555555555, + "grad_norm": 0.6151717226672565, + "learning_rate": 7.476221714529167e-06, + "loss": 0.3428732454776764, + "step": 87 + }, + { + "epoch": 1.2417777777777779, + "grad_norm": 0.6359199055571157, + "learning_rate": 7.404443116588548e-06, + "loss": 0.3575543761253357, + "step": 88 + }, + { + "epoch": 1.256, + "grad_norm": 0.5764223898323285, + "learning_rate": 7.332014032988123e-06, + "loss": 0.3414611518383026, + "step": 89 + }, + { + "epoch": 1.2702222222222221, + "grad_norm": 0.5701344890989715, + "learning_rate": 7.2589540583192165e-06, + "loss": 0.3387766480445862, + "step": 90 + }, + { + "epoch": 1.2844444444444445, + "grad_norm": 0.5603334736031269, + "learning_rate": 7.185282957851175e-06, + "loss": 0.32765448093414307, + "step": 91 + }, + { + "epoch": 1.2986666666666666, + "grad_norm": 0.5637583167829048, + "learning_rate": 7.111020662184174e-06, + "loss": 0.3432917296886444, + "step": 92 + }, + { + "epoch": 1.3128888888888888, + "grad_norm": 0.5811322458912195, + "learning_rate": 7.036187261857289e-06, + "loss": 0.3334134817123413, + "step": 93 + }, + { + "epoch": 1.3271111111111111, + "grad_norm": 0.6069998855060962, + "learning_rate": 6.960803001913315e-06, + "loss": 0.3528696894645691, + "step": 94 + }, + { + "epoch": 1.3413333333333333, + "grad_norm": 0.5846537103757758, + "learning_rate": 6.884888276421766e-06, + "loss": 0.3428453207015991, + "step": 95 + }, + { + "epoch": 1.3555555555555556, + "grad_norm": 0.5679194938574699, + "learning_rate": 6.8084636229615786e-06, + "loss": 0.33261382579803467, + "step": 96 + }, + { + "epoch": 1.3697777777777778, + "grad_norm": 0.5611361976965846, + "learning_rate": 6.731549717064975e-06, + "loss": 0.33263495564460754, + "step": 97 + }, + { + "epoch": 1.384, + "grad_norm": 0.5958752887464758, + "learning_rate": 6.654167366624009e-06, + "loss": 0.3409455418586731, + "step": 98 + }, + { + "epoch": 1.3982222222222223, + "grad_norm": 0.5821747639433966, + "learning_rate": 6.576337506261314e-06, + "loss": 0.34175097942352295, + "step": 99 + }, + { + "epoch": 1.4124444444444444, + "grad_norm": 0.5474416142301471, + "learning_rate": 6.498081191666549e-06, + "loss": 0.3422796130180359, + "step": 100 + }, + { + "epoch": 1.4124444444444444, + "eval_loss": 0.39906859397888184, + "eval_runtime": 28.0052, + "eval_samples_per_second": 7.142, + "eval_steps_per_second": 1.785, + "step": 100 + }, + { + "epoch": 1.4266666666666667, + "grad_norm": 0.5470999547878086, + "learning_rate": 6.419419593900109e-06, + "loss": 0.3311368227005005, + "step": 101 + }, + { + "epoch": 1.4408888888888889, + "grad_norm": 0.5417978810198306, + "learning_rate": 6.340373993665607e-06, + "loss": 0.32347068190574646, + "step": 102 + }, + { + "epoch": 1.455111111111111, + "grad_norm": 0.5408071080127892, + "learning_rate": 6.2609657755527135e-06, + "loss": 0.32194703817367554, + "step": 103 + }, + { + "epoch": 1.4693333333333334, + "grad_norm": 0.5610127885481652, + "learning_rate": 6.1812164222518626e-06, + "loss": 0.3477523922920227, + "step": 104 + }, + { + "epoch": 1.4835555555555555, + "grad_norm": 0.6035742435126064, + "learning_rate": 6.101147508742456e-06, + "loss": 0.33583635091781616, + "step": 105 + }, + { + "epoch": 1.4977777777777779, + "grad_norm": 0.5482179611450086, + "learning_rate": 6.020780696456059e-06, + "loss": 0.3305099904537201, + "step": 106 + }, + { + "epoch": 1.512, + "grad_norm": 0.5549975206757557, + "learning_rate": 5.940137727416247e-06, + "loss": 0.3392476439476013, + "step": 107 + }, + { + "epoch": 1.5262222222222221, + "grad_norm": 0.5452500081271687, + "learning_rate": 5.859240418356614e-06, + "loss": 0.3281456232070923, + "step": 108 + }, + { + "epoch": 1.5404444444444443, + "grad_norm": 0.569343792950831, + "learning_rate": 5.778110654818602e-06, + "loss": 0.3557584285736084, + "step": 109 + }, + { + "epoch": 1.5546666666666666, + "grad_norm": 0.5609073891735213, + "learning_rate": 5.696770385230679e-06, + "loss": 0.3375227153301239, + "step": 110 + }, + { + "epoch": 1.568888888888889, + "grad_norm": 0.5979889868516021, + "learning_rate": 5.615241614970546e-06, + "loss": 0.33440524339675903, + "step": 111 + }, + { + "epoch": 1.5831111111111111, + "grad_norm": 0.5752429374657604, + "learning_rate": 5.533546400411899e-06, + "loss": 0.3354341983795166, + "step": 112 + }, + { + "epoch": 1.5973333333333333, + "grad_norm": 0.5583834084640693, + "learning_rate": 5.4517068429574215e-06, + "loss": 0.34512901306152344, + "step": 113 + }, + { + "epoch": 1.6115555555555554, + "grad_norm": 0.5366739842708482, + "learning_rate": 5.3697450830595775e-06, + "loss": 0.3282182812690735, + "step": 114 + }, + { + "epoch": 1.6257777777777778, + "grad_norm": 0.5923091160755782, + "learning_rate": 5.287683294230855e-06, + "loss": 0.3325689136981964, + "step": 115 + }, + { + "epoch": 1.6400000000000001, + "grad_norm": 0.5603788239590529, + "learning_rate": 5.20554367704505e-06, + "loss": 0.3167092204093933, + "step": 116 + }, + { + "epoch": 1.6542222222222223, + "grad_norm": 0.5675689042120237, + "learning_rate": 5.123348453131242e-06, + "loss": 0.3419051766395569, + "step": 117 + }, + { + "epoch": 1.6684444444444444, + "grad_norm": 0.5456542923932385, + "learning_rate": 5.041119859162068e-06, + "loss": 0.333318293094635, + "step": 118 + }, + { + "epoch": 1.6826666666666665, + "grad_norm": 0.570188952777049, + "learning_rate": 4.958880140837934e-06, + "loss": 0.33905792236328125, + "step": 119 + }, + { + "epoch": 1.696888888888889, + "grad_norm": 0.5534588670481234, + "learning_rate": 4.876651546868759e-06, + "loss": 0.32569897174835205, + "step": 120 + }, + { + "epoch": 1.7111111111111112, + "grad_norm": 0.5546097096941289, + "learning_rate": 4.794456322954953e-06, + "loss": 0.3532940745353699, + "step": 121 + }, + { + "epoch": 1.7253333333333334, + "grad_norm": 0.5857002004886875, + "learning_rate": 4.7123167057691446e-06, + "loss": 0.35066288709640503, + "step": 122 + }, + { + "epoch": 1.7395555555555555, + "grad_norm": 0.5351298499201909, + "learning_rate": 4.630254916940424e-06, + "loss": 0.35048598051071167, + "step": 123 + }, + { + "epoch": 1.7537777777777777, + "grad_norm": 0.5572795301130913, + "learning_rate": 4.548293157042581e-06, + "loss": 0.3389701843261719, + "step": 124 + }, + { + "epoch": 1.768, + "grad_norm": 0.5503673971003669, + "learning_rate": 4.466453599588103e-06, + "loss": 0.3477008044719696, + "step": 125 + }, + { + "epoch": 1.7822222222222224, + "grad_norm": 0.5485965256926039, + "learning_rate": 4.384758385029457e-06, + "loss": 0.3188989758491516, + "step": 126 + }, + { + "epoch": 1.7964444444444445, + "grad_norm": 0.5566326955710196, + "learning_rate": 4.3032296147693225e-06, + "loss": 0.3296913504600525, + "step": 127 + }, + { + "epoch": 1.8106666666666666, + "grad_norm": 0.5592929839051507, + "learning_rate": 4.2218893451814005e-06, + "loss": 0.3278958797454834, + "step": 128 + }, + { + "epoch": 1.8248888888888888, + "grad_norm": 0.5372208136274655, + "learning_rate": 4.140759581643386e-06, + "loss": 0.3338126838207245, + "step": 129 + }, + { + "epoch": 1.8391111111111111, + "grad_norm": 0.5506377641318588, + "learning_rate": 4.059862272583755e-06, + "loss": 0.33511388301849365, + "step": 130 + }, + { + "epoch": 1.8533333333333335, + "grad_norm": 0.5509814672788427, + "learning_rate": 3.979219303543942e-06, + "loss": 0.3190721869468689, + "step": 131 + }, + { + "epoch": 1.8675555555555556, + "grad_norm": 0.5437024683532695, + "learning_rate": 3.898852491257547e-06, + "loss": 0.3397029638290405, + "step": 132 + }, + { + "epoch": 1.8817777777777778, + "grad_norm": 0.5577533834862016, + "learning_rate": 3.818783577748138e-06, + "loss": 0.3368285894393921, + "step": 133 + }, + { + "epoch": 1.896, + "grad_norm": 0.5654570173541145, + "learning_rate": 3.7390342244472886e-06, + "loss": 0.34375038743019104, + "step": 134 + }, + { + "epoch": 1.9102222222222223, + "grad_norm": 0.5435541441710147, + "learning_rate": 3.659626006334395e-06, + "loss": 0.32158786058425903, + "step": 135 + }, + { + "epoch": 1.9244444444444444, + "grad_norm": 0.5517597326060792, + "learning_rate": 3.5805804060998926e-06, + "loss": 0.32734596729278564, + "step": 136 + }, + { + "epoch": 1.9386666666666668, + "grad_norm": 0.5529004589065376, + "learning_rate": 3.501918808333453e-06, + "loss": 0.3495728671550751, + "step": 137 + }, + { + "epoch": 1.952888888888889, + "grad_norm": 0.5497949696155701, + "learning_rate": 3.4236624937386874e-06, + "loss": 0.3545406758785248, + "step": 138 + }, + { + "epoch": 1.967111111111111, + "grad_norm": 0.50810534397043, + "learning_rate": 3.3458326333759927e-06, + "loss": 0.3283476233482361, + "step": 139 + }, + { + "epoch": 1.9813333333333332, + "grad_norm": 0.5410931225650443, + "learning_rate": 3.268450282935026e-06, + "loss": 0.3362443447113037, + "step": 140 + }, + { + "epoch": 1.9955555555555555, + "grad_norm": 0.5526133533239165, + "learning_rate": 3.1915363770384223e-06, + "loss": 0.33009064197540283, + "step": 141 + }, + { + "epoch": 2.0, + "grad_norm": 0.8788381822845209, + "learning_rate": 3.1151117235782346e-06, + "loss": 0.34658753871917725, + "step": 142 + }, + { + "epoch": 2.014222222222222, + "grad_norm": 0.6321149682877057, + "learning_rate": 3.0391969980866874e-06, + "loss": 0.2953590750694275, + "step": 143 + }, + { + "epoch": 2.0284444444444443, + "grad_norm": 0.5718010363312446, + "learning_rate": 2.963812738142713e-06, + "loss": 0.28190308809280396, + "step": 144 + }, + { + "epoch": 2.042666666666667, + "grad_norm": 0.5860488240075866, + "learning_rate": 2.8889793378158284e-06, + "loss": 0.29579877853393555, + "step": 145 + }, + { + "epoch": 2.056888888888889, + "grad_norm": 0.5087999304214683, + "learning_rate": 2.814717042148827e-06, + "loss": 0.2849843502044678, + "step": 146 + }, + { + "epoch": 2.071111111111111, + "grad_norm": 0.5097817184428313, + "learning_rate": 2.7410459416807856e-06, + "loss": 0.2797221541404724, + "step": 147 + }, + { + "epoch": 2.0853333333333333, + "grad_norm": 0.5476441083873779, + "learning_rate": 2.6679859670118785e-06, + "loss": 0.2898455262184143, + "step": 148 + }, + { + "epoch": 2.0995555555555554, + "grad_norm": 0.5438939155777798, + "learning_rate": 2.5955568834114523e-06, + "loss": 0.27163612842559814, + "step": 149 + }, + { + "epoch": 2.113777777777778, + "grad_norm": 0.5562699539267528, + "learning_rate": 2.523778285470835e-06, + "loss": 0.28451889753341675, + "step": 150 + }, + { + "epoch": 2.113777777777778, + "eval_loss": 0.4041806757450104, + "eval_runtime": 27.9066, + "eval_samples_per_second": 7.167, + "eval_steps_per_second": 1.792, + "step": 150 + }, + { + "epoch": 2.128, + "grad_norm": 0.5511756829107596, + "learning_rate": 2.452669591802307e-06, + "loss": 0.2696727514266968, + "step": 151 + }, + { + "epoch": 2.1422222222222222, + "grad_norm": 0.5230762023952198, + "learning_rate": 2.3822500397857016e-06, + "loss": 0.2786106467247009, + "step": 152 + }, + { + "epoch": 2.1564444444444444, + "grad_norm": 0.5376649924382556, + "learning_rate": 2.3125386803640188e-06, + "loss": 0.29131805896759033, + "step": 153 + }, + { + "epoch": 2.1706666666666665, + "grad_norm": 0.5176483788211893, + "learning_rate": 2.243554372889479e-06, + "loss": 0.2755666971206665, + "step": 154 + }, + { + "epoch": 2.1848888888888887, + "grad_norm": 0.5045947338221831, + "learning_rate": 2.1753157800214107e-06, + "loss": 0.27331122756004333, + "step": 155 + }, + { + "epoch": 2.1991111111111112, + "grad_norm": 0.5193109465079893, + "learning_rate": 2.1078413626773547e-06, + "loss": 0.28715455532073975, + "step": 156 + }, + { + "epoch": 2.2133333333333334, + "grad_norm": 0.5212127411265028, + "learning_rate": 2.0411493750387423e-06, + "loss": 0.26138654351234436, + "step": 157 + }, + { + "epoch": 2.2275555555555555, + "grad_norm": 0.5448628823500382, + "learning_rate": 1.9752578596124955e-06, + "loss": 0.29015523195266724, + "step": 158 + }, + { + "epoch": 2.2417777777777776, + "grad_norm": 0.5501585558078743, + "learning_rate": 1.9101846423499113e-06, + "loss": 0.2733340561389923, + "step": 159 + }, + { + "epoch": 2.2560000000000002, + "grad_norm": 0.5315657420322433, + "learning_rate": 1.8459473278241125e-06, + "loss": 0.2742033302783966, + "step": 160 + }, + { + "epoch": 2.2702222222222224, + "grad_norm": 0.4902572304429622, + "learning_rate": 1.7825632944674016e-06, + "loss": 0.2752198874950409, + "step": 161 + }, + { + "epoch": 2.2844444444444445, + "grad_norm": 0.534591959936222, + "learning_rate": 1.7200496898697832e-06, + "loss": 0.28458085656166077, + "step": 162 + }, + { + "epoch": 2.2986666666666666, + "grad_norm": 0.5106425495762753, + "learning_rate": 1.6584234261399535e-06, + "loss": 0.277709424495697, + "step": 163 + }, + { + "epoch": 2.3128888888888888, + "grad_norm": 0.5539379653737809, + "learning_rate": 1.5977011753299726e-06, + "loss": 0.2714066505432129, + "step": 164 + }, + { + "epoch": 2.327111111111111, + "grad_norm": 0.5579822508546088, + "learning_rate": 1.5378993649249053e-06, + "loss": 0.2929142117500305, + "step": 165 + }, + { + "epoch": 2.3413333333333335, + "grad_norm": 0.5128512129047608, + "learning_rate": 1.4790341733986085e-06, + "loss": 0.274400919675827, + "step": 166 + }, + { + "epoch": 2.3555555555555556, + "grad_norm": 0.5180316098734932, + "learning_rate": 1.4211215258368866e-06, + "loss": 0.2633841633796692, + "step": 167 + }, + { + "epoch": 2.3697777777777778, + "grad_norm": 0.5226866839878248, + "learning_rate": 1.3641770896292083e-06, + "loss": 0.271608829498291, + "step": 168 + }, + { + "epoch": 2.384, + "grad_norm": 0.5213295087948763, + "learning_rate": 1.3082162702301276e-06, + "loss": 0.2772142291069031, + "step": 169 + }, + { + "epoch": 2.398222222222222, + "grad_norm": 0.5200525913463248, + "learning_rate": 1.2532542069915722e-06, + "loss": 0.2833596467971802, + "step": 170 + }, + { + "epoch": 2.4124444444444446, + "grad_norm": 0.5158405462335311, + "learning_rate": 1.1993057690671174e-06, + "loss": 0.27554380893707275, + "step": 171 + }, + { + "epoch": 2.4266666666666667, + "grad_norm": 0.5070004563853152, + "learning_rate": 1.1463855513893695e-06, + "loss": 0.291002094745636, + "step": 172 + }, + { + "epoch": 2.440888888888889, + "grad_norm": 0.5227423053894303, + "learning_rate": 1.0945078707215224e-06, + "loss": 0.2779809832572937, + "step": 173 + }, + { + "epoch": 2.455111111111111, + "grad_norm": 0.530364411528684, + "learning_rate": 1.0436867617841768e-06, + "loss": 0.2763213515281677, + "step": 174 + }, + { + "epoch": 2.469333333333333, + "grad_norm": 0.5210228993927588, + "learning_rate": 9.939359734584552e-07, + "loss": 0.28753769397735596, + "step": 175 + }, + { + "epoch": 2.4835555555555557, + "grad_norm": 0.5214828400537481, + "learning_rate": 9.452689650664515e-07, + "loss": 0.2837107181549072, + "step": 176 + }, + { + "epoch": 2.497777777777778, + "grad_norm": 0.5036225380576258, + "learning_rate": 8.976989027300265e-07, + "loss": 0.2816150188446045, + "step": 177 + }, + { + "epoch": 2.512, + "grad_norm": 0.5488666037756793, + "learning_rate": 8.512386558088919e-07, + "loss": 0.2889126241207123, + "step": 178 + }, + { + "epoch": 2.526222222222222, + "grad_norm": 0.5181437270813625, + "learning_rate": 8.059007934190194e-07, + "loss": 0.2788556218147278, + "step": 179 + }, + { + "epoch": 2.5404444444444443, + "grad_norm": 0.5160804538602589, + "learning_rate": 7.61697581032243e-07, + "loss": 0.29226237535476685, + "step": 180 + }, + { + "epoch": 2.554666666666667, + "grad_norm": 0.5105467594147919, + "learning_rate": 7.186409771580355e-07, + "loss": 0.2915460765361786, + "step": 181 + }, + { + "epoch": 2.568888888888889, + "grad_norm": 0.49398572364663706, + "learning_rate": 6.76742630108298e-07, + "loss": 0.2790933847427368, + "step": 182 + }, + { + "epoch": 2.583111111111111, + "grad_norm": 1.14219599430935e+31, + "learning_rate": 6.360138748461015e-07, + "loss": 0.27808326482772827, + "step": 183 + }, + { + "epoch": 2.5973333333333333, + "grad_norm": 0.4983345243585802, + "learning_rate": 5.964657299191712e-07, + "loss": 0.2690117359161377, + "step": 184 + }, + { + "epoch": 2.6115555555555554, + "grad_norm": 0.5069008538717243, + "learning_rate": 5.581088944789953e-07, + "loss": 0.27599871158599854, + "step": 185 + }, + { + "epoch": 2.6257777777777775, + "grad_norm": 0.521476062094092, + "learning_rate": 5.20953745386329e-07, + "loss": 0.2919716238975525, + "step": 186 + }, + { + "epoch": 2.64, + "grad_norm": 0.510790594456331, + "learning_rate": 4.850103344038853e-07, + "loss": 0.28502631187438965, + "step": 187 + }, + { + "epoch": 2.6542222222222223, + "grad_norm": 0.49201617380837526, + "learning_rate": 4.502883854769935e-07, + "loss": 0.26374831795692444, + "step": 188 + }, + { + "epoch": 2.6684444444444444, + "grad_norm": 0.5211970358874106, + "learning_rate": 4.167972921029262e-07, + "loss": 0.28197193145751953, + "step": 189 + }, + { + "epoch": 2.6826666666666665, + "grad_norm": 0.5370264524554923, + "learning_rate": 3.8454611478963235e-07, + "loss": 0.2938516139984131, + "step": 190 + }, + { + "epoch": 2.696888888888889, + "grad_norm": 0.5232192741343314, + "learning_rate": 3.535435786045538e-07, + "loss": 0.27452725172042847, + "step": 191 + }, + { + "epoch": 2.7111111111111112, + "grad_norm": 0.5024016728659283, + "learning_rate": 3.237980708141919e-07, + "loss": 0.2790898084640503, + "step": 192 + }, + { + "epoch": 2.7253333333333334, + "grad_norm": 0.5309491933991259, + "learning_rate": 2.9531763861505967e-07, + "loss": 0.2802097201347351, + "step": 193 + }, + { + "epoch": 2.7395555555555555, + "grad_norm": 0.5117668670583797, + "learning_rate": 2.681099869566328e-07, + "loss": 0.2703718841075897, + "step": 194 + }, + { + "epoch": 2.7537777777777777, + "grad_norm": 0.5093979232137983, + "learning_rate": 2.4218247645689306e-07, + "loss": 0.2852793335914612, + "step": 195 + }, + { + "epoch": 2.768, + "grad_norm": 0.49789356969379145, + "learning_rate": 2.175421214110235e-07, + "loss": 0.27227064967155457, + "step": 196 + }, + { + "epoch": 2.7822222222222224, + "grad_norm": 0.5255831952638869, + "learning_rate": 1.941955878938029e-07, + "loss": 0.28222954273223877, + "step": 197 + }, + { + "epoch": 2.7964444444444445, + "grad_norm": 0.5060979808451155, + "learning_rate": 1.721491919561913e-07, + "loss": 0.2887798249721527, + "step": 198 + }, + { + "epoch": 2.8106666666666666, + "grad_norm": 0.5024415391860155, + "learning_rate": 1.514088979166256e-07, + "loss": 0.2730322480201721, + "step": 199 + }, + { + "epoch": 2.824888888888889, + "grad_norm": 0.5071775702854565, + "learning_rate": 1.3198031674745814e-07, + "loss": 0.28655844926834106, + "step": 200 + }, + { + "epoch": 2.824888888888889, + "eval_loss": 0.3998975455760956, + "eval_runtime": 27.4547, + "eval_samples_per_second": 7.285, + "eval_steps_per_second": 1.821, + "step": 200 + }, + { + "epoch": 2.8391111111111114, + "grad_norm": 0.5098506733931571, + "learning_rate": 1.138687045569975e-07, + "loss": 0.28616365790367126, + "step": 201 + }, + { + "epoch": 2.8533333333333335, + "grad_norm": 0.5014028056158423, + "learning_rate": 9.70789611675449e-08, + "loss": 0.28517472743988037, + "step": 202 + }, + { + "epoch": 2.8675555555555556, + "grad_norm": 0.5046489746973516, + "learning_rate": 8.161562878982399e-08, + "loss": 0.27302050590515137, + "step": 203 + }, + { + "epoch": 2.8817777777777778, + "grad_norm": 0.5244675639408302, + "learning_rate": 6.74828907941516e-08, + "loss": 0.28563594818115234, + "step": 204 + }, + { + "epoch": 2.896, + "grad_norm": 0.5147937769432819, + "learning_rate": 5.468457057869358e-08, + "loss": 0.2887130379676819, + "step": 205 + }, + { + "epoch": 2.910222222222222, + "grad_norm": 0.49008819658339253, + "learning_rate": 4.322413053509944e-08, + "loss": 0.2721640467643738, + "step": 206 + }, + { + "epoch": 2.924444444444444, + "grad_norm": 0.5051230135404199, + "learning_rate": 3.31046711118066e-08, + "loss": 0.27188998460769653, + "step": 207 + }, + { + "epoch": 2.9386666666666668, + "grad_norm": 0.5046747206900266, + "learning_rate": 2.4328929975260262e-08, + "loss": 0.2795502245426178, + "step": 208 + }, + { + "epoch": 2.952888888888889, + "grad_norm": 0.5035617947795644, + "learning_rate": 1.6899281269279756e-08, + "loss": 0.2727120816707611, + "step": 209 + }, + { + "epoch": 2.967111111111111, + "grad_norm": 0.5078812650660502, + "learning_rate": 1.0817734972768946e-08, + "loss": 0.2795644998550415, + "step": 210 + }, + { + "epoch": 2.981333333333333, + "grad_norm": 0.5269813851443549, + "learning_rate": 6.085936355947897e-09, + "loss": 0.28946471214294434, + "step": 211 + }, + { + "epoch": 2.9955555555555557, + "grad_norm": 0.503969003244228, + "learning_rate": 2.7051655352494654e-09, + "loss": 0.26888906955718994, + "step": 212 + }, + { + "epoch": 3.0, + "grad_norm": 0.9730892144000087, + "learning_rate": 6.763371270035457e-10, + "loss": 0.33630549907684326, + "step": 213 + }, + { + "epoch": 3.0, + "step": 213, + "total_flos": 62893478699008.0, + "train_loss": 0.36721718521185326, + "train_runtime": 28478.4394, + "train_samples_per_second": 1.896, + "train_steps_per_second": 0.007 + } + ], + "logging_steps": 1, + "max_steps": 213, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 62893478699008.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/qwen2.5-7b-instruct/training_args.bin b/qwen2.5-7b-instruct/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..e59d7187e91e3dd0138c38f45c246765301e22e6 --- /dev/null +++ b/qwen2.5-7b-instruct/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:238e1f7259942c35813600da4056c3d822147cad18b0d1147780f439481f9c35 +size 7377 diff --git a/qwen2.5-7b-instruct/training_eval_loss.png b/qwen2.5-7b-instruct/training_eval_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..db8699b1482320dcdaa64e7e37c4199392da85c9 Binary files /dev/null and b/qwen2.5-7b-instruct/training_eval_loss.png differ diff --git a/qwen2.5-7b-instruct/training_loss.png b/qwen2.5-7b-instruct/training_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..a2ea7e9eb578d4ec40b8f2eb0628fd3f661778d5 Binary files /dev/null and b/qwen2.5-7b-instruct/training_loss.png differ