diff --git a/save_lora_math_r8_2e4/README.md b/save_lora_math_r8_2e4/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d380bbbf0bf0fa1f211b53410559607e5007a209 --- /dev/null +++ b/save_lora_math_r8_2e4/README.md @@ -0,0 +1,202 @@ +--- +library_name: peft +base_model: meta-llama/Llama-2-7b-hf +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.9.0 \ No newline at end of file diff --git a/save_lora_math_r8_2e4/adapter_config.json b/save_lora_math_r8_2e4/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9c81a592f0ff08374a4f61f6003029070a8429ae --- /dev/null +++ b/save_lora_math_r8_2e4/adapter_config.json @@ -0,0 +1,35 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "meta-llama/Llama-2-7b-hf", + "bias": "none", + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "mora_type": 1, + "peft_type": "LORA", + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "down_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "use_dora": false, + "use_mora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/save_lora_math_r8_2e4/adapter_model.safetensors b/save_lora_math_r8_2e4/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..84d593cf0b9252ca07135b80ad509d25326195d6 --- /dev/null +++ b/save_lora_math_r8_2e4/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7922bfeb8ec4dcc0d2ea69364c816ed1fd5b311f4e35780a366c6cc077c8c3b7 +size 40036488 diff --git a/save_lora_math_r8_2e4/checkpoint-3000/README.md b/save_lora_math_r8_2e4/checkpoint-3000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d380bbbf0bf0fa1f211b53410559607e5007a209 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-3000/README.md @@ -0,0 +1,202 @@ +--- +library_name: peft +base_model: meta-llama/Llama-2-7b-hf +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.9.0 \ No newline at end of file diff --git a/save_lora_math_r8_2e4/checkpoint-3000/adapter_config.json b/save_lora_math_r8_2e4/checkpoint-3000/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9c81a592f0ff08374a4f61f6003029070a8429ae --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-3000/adapter_config.json @@ -0,0 +1,35 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "meta-llama/Llama-2-7b-hf", + "bias": "none", + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "mora_type": 1, + "peft_type": "LORA", + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "down_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "use_dora": false, + "use_mora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/save_lora_math_r8_2e4/checkpoint-3000/adapter_model.safetensors b/save_lora_math_r8_2e4/checkpoint-3000/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..718b983ba8a9d597c17b9ca079c655f989408133 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-3000/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d8bbeaf7a13e033a30d3ff93e907fd1c800246e07f92ab8b3bea5216819667b +size 40036488 diff --git a/save_lora_math_r8_2e4/checkpoint-3000/global_step3000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/save_lora_math_r8_2e4/checkpoint-3000/global_step3000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..61bbbb77a9b3e0de7add87cfa32f5489076811f3 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-3000/global_step3000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:097f4fc2dff4c0019e76c4d82f463db72d19f9eb97bf4ba0f45fabab054e96df +size 119958768 diff --git a/save_lora_math_r8_2e4/checkpoint-3000/global_step3000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/save_lora_math_r8_2e4/checkpoint-3000/global_step3000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4b31ef968b5721bde717c003392aa960b7256e1f --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-3000/global_step3000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30f85058b3ba7905cff74cf14d50def4643f98230994f9e5161ce167106ef580 +size 119958896 diff --git a/save_lora_math_r8_2e4/checkpoint-3000/global_step3000/mp_rank_00_model_states.pt b/save_lora_math_r8_2e4/checkpoint-3000/global_step3000/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6f9cb63210ffc1b348ce37287fad72e789a6faa --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-3000/global_step3000/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7f04aa811d6e5abf112e655d5f8f8fd9ada01b87e89d8b97e3f29aa6c343635 +size 40303404 diff --git a/save_lora_math_r8_2e4/checkpoint-3000/latest b/save_lora_math_r8_2e4/checkpoint-3000/latest new file mode 100644 index 0000000000000000000000000000000000000000..1d7ba373feda3875a77d01589035020625a7c98e --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-3000/latest @@ -0,0 +1 @@ +global_step3000 \ No newline at end of file diff --git a/save_lora_math_r8_2e4/checkpoint-3000/rng_state_0.pth b/save_lora_math_r8_2e4/checkpoint-3000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..d904b6b75a6a4721c5fd131985940e98684429d6 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-3000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56526f77903f85018631e6fb08126716f9d9398f225ded68d3f473ea442210e0 +size 14512 diff --git a/save_lora_math_r8_2e4/checkpoint-3000/rng_state_1.pth b/save_lora_math_r8_2e4/checkpoint-3000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..16bd265655668174f09eab968a14c2b9feb3a276 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-3000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0314a2d2d554ff6b3c58119915efe7d7b24024415be962fcccc00d862ee62ab0 +size 14512 diff --git a/save_lora_math_r8_2e4/checkpoint-3000/scheduler.pt b/save_lora_math_r8_2e4/checkpoint-3000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..a72fd31a35d51553db20c035490aafefc2fa6827 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-3000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8313ff4ba783fba32220027d8d9836ff033194b27aa5b0f87929a3fbcf4da48b +size 1064 diff --git a/save_lora_math_r8_2e4/checkpoint-3000/trainer_state.json b/save_lora_math_r8_2e4/checkpoint-3000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..83fa479f4616f3a63189cebf94efd1b785d72e6e --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-3000/trainer_state.json @@ -0,0 +1,3621 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9721518987341772, + "eval_steps": 500, + "global_step": 3000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 3.5971223021582732e-06, + "loss": 0.7628, + "step": 5 + }, + { + "epoch": 0.0, + "learning_rate": 7.1942446043165465e-06, + "loss": 0.769, + "step": 10 + }, + { + "epoch": 0.0, + "learning_rate": 1.0791366906474821e-05, + "loss": 0.7619, + "step": 15 + }, + { + "epoch": 0.01, + "learning_rate": 1.4388489208633093e-05, + "loss": 0.7564, + "step": 20 + }, + { + "epoch": 0.01, + "learning_rate": 1.7985611510791367e-05, + "loss": 0.7549, + "step": 25 + }, + { + "epoch": 0.01, + "learning_rate": 2.1582733812949642e-05, + "loss": 0.7001, + "step": 30 + }, + { + "epoch": 0.01, + "learning_rate": 2.5179856115107914e-05, + "loss": 0.6289, + "step": 35 + }, + { + "epoch": 0.01, + "learning_rate": 2.8776978417266186e-05, + "loss": 0.5632, + "step": 40 + }, + { + "epoch": 0.01, + "learning_rate": 3.237410071942446e-05, + "loss": 0.4923, + "step": 45 + }, + { + "epoch": 0.02, + "learning_rate": 3.597122302158273e-05, + "loss": 0.4582, + "step": 50 + }, + { + "epoch": 0.02, + "learning_rate": 3.956834532374101e-05, + "loss": 0.4357, + "step": 55 + }, + { + "epoch": 0.02, + "learning_rate": 4.3165467625899284e-05, + "loss": 0.4079, + "step": 60 + }, + { + "epoch": 0.02, + "learning_rate": 4.676258992805755e-05, + "loss": 0.4047, + "step": 65 + }, + { + "epoch": 0.02, + "learning_rate": 5.035971223021583e-05, + "loss": 0.3916, + "step": 70 + }, + { + "epoch": 0.02, + "learning_rate": 5.39568345323741e-05, + "loss": 0.3926, + "step": 75 + }, + { + "epoch": 0.03, + "learning_rate": 5.755395683453237e-05, + "loss": 0.3725, + "step": 80 + }, + { + "epoch": 0.03, + "learning_rate": 6.115107913669065e-05, + "loss": 0.3769, + "step": 85 + }, + { + "epoch": 0.03, + "learning_rate": 6.474820143884892e-05, + "loss": 0.352, + "step": 90 + }, + { + "epoch": 0.03, + "learning_rate": 6.83453237410072e-05, + "loss": 0.3534, + "step": 95 + }, + { + "epoch": 0.03, + "learning_rate": 7.194244604316547e-05, + "loss": 0.3432, + "step": 100 + }, + { + "epoch": 0.03, + "learning_rate": 7.553956834532374e-05, + "loss": 0.3286, + "step": 105 + }, + { + "epoch": 0.04, + "learning_rate": 7.913669064748202e-05, + "loss": 0.3298, + "step": 110 + }, + { + "epoch": 0.04, + "learning_rate": 8.273381294964029e-05, + "loss": 0.3223, + "step": 115 + }, + { + "epoch": 0.04, + "learning_rate": 8.633093525179857e-05, + "loss": 0.3181, + "step": 120 + }, + { + "epoch": 0.04, + "learning_rate": 8.992805755395684e-05, + "loss": 0.3225, + "step": 125 + }, + { + "epoch": 0.04, + "learning_rate": 9.35251798561151e-05, + "loss": 0.3145, + "step": 130 + }, + { + "epoch": 0.04, + "learning_rate": 9.712230215827339e-05, + "loss": 0.3137, + "step": 135 + }, + { + "epoch": 0.05, + "learning_rate": 0.00010071942446043166, + "loss": 0.3102, + "step": 140 + }, + { + "epoch": 0.05, + "learning_rate": 0.00010431654676258992, + "loss": 0.2951, + "step": 145 + }, + { + "epoch": 0.05, + "learning_rate": 0.0001079136690647482, + "loss": 0.3021, + "step": 150 + }, + { + "epoch": 0.05, + "learning_rate": 0.00011151079136690648, + "loss": 0.2923, + "step": 155 + }, + { + "epoch": 0.05, + "learning_rate": 0.00011510791366906474, + "loss": 0.2953, + "step": 160 + }, + { + "epoch": 0.05, + "learning_rate": 0.00011870503597122303, + "loss": 0.2893, + "step": 165 + }, + { + "epoch": 0.06, + "learning_rate": 0.0001223021582733813, + "loss": 0.2953, + "step": 170 + }, + { + "epoch": 0.06, + "learning_rate": 0.00012589928057553956, + "loss": 0.2931, + "step": 175 + }, + { + "epoch": 0.06, + "learning_rate": 0.00012949640287769783, + "loss": 0.2879, + "step": 180 + }, + { + "epoch": 0.06, + "learning_rate": 0.00013309352517985613, + "loss": 0.2642, + "step": 185 + }, + { + "epoch": 0.06, + "learning_rate": 0.0001366906474820144, + "loss": 0.2772, + "step": 190 + }, + { + "epoch": 0.06, + "learning_rate": 0.00014028776978417266, + "loss": 0.2858, + "step": 195 + }, + { + "epoch": 0.06, + "learning_rate": 0.00014388489208633093, + "loss": 0.2724, + "step": 200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0001474820143884892, + "loss": 0.2726, + "step": 205 + }, + { + "epoch": 0.07, + "learning_rate": 0.00015107913669064747, + "loss": 0.2678, + "step": 210 + }, + { + "epoch": 0.07, + "learning_rate": 0.00015467625899280577, + "loss": 0.2789, + "step": 215 + }, + { + "epoch": 0.07, + "learning_rate": 0.00015827338129496403, + "loss": 0.2742, + "step": 220 + }, + { + "epoch": 0.07, + "learning_rate": 0.0001618705035971223, + "loss": 0.2732, + "step": 225 + }, + { + "epoch": 0.07, + "learning_rate": 0.00016546762589928057, + "loss": 0.2741, + "step": 230 + }, + { + "epoch": 0.08, + "learning_rate": 0.00016906474820143884, + "loss": 0.2665, + "step": 235 + }, + { + "epoch": 0.08, + "learning_rate": 0.00017266187050359714, + "loss": 0.2771, + "step": 240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0001762589928057554, + "loss": 0.2751, + "step": 245 + }, + { + "epoch": 0.08, + "learning_rate": 0.00017985611510791367, + "loss": 0.2597, + "step": 250 + }, + { + "epoch": 0.08, + "learning_rate": 0.00018345323741007194, + "loss": 0.275, + "step": 255 + }, + { + "epoch": 0.08, + "learning_rate": 0.0001870503597122302, + "loss": 0.2791, + "step": 260 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019064748201438848, + "loss": 0.2493, + "step": 265 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019424460431654677, + "loss": 0.27, + "step": 270 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019784172661870504, + "loss": 0.2676, + "step": 275 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019999997550557367, + "loss": 0.2613, + "step": 280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019999969994341502, + "loss": 0.2747, + "step": 285 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019999911820191125, + "loss": 0.2699, + "step": 290 + }, + { + "epoch": 0.1, + "learning_rate": 0.0001999982302828436, + "loss": 0.2692, + "step": 295 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999703618893072, + "loss": 0.2522, + "step": 300 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999553592382863, + "loss": 0.2541, + "step": 305 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999372949213085, + "loss": 0.2614, + "step": 310 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999161689936837, + "loss": 0.2481, + "step": 315 + }, + { + "epoch": 0.1, + "learning_rate": 0.0001999891981520095, + "loss": 0.2497, + "step": 320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019998647325745995, + "loss": 0.2545, + "step": 325 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019998344222406283, + "loss": 0.2501, + "step": 330 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019998010506109854, + "loss": 0.2577, + "step": 335 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019997646177878487, + "loss": 0.2585, + "step": 340 + }, + { + "epoch": 0.11, + "learning_rate": 0.0001999725123882768, + "loss": 0.2494, + "step": 345 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019996825690166657, + "loss": 0.2538, + "step": 350 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019996369533198364, + "loss": 0.2469, + "step": 355 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019995882769319466, + "loss": 0.2502, + "step": 360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0001999536540002034, + "loss": 0.2361, + "step": 365 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019994817426885057, + "loss": 0.2446, + "step": 370 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019994238851591417, + "loss": 0.256, + "step": 375 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019993629675910896, + "loss": 0.2352, + "step": 380 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019992989901708672, + "loss": 0.2538, + "step": 385 + }, + { + "epoch": 0.13, + "learning_rate": 0.0001999231953094361, + "loss": 0.2523, + "step": 390 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019991618565668243, + "loss": 0.2512, + "step": 395 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019990887008028798, + "loss": 0.2402, + "step": 400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019990124860265156, + "loss": 0.2423, + "step": 405 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019989332124710862, + "loss": 0.2335, + "step": 410 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019988508803793124, + "loss": 0.2284, + "step": 415 + }, + { + "epoch": 0.14, + "learning_rate": 0.00019987654900032774, + "loss": 0.2382, + "step": 420 + }, + { + "epoch": 0.14, + "learning_rate": 0.0001998677041604431, + "loss": 0.2526, + "step": 425 + }, + { + "epoch": 0.14, + "learning_rate": 0.0001998585535453584, + "loss": 0.2417, + "step": 430 + }, + { + "epoch": 0.14, + "learning_rate": 0.00019984909718309105, + "loss": 0.2381, + "step": 435 + }, + { + "epoch": 0.14, + "learning_rate": 0.0001998393351025945, + "loss": 0.2418, + "step": 440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00019982926733375843, + "loss": 0.2409, + "step": 445 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001998188939074083, + "loss": 0.2467, + "step": 450 + }, + { + "epoch": 0.15, + "learning_rate": 0.00019980821485530544, + "loss": 0.2305, + "step": 455 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001997972302101471, + "loss": 0.232, + "step": 460 + }, + { + "epoch": 0.15, + "learning_rate": 0.00019978594000556602, + "loss": 0.225, + "step": 465 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001997743442761306, + "loss": 0.2259, + "step": 470 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001997624430573447, + "loss": 0.2414, + "step": 475 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019975023638564747, + "loss": 0.238, + "step": 480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019973772429841336, + "loss": 0.224, + "step": 485 + }, + { + "epoch": 0.16, + "learning_rate": 0.0001997249068339519, + "loss": 0.2369, + "step": 490 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019971178403150762, + "loss": 0.2393, + "step": 495 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019969835593125997, + "loss": 0.2249, + "step": 500 + }, + { + "epoch": 0.16, + "learning_rate": 0.0001996846225743231, + "loss": 0.2369, + "step": 505 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019967058400274592, + "loss": 0.2267, + "step": 510 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019965624025951168, + "loss": 0.2294, + "step": 515 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019964159138853815, + "loss": 0.2255, + "step": 520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019962663743467726, + "loss": 0.22, + "step": 525 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019961137844371507, + "loss": 0.2398, + "step": 530 + }, + { + "epoch": 0.17, + "learning_rate": 0.0001995958144623716, + "loss": 0.2431, + "step": 535 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019957994553830067, + "loss": 0.2245, + "step": 540 + }, + { + "epoch": 0.18, + "learning_rate": 0.0001995637717200898, + "loss": 0.2295, + "step": 545 + }, + { + "epoch": 0.18, + "learning_rate": 0.00019954729305726008, + "loss": 0.2288, + "step": 550 + }, + { + "epoch": 0.18, + "learning_rate": 0.00019953050960026586, + "loss": 0.2333, + "step": 555 + }, + { + "epoch": 0.18, + "learning_rate": 0.0001995134214004948, + "loss": 0.2316, + "step": 560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0001994960285102676, + "loss": 0.2285, + "step": 565 + }, + { + "epoch": 0.18, + "learning_rate": 0.00019947833098283782, + "loss": 0.2273, + "step": 570 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019946032887239182, + "loss": 0.2344, + "step": 575 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019944202223404854, + "loss": 0.2251, + "step": 580 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019942341112385922, + "loss": 0.226, + "step": 585 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019940449559880752, + "loss": 0.2353, + "step": 590 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019938527571680892, + "loss": 0.226, + "step": 595 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019936575153671097, + "loss": 0.221, + "step": 600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019934592311829285, + "loss": 0.2219, + "step": 605 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019932579052226528, + "loss": 0.234, + "step": 610 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019930535381027027, + "loss": 0.2181, + "step": 615 + }, + { + "epoch": 0.2, + "learning_rate": 0.000199284613044881, + "loss": 0.2191, + "step": 620 + }, + { + "epoch": 0.2, + "learning_rate": 0.0001992635682896016, + "loss": 0.235, + "step": 625 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019924221960886698, + "loss": 0.2256, + "step": 630 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019922056706804257, + "loss": 0.2265, + "step": 635 + }, + { + "epoch": 0.21, + "learning_rate": 0.0001991986107334242, + "loss": 0.2158, + "step": 640 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019917635067223777, + "loss": 0.2238, + "step": 645 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019915378695263924, + "loss": 0.2182, + "step": 650 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019913091964371423, + "loss": 0.2185, + "step": 655 + }, + { + "epoch": 0.21, + "learning_rate": 0.000199107748815478, + "loss": 0.2063, + "step": 660 + }, + { + "epoch": 0.22, + "learning_rate": 0.000199084274538875, + "loss": 0.2162, + "step": 665 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019906049688577887, + "loss": 0.2185, + "step": 670 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019903641592899205, + "loss": 0.2186, + "step": 675 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019901203174224564, + "loss": 0.2292, + "step": 680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019898734440019933, + "loss": 0.2162, + "step": 685 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019896235397844078, + "loss": 0.219, + "step": 690 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019893706055348578, + "loss": 0.218, + "step": 695 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019891146420277777, + "loss": 0.2211, + "step": 700 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019888556500468775, + "loss": 0.2227, + "step": 705 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019885936303851395, + "loss": 0.2214, + "step": 710 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019883285838448163, + "loss": 0.2204, + "step": 715 + }, + { + "epoch": 0.23, + "learning_rate": 0.0001988060511237428, + "loss": 0.2228, + "step": 720 + }, + { + "epoch": 0.23, + "learning_rate": 0.000198778941338376, + "loss": 0.2259, + "step": 725 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019875152911138602, + "loss": 0.2062, + "step": 730 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019872381452670372, + "loss": 0.2073, + "step": 735 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019869579766918568, + "loss": 0.2172, + "step": 740 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019866747862461396, + "loss": 0.21, + "step": 745 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019863885747969598, + "loss": 0.2195, + "step": 750 + }, + { + "epoch": 0.24, + "learning_rate": 0.0001986099343220639, + "loss": 0.2129, + "step": 755 + }, + { + "epoch": 0.25, + "learning_rate": 0.0001985807092402748, + "loss": 0.215, + "step": 760 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019855118232381015, + "loss": 0.2085, + "step": 765 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019852135366307545, + "loss": 0.2196, + "step": 770 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019849122334940022, + "loss": 0.2129, + "step": 775 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019846079147503755, + "loss": 0.2221, + "step": 780 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019843005813316377, + "loss": 0.2133, + "step": 785 + }, + { + "epoch": 0.26, + "learning_rate": 0.0001983990234178784, + "loss": 0.2066, + "step": 790 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019836768742420352, + "loss": 0.2188, + "step": 795 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019833605024808382, + "loss": 0.2128, + "step": 800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019830411198638607, + "loss": 0.2109, + "step": 805 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019827187273689894, + "loss": 0.2141, + "step": 810 + }, + { + "epoch": 0.26, + "learning_rate": 0.0001982393325983326, + "loss": 0.2098, + "step": 815 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019820649167031856, + "loss": 0.2188, + "step": 820 + }, + { + "epoch": 0.27, + "learning_rate": 0.0001981733500534093, + "loss": 0.2107, + "step": 825 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019813990784907785, + "loss": 0.2162, + "step": 830 + }, + { + "epoch": 0.27, + "learning_rate": 0.0001981061651597177, + "loss": 0.2189, + "step": 835 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019807212208864227, + "loss": 0.2117, + "step": 840 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019803777874008468, + "loss": 0.209, + "step": 845 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019800313521919753, + "loss": 0.206, + "step": 850 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019796819163205246, + "loss": 0.2104, + "step": 855 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019793294808563984, + "loss": 0.2016, + "step": 860 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019789740468786841, + "loss": 0.212, + "step": 865 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019786156154756506, + "loss": 0.2051, + "step": 870 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019782541877447443, + "loss": 0.21, + "step": 875 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019778897647925855, + "loss": 0.2104, + "step": 880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0001977522347734965, + "loss": 0.2179, + "step": 885 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019771519376968416, + "loss": 0.2139, + "step": 890 + }, + { + "epoch": 0.29, + "learning_rate": 0.0001976778535812338, + "loss": 0.2034, + "step": 895 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019764021432247368, + "loss": 0.2067, + "step": 900 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019760227610864776, + "loss": 0.2189, + "step": 905 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019756403905591538, + "loss": 0.2119, + "step": 910 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019752550328135086, + "loss": 0.2004, + "step": 915 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019748666890294315, + "loss": 0.2073, + "step": 920 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019744753603959545, + "loss": 0.1995, + "step": 925 + }, + { + "epoch": 0.3, + "learning_rate": 0.0001974081048111248, + "loss": 0.2034, + "step": 930 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019736837533826192, + "loss": 0.2015, + "step": 935 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019732834774265057, + "loss": 0.2166, + "step": 940 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019728802214684734, + "loss": 0.1987, + "step": 945 + }, + { + "epoch": 0.31, + "learning_rate": 0.0001972473986743213, + "loss": 0.2051, + "step": 950 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019720647744945344, + "loss": 0.2055, + "step": 955 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019716525859753652, + "loss": 0.2005, + "step": 960 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019712374224477452, + "loss": 0.1995, + "step": 965 + }, + { + "epoch": 0.31, + "learning_rate": 0.0001970819285182823, + "loss": 0.2096, + "step": 970 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019703981754608525, + "loss": 0.2024, + "step": 975 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019699740945711884, + "loss": 0.2006, + "step": 980 + }, + { + "epoch": 0.32, + "learning_rate": 0.0001969547043812283, + "loss": 0.2077, + "step": 985 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019691170244916814, + "loss": 0.2094, + "step": 990 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019686840379260176, + "loss": 0.2051, + "step": 995 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019682480854410115, + "loss": 0.2158, + "step": 1000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0001967809168371463, + "loss": 0.2035, + "step": 1005 + }, + { + "epoch": 0.33, + "learning_rate": 0.000196736728806125, + "loss": 0.2104, + "step": 1010 + }, + { + "epoch": 0.33, + "learning_rate": 0.00019669224458633222, + "loss": 0.1969, + "step": 1015 + }, + { + "epoch": 0.33, + "learning_rate": 0.0001966474643139699, + "loss": 0.2058, + "step": 1020 + }, + { + "epoch": 0.33, + "learning_rate": 0.00019660238812614647, + "loss": 0.2045, + "step": 1025 + }, + { + "epoch": 0.33, + "learning_rate": 0.0001965570161608762, + "loss": 0.1916, + "step": 1030 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019651134855707913, + "loss": 0.2008, + "step": 1035 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019646538545458047, + "loss": 0.2095, + "step": 1040 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019641912699411016, + "loss": 0.2009, + "step": 1045 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019637257331730248, + "loss": 0.2059, + "step": 1050 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019632572456669558, + "loss": 0.2084, + "step": 1055 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019627858088573112, + "loss": 0.1997, + "step": 1060 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001962311424187537, + "loss": 0.2068, + "step": 1065 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001961834093110106, + "loss": 0.1978, + "step": 1070 + }, + { + "epoch": 0.35, + "learning_rate": 0.00019613538170865118, + "loss": 0.2028, + "step": 1075 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001960870597587264, + "loss": 0.1996, + "step": 1080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001960384436091886, + "loss": 0.2002, + "step": 1085 + }, + { + "epoch": 0.35, + "learning_rate": 0.00019598953340889082, + "loss": 0.198, + "step": 1090 + }, + { + "epoch": 0.35, + "learning_rate": 0.00019594032930758643, + "loss": 0.1974, + "step": 1095 + }, + { + "epoch": 0.36, + "learning_rate": 0.0001958908314559287, + "loss": 0.1983, + "step": 1100 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019584104000547025, + "loss": 0.1987, + "step": 1105 + }, + { + "epoch": 0.36, + "learning_rate": 0.0001957909551086627, + "loss": 0.2003, + "step": 1110 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019574057691885613, + "loss": 0.1982, + "step": 1115 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019568990559029858, + "loss": 0.2031, + "step": 1120 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019563894127813564, + "loss": 0.1969, + "step": 1125 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019558768413841006, + "loss": 0.1916, + "step": 1130 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019553613432806102, + "loss": 0.2124, + "step": 1135 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019548429200492388, + "loss": 0.2042, + "step": 1140 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019543215732772956, + "loss": 0.201, + "step": 1145 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019537973045610424, + "loss": 0.2001, + "step": 1150 + }, + { + "epoch": 0.37, + "learning_rate": 0.0001953270115505686, + "loss": 0.1978, + "step": 1155 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019527400077253755, + "loss": 0.192, + "step": 1160 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019522069828431957, + "loss": 0.1946, + "step": 1165 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019516710424911642, + "loss": 0.1848, + "step": 1170 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019511321883102244, + "loss": 0.2005, + "step": 1175 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019505904219502414, + "loss": 0.1966, + "step": 1180 + }, + { + "epoch": 0.38, + "learning_rate": 0.0001950045745069997, + "loss": 0.1863, + "step": 1185 + }, + { + "epoch": 0.39, + "learning_rate": 0.0001949498159337184, + "loss": 0.1983, + "step": 1190 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019489476664284023, + "loss": 0.2113, + "step": 1195 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019483942680291523, + "loss": 0.1956, + "step": 1200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019478379658338305, + "loss": 0.2055, + "step": 1205 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019472787615457246, + "loss": 0.1879, + "step": 1210 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019467166568770078, + "loss": 0.1924, + "step": 1215 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019461516535487338, + "loss": 0.1971, + "step": 1220 + }, + { + "epoch": 0.4, + "learning_rate": 0.0001945583753290831, + "loss": 0.1952, + "step": 1225 + }, + { + "epoch": 0.4, + "learning_rate": 0.0001945012957842098, + "loss": 0.1838, + "step": 1230 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019444392689501978, + "loss": 0.1931, + "step": 1235 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019438626883716526, + "loss": 0.2093, + "step": 1240 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019432832178718385, + "loss": 0.1992, + "step": 1245 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019427008592249795, + "loss": 0.1963, + "step": 1250 + }, + { + "epoch": 0.41, + "learning_rate": 0.0001942115614214143, + "loss": 0.2003, + "step": 1255 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019415274846312338, + "loss": 0.1914, + "step": 1260 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019409364722769882, + "loss": 0.1917, + "step": 1265 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019403425789609697, + "loss": 0.2033, + "step": 1270 + }, + { + "epoch": 0.41, + "learning_rate": 0.0001939745806501563, + "loss": 0.1975, + "step": 1275 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019391461567259662, + "loss": 0.1922, + "step": 1280 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019385436314701897, + "loss": 0.1999, + "step": 1285 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019379382325790468, + "loss": 0.1997, + "step": 1290 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019373299619061487, + "loss": 0.1872, + "step": 1295 + }, + { + "epoch": 0.42, + "learning_rate": 0.0001936718821313901, + "loss": 0.1936, + "step": 1300 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019361048126734952, + "loss": 0.1798, + "step": 1305 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019354879378649046, + "loss": 0.1959, + "step": 1310 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019348681987768783, + "loss": 0.1924, + "step": 1315 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019342455973069345, + "loss": 0.1967, + "step": 1320 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019336201353613568, + "loss": 0.1925, + "step": 1325 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019329918148551856, + "loss": 0.1878, + "step": 1330 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019323606377122146, + "loss": 0.1979, + "step": 1335 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019317266058649834, + "loss": 0.189, + "step": 1340 + }, + { + "epoch": 0.44, + "learning_rate": 0.0001931089721254773, + "loss": 0.1995, + "step": 1345 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019304499858315976, + "loss": 0.1874, + "step": 1350 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019298074015542015, + "loss": 0.1991, + "step": 1355 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019291619703900502, + "loss": 0.188, + "step": 1360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0001928513694315327, + "loss": 0.1914, + "step": 1365 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019278625753149254, + "loss": 0.1843, + "step": 1370 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019272086153824427, + "loss": 0.1959, + "step": 1375 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019265518165201758, + "loss": 0.1867, + "step": 1380 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019258921807391131, + "loss": 0.1956, + "step": 1385 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019252297100589286, + "loss": 0.1991, + "step": 1390 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019245644065079775, + "loss": 0.1934, + "step": 1395 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019238962721232876, + "loss": 0.1953, + "step": 1400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019232253089505546, + "loss": 0.1865, + "step": 1405 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019225515190441354, + "loss": 0.1864, + "step": 1410 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019218749044670413, + "loss": 0.194, + "step": 1415 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019211954672909327, + "loss": 0.1853, + "step": 1420 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019205132095961126, + "loss": 0.1967, + "step": 1425 + }, + { + "epoch": 0.46, + "learning_rate": 0.0001919828133471519, + "loss": 0.1852, + "step": 1430 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019191402410147204, + "loss": 0.1913, + "step": 1435 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019184495343319068, + "loss": 0.1911, + "step": 1440 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019177560155378865, + "loss": 0.189, + "step": 1445 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019170596867560766, + "loss": 0.1854, + "step": 1450 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019163605501184986, + "loss": 0.1974, + "step": 1455 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019156586077657712, + "loss": 0.1933, + "step": 1460 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019149538618471032, + "loss": 0.1877, + "step": 1465 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001914246314520287, + "loss": 0.1904, + "step": 1470 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001913535967951694, + "loss": 0.1841, + "step": 1475 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001912822824316264, + "loss": 0.1876, + "step": 1480 + }, + { + "epoch": 0.48, + "learning_rate": 0.00019121068857975027, + "loss": 0.1858, + "step": 1485 + }, + { + "epoch": 0.48, + "learning_rate": 0.00019113881545874722, + "loss": 0.1898, + "step": 1490 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001910666632886786, + "loss": 0.1923, + "step": 1495 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019099423229046014, + "loss": 0.1803, + "step": 1500 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019092152268586118, + "loss": 0.1945, + "step": 1505 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019084853469750424, + "loss": 0.1802, + "step": 1510 + }, + { + "epoch": 0.49, + "learning_rate": 0.0001907752685488641, + "loss": 0.197, + "step": 1515 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019070172446426724, + "loss": 0.1791, + "step": 1520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019062790266889117, + "loss": 0.1874, + "step": 1525 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019055380338876363, + "loss": 0.1806, + "step": 1530 + }, + { + "epoch": 0.5, + "learning_rate": 0.000190479426850762, + "loss": 0.1834, + "step": 1535 + }, + { + "epoch": 0.5, + "learning_rate": 0.0001904047732826126, + "loss": 0.1921, + "step": 1540 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019032984291288986, + "loss": 0.1826, + "step": 1545 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019025463597101584, + "loss": 0.1966, + "step": 1550 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019017915268725925, + "loss": 0.1867, + "step": 1555 + }, + { + "epoch": 0.51, + "learning_rate": 0.00019010339329273514, + "loss": 0.196, + "step": 1560 + }, + { + "epoch": 0.51, + "learning_rate": 0.00019002735801940373, + "loss": 0.1971, + "step": 1565 + }, + { + "epoch": 0.51, + "learning_rate": 0.00018995104710007004, + "loss": 0.187, + "step": 1570 + }, + { + "epoch": 0.51, + "learning_rate": 0.00018987446076838312, + "loss": 0.1897, + "step": 1575 + }, + { + "epoch": 0.51, + "learning_rate": 0.00018979759925883509, + "loss": 0.1887, + "step": 1580 + }, + { + "epoch": 0.51, + "learning_rate": 0.0001897204628067608, + "loss": 0.1849, + "step": 1585 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018964305164833678, + "loss": 0.1795, + "step": 1590 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018956536602058078, + "loss": 0.1984, + "step": 1595 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018948740616135085, + "loss": 0.1876, + "step": 1600 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018940917230934464, + "loss": 0.1879, + "step": 1605 + }, + { + "epoch": 0.52, + "learning_rate": 0.0001893306647040988, + "loss": 0.1906, + "step": 1610 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018925188358598813, + "loss": 0.1917, + "step": 1615 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018917282919622488, + "loss": 0.1912, + "step": 1620 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018909350177685795, + "loss": 0.1813, + "step": 1625 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018901390157077225, + "loss": 0.183, + "step": 1630 + }, + { + "epoch": 0.53, + "learning_rate": 0.0001889340288216879, + "loss": 0.1782, + "step": 1635 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018885388377415957, + "loss": 0.1866, + "step": 1640 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018877346667357548, + "loss": 0.1915, + "step": 1645 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018869277776615691, + "loss": 0.1902, + "step": 1650 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018861181729895745, + "loss": 0.1829, + "step": 1655 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018853058551986202, + "loss": 0.1855, + "step": 1660 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018844908267758633, + "loss": 0.1845, + "step": 1665 + }, + { + "epoch": 0.54, + "learning_rate": 0.000188367309021676, + "loss": 0.1794, + "step": 1670 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018828526480250575, + "loss": 0.1796, + "step": 1675 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018820295027127885, + "loss": 0.1806, + "step": 1680 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018812036568002617, + "loss": 0.1846, + "step": 1685 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018803751128160538, + "loss": 0.189, + "step": 1690 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018795438732970032, + "loss": 0.1856, + "step": 1695 + }, + { + "epoch": 0.55, + "learning_rate": 0.0001878709940788201, + "loss": 0.1797, + "step": 1700 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018778733178429844, + "loss": 0.1876, + "step": 1705 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018770340070229272, + "loss": 0.1775, + "step": 1710 + }, + { + "epoch": 0.56, + "learning_rate": 0.0001876192010897834, + "loss": 0.1862, + "step": 1715 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018753473320457306, + "loss": 0.1835, + "step": 1720 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018744999730528568, + "loss": 0.1909, + "step": 1725 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018736499365136584, + "loss": 0.189, + "step": 1730 + }, + { + "epoch": 0.56, + "learning_rate": 0.000187279722503078, + "loss": 0.1861, + "step": 1735 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018719418412150557, + "loss": 0.1809, + "step": 1740 + }, + { + "epoch": 0.57, + "learning_rate": 0.00018710837876855017, + "loss": 0.1829, + "step": 1745 + }, + { + "epoch": 0.57, + "learning_rate": 0.0001870223067069309, + "loss": 0.1886, + "step": 1750 + }, + { + "epoch": 0.57, + "learning_rate": 0.00018693596820018336, + "loss": 0.1787, + "step": 1755 + }, + { + "epoch": 0.57, + "learning_rate": 0.0001868493635126591, + "loss": 0.1816, + "step": 1760 + }, + { + "epoch": 0.57, + "learning_rate": 0.0001867624929095245, + "loss": 0.1759, + "step": 1765 + }, + { + "epoch": 0.57, + "learning_rate": 0.00018667535665676025, + "loss": 0.1702, + "step": 1770 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018658795502116035, + "loss": 0.1802, + "step": 1775 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018650028827033136, + "loss": 0.1836, + "step": 1780 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018641235667269155, + "loss": 0.1787, + "step": 1785 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018632416049747016, + "loss": 0.1811, + "step": 1790 + }, + { + "epoch": 0.58, + "learning_rate": 0.0001862357000147064, + "loss": 0.1835, + "step": 1795 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018614697549524889, + "loss": 0.1857, + "step": 1800 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018605798721075455, + "loss": 0.1867, + "step": 1805 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018596873543368797, + "loss": 0.1766, + "step": 1810 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018587922043732046, + "loss": 0.1895, + "step": 1815 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018578944249572925, + "loss": 0.1888, + "step": 1820 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018569940188379672, + "loss": 0.1871, + "step": 1825 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018560909887720944, + "loss": 0.1804, + "step": 1830 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018551853375245737, + "loss": 0.1877, + "step": 1835 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018542770678683308, + "loss": 0.182, + "step": 1840 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001853366182584308, + "loss": 0.1809, + "step": 1845 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018524526844614564, + "loss": 0.1913, + "step": 1850 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018515365762967266, + "loss": 0.185, + "step": 1855 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018506178608950613, + "loss": 0.1753, + "step": 1860 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018496965410693856, + "loss": 0.1786, + "step": 1865 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001848772619640599, + "loss": 0.1841, + "step": 1870 + }, + { + "epoch": 0.61, + "learning_rate": 0.00018478460994375667, + "loss": 0.184, + "step": 1875 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001846916983297111, + "loss": 0.179, + "step": 1880 + }, + { + "epoch": 0.61, + "learning_rate": 0.00018459852740640014, + "loss": 0.1833, + "step": 1885 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001845050974590948, + "loss": 0.186, + "step": 1890 + }, + { + "epoch": 0.61, + "learning_rate": 0.00018441140877385913, + "loss": 0.1778, + "step": 1895 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018431746163754943, + "loss": 0.1786, + "step": 1900 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018422325633781327, + "loss": 0.1826, + "step": 1905 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018412879316308866, + "loss": 0.1738, + "step": 1910 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018403407240260327, + "loss": 0.1852, + "step": 1915 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018393909434637329, + "loss": 0.1821, + "step": 1920 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001838438592852028, + "loss": 0.1773, + "step": 1925 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018374836751068282, + "loss": 0.1772, + "step": 1930 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018365261931519022, + "loss": 0.1766, + "step": 1935 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018355661499188717, + "loss": 0.1858, + "step": 1940 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018346035483471994, + "loss": 0.1786, + "step": 1945 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001833638391384181, + "loss": 0.1745, + "step": 1950 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018326706819849365, + "loss": 0.1713, + "step": 1955 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018317004231124017, + "loss": 0.1738, + "step": 1960 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018307276177373174, + "loss": 0.1807, + "step": 1965 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018297522688382218, + "loss": 0.1806, + "step": 1970 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018287743794014407, + "loss": 0.1735, + "step": 1975 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018277939524210783, + "loss": 0.1812, + "step": 1980 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001826810990899009, + "loss": 0.1734, + "step": 1985 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001825825497844867, + "loss": 0.1703, + "step": 1990 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018248374762760365, + "loss": 0.1823, + "step": 1995 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018238469292176457, + "loss": 0.1837, + "step": 2000 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018228538597025537, + "loss": 0.1807, + "step": 2005 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018218582707713434, + "loss": 0.1904, + "step": 2010 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001820860165472312, + "loss": 0.1844, + "step": 2015 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018198595468614604, + "loss": 0.1792, + "step": 2020 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018188564180024854, + "loss": 0.1845, + "step": 2025 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018178507819667698, + "loss": 0.1853, + "step": 2030 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018168426418333725, + "loss": 0.1875, + "step": 2035 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018158320006890198, + "loss": 0.1759, + "step": 2040 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018148188616280955, + "loss": 0.1816, + "step": 2045 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001813803227752631, + "loss": 0.1787, + "step": 2050 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001812785102172297, + "loss": 0.1758, + "step": 2055 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018117644880043932, + "loss": 0.1802, + "step": 2060 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018107413883738387, + "loss": 0.186, + "step": 2065 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018097158064131626, + "loss": 0.1743, + "step": 2070 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018086877452624944, + "loss": 0.1877, + "step": 2075 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018076572080695543, + "loss": 0.1742, + "step": 2080 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018066241979896444, + "loss": 0.1727, + "step": 2085 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018055887181856375, + "loss": 0.1806, + "step": 2090 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018045507718279677, + "loss": 0.175, + "step": 2095 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018035103620946225, + "loss": 0.1727, + "step": 2100 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001802467492171131, + "loss": 0.1779, + "step": 2105 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018014221652505556, + "loss": 0.1823, + "step": 2110 + }, + { + "epoch": 0.69, + "learning_rate": 0.000180037438453348, + "loss": 0.1814, + "step": 2115 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001799324153228003, + "loss": 0.1707, + "step": 2120 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001798271474549725, + "loss": 0.1778, + "step": 2125 + }, + { + "epoch": 0.69, + "learning_rate": 0.000179721635172174, + "loss": 0.1888, + "step": 2130 + }, + { + "epoch": 0.69, + "learning_rate": 0.00017961587879746266, + "loss": 0.1785, + "step": 2135 + }, + { + "epoch": 0.69, + "learning_rate": 0.00017950987865464365, + "loss": 0.1661, + "step": 2140 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017940363506826836, + "loss": 0.1792, + "step": 2145 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001792971483636338, + "loss": 0.1732, + "step": 2150 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017919041886678118, + "loss": 0.1771, + "step": 2155 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001790834469044952, + "loss": 0.173, + "step": 2160 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017897623280430284, + "loss": 0.1768, + "step": 2165 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017886877689447258, + "loss": 0.175, + "step": 2170 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001787610795040132, + "loss": 0.1691, + "step": 2175 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001786531409626728, + "loss": 0.1774, + "step": 2180 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001785449616009379, + "loss": 0.1709, + "step": 2185 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001784365417500324, + "loss": 0.1809, + "step": 2190 + }, + { + "epoch": 0.71, + "learning_rate": 0.00017832788174191645, + "loss": 0.1764, + "step": 2195 + }, + { + "epoch": 0.71, + "learning_rate": 0.00017821898190928558, + "loss": 0.1739, + "step": 2200 + }, + { + "epoch": 0.71, + "learning_rate": 0.00017810984258556957, + "loss": 0.1772, + "step": 2205 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017800046410493145, + "loss": 0.1779, + "step": 2210 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001778908468022666, + "loss": 0.1828, + "step": 2215 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017778099101320156, + "loss": 0.1722, + "step": 2220 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017767089707409304, + "loss": 0.1737, + "step": 2225 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017756056532202696, + "loss": 0.1688, + "step": 2230 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017744999609481738, + "loss": 0.1784, + "step": 2235 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017733918973100545, + "loss": 0.1747, + "step": 2240 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017722814656985841, + "loss": 0.1667, + "step": 2245 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001771168669513685, + "loss": 0.1648, + "step": 2250 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017700535121625187, + "loss": 0.1777, + "step": 2255 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017689359970594779, + "loss": 0.175, + "step": 2260 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001767816127626173, + "loss": 0.1715, + "step": 2265 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001766693907291423, + "loss": 0.1742, + "step": 2270 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017655693394912452, + "loss": 0.1868, + "step": 2275 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017644424276688441, + "loss": 0.1757, + "step": 2280 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017633131752746017, + "loss": 0.1803, + "step": 2285 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017621815857660656, + "loss": 0.1812, + "step": 2290 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017610476626079395, + "loss": 0.1691, + "step": 2295 + }, + { + "epoch": 0.75, + "learning_rate": 0.00017599114092720726, + "loss": 0.1662, + "step": 2300 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001758772829237448, + "loss": 0.1709, + "step": 2305 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001757631925990174, + "loss": 0.1822, + "step": 2310 + }, + { + "epoch": 0.75, + "learning_rate": 0.00017564887030234696, + "loss": 0.1732, + "step": 2315 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001755343163837659, + "loss": 0.1854, + "step": 2320 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001754195311940157, + "loss": 0.1768, + "step": 2325 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017530451508454594, + "loss": 0.1785, + "step": 2330 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017518926840751325, + "loss": 0.1724, + "step": 2335 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017507379151578017, + "loss": 0.1812, + "step": 2340 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017495808476291423, + "loss": 0.1754, + "step": 2345 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017484214850318664, + "loss": 0.1799, + "step": 2350 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001747259830915713, + "loss": 0.1769, + "step": 2355 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017460958888374385, + "loss": 0.1764, + "step": 2360 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001744929662360803, + "loss": 0.1716, + "step": 2365 + }, + { + "epoch": 0.77, + "learning_rate": 0.00017437611550565624, + "loss": 0.1699, + "step": 2370 + }, + { + "epoch": 0.77, + "learning_rate": 0.00017425903705024553, + "loss": 0.1661, + "step": 2375 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001741417312283193, + "loss": 0.1699, + "step": 2380 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001740241983990448, + "loss": 0.1682, + "step": 2385 + }, + { + "epoch": 0.77, + "learning_rate": 0.00017390643892228438, + "loss": 0.173, + "step": 2390 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001737884531585943, + "loss": 0.1852, + "step": 2395 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017367024146922372, + "loss": 0.1681, + "step": 2400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017355180421611347, + "loss": 0.1798, + "step": 2405 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017343314176189502, + "loss": 0.1705, + "step": 2410 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017331425446988946, + "loss": 0.177, + "step": 2415 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017319514270410614, + "loss": 0.1754, + "step": 2420 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017307580682924178, + "loss": 0.1679, + "step": 2425 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017295624721067932, + "loss": 0.1763, + "step": 2430 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017283646421448673, + "loss": 0.1769, + "step": 2435 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017271645820741583, + "loss": 0.1753, + "step": 2440 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001725962295569014, + "loss": 0.1715, + "step": 2445 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017247577863105983, + "loss": 0.1784, + "step": 2450 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017235510579868802, + "loss": 0.17, + "step": 2455 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017223421142926245, + "loss": 0.1686, + "step": 2460 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017211309589293774, + "loss": 0.166, + "step": 2465 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017199175956054584, + "loss": 0.1652, + "step": 2470 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017187020280359455, + "loss": 0.1732, + "step": 2475 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001717484259942668, + "loss": 0.1754, + "step": 2480 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017162642950541902, + "loss": 0.1748, + "step": 2485 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017150421371058049, + "loss": 0.1733, + "step": 2490 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017138177898395177, + "loss": 0.1737, + "step": 2495 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017125912570040383, + "loss": 0.175, + "step": 2500 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001711362542354769, + "loss": 0.173, + "step": 2505 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017101316496537905, + "loss": 0.1796, + "step": 2510 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017088985826698542, + "loss": 0.1722, + "step": 2515 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017076633451783675, + "loss": 0.1738, + "step": 2520 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017064259409613834, + "loss": 0.1743, + "step": 2525 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017051863738075902, + "loss": 0.1653, + "step": 2530 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001703944647512297, + "loss": 0.1761, + "step": 2535 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017027007658774256, + "loss": 0.1649, + "step": 2540 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017014547327114956, + "loss": 0.1738, + "step": 2545 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001700206551829614, + "loss": 0.1585, + "step": 2550 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016989562270534655, + "loss": 0.183, + "step": 2555 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001697703762211297, + "loss": 0.1693, + "step": 2560 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001696449161137909, + "loss": 0.173, + "step": 2565 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016951924276746425, + "loss": 0.1691, + "step": 2570 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016939335656693667, + "loss": 0.1698, + "step": 2575 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016926725789764691, + "loss": 0.1706, + "step": 2580 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016914094714568417, + "loss": 0.1691, + "step": 2585 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016901442469778713, + "loss": 0.1665, + "step": 2590 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016888769094134247, + "loss": 0.1831, + "step": 2595 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016876074626438395, + "loss": 0.1637, + "step": 2600 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016863359105559117, + "loss": 0.1757, + "step": 2605 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016850622570428826, + "loss": 0.1721, + "step": 2610 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016837865060044272, + "loss": 0.1647, + "step": 2615 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016825086613466442, + "loss": 0.1734, + "step": 2620 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016812287269820416, + "loss": 0.1847, + "step": 2625 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016799467068295257, + "loss": 0.1682, + "step": 2630 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001678662604814389, + "loss": 0.1741, + "step": 2635 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016773764248682988, + "loss": 0.1692, + "step": 2640 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016760881709292842, + "loss": 0.1805, + "step": 2645 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001674797846941724, + "loss": 0.1685, + "step": 2650 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016735054568563364, + "loss": 0.1673, + "step": 2655 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001672211004630164, + "loss": 0.1686, + "step": 2660 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016709144942265644, + "loss": 0.1749, + "step": 2665 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001669615929615197, + "loss": 0.1644, + "step": 2670 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016683153147720097, + "loss": 0.1673, + "step": 2675 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016670126536792287, + "loss": 0.1722, + "step": 2680 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001665707950325345, + "loss": 0.1588, + "step": 2685 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001664401208705103, + "loss": 0.164, + "step": 2690 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016630924328194877, + "loss": 0.1708, + "step": 2695 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016617816266757123, + "loss": 0.1711, + "step": 2700 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016604687942872067, + "loss": 0.1701, + "step": 2705 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016591539396736046, + "loss": 0.1708, + "step": 2710 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016578370668607307, + "loss": 0.1669, + "step": 2715 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016565181798805907, + "loss": 0.1601, + "step": 2720 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016551972827713555, + "loss": 0.1733, + "step": 2725 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016538743795773515, + "loss": 0.1641, + "step": 2730 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016525494743490474, + "loss": 0.1723, + "step": 2735 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016512225711430415, + "loss": 0.1655, + "step": 2740 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016498936740220492, + "loss": 0.172, + "step": 2745 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016485627870548917, + "loss": 0.1686, + "step": 2750 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016472299143164816, + "loss": 0.1709, + "step": 2755 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016458950598878126, + "loss": 0.1648, + "step": 2760 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016445582278559458, + "loss": 0.1641, + "step": 2765 + }, + { + "epoch": 0.9, + "learning_rate": 0.0001643219422313996, + "loss": 0.1662, + "step": 2770 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016418786473611227, + "loss": 0.1763, + "step": 2775 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016405359071025138, + "loss": 0.1773, + "step": 2780 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016391912056493743, + "loss": 0.1678, + "step": 2785 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016378445471189156, + "loss": 0.1632, + "step": 2790 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016364959356343398, + "loss": 0.1671, + "step": 2795 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016351453753248296, + "loss": 0.1739, + "step": 2800 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016337928703255336, + "loss": 0.1727, + "step": 2805 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016324384247775554, + "loss": 0.1756, + "step": 2810 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016310820428279397, + "loss": 0.1671, + "step": 2815 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016297237286296606, + "loss": 0.169, + "step": 2820 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016283634863416082, + "loss": 0.1693, + "step": 2825 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016270013201285757, + "loss": 0.164, + "step": 2830 + }, + { + "epoch": 0.92, + "learning_rate": 0.0001625637234161247, + "loss": 0.1663, + "step": 2835 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016242712326161845, + "loss": 0.1742, + "step": 2840 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016229033196758154, + "loss": 0.169, + "step": 2845 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016215334995284188, + "loss": 0.1703, + "step": 2850 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001620161776368114, + "loss": 0.1669, + "step": 2855 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001618788154394846, + "loss": 0.1634, + "step": 2860 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001617412637814375, + "loss": 0.163, + "step": 2865 + }, + { + "epoch": 0.93, + "learning_rate": 0.00016160352308382608, + "loss": 0.1651, + "step": 2870 + }, + { + "epoch": 0.93, + "learning_rate": 0.00016146559376838514, + "loss": 0.1657, + "step": 2875 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001613274762574271, + "loss": 0.1638, + "step": 2880 + }, + { + "epoch": 0.93, + "learning_rate": 0.00016118917097384048, + "loss": 0.1691, + "step": 2885 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016105067834108874, + "loss": 0.1627, + "step": 2890 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016091199878320902, + "loss": 0.1608, + "step": 2895 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016077313272481076, + "loss": 0.1617, + "step": 2900 + }, + { + "epoch": 0.94, + "learning_rate": 0.0001606340805910744, + "loss": 0.1696, + "step": 2905 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016049484280775012, + "loss": 0.1604, + "step": 2910 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016035541980115653, + "loss": 0.161, + "step": 2915 + }, + { + "epoch": 0.95, + "learning_rate": 0.00016021581199817943, + "loss": 0.1703, + "step": 2920 + }, + { + "epoch": 0.95, + "learning_rate": 0.0001600760198262703, + "loss": 0.1671, + "step": 2925 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015993604371344517, + "loss": 0.1603, + "step": 2930 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015979588408828333, + "loss": 0.1655, + "step": 2935 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015965554137992584, + "loss": 0.1675, + "step": 2940 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015951501601807437, + "loss": 0.1641, + "step": 2945 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015937430843298986, + "loss": 0.165, + "step": 2950 + }, + { + "epoch": 0.96, + "learning_rate": 0.0001592334190554912, + "loss": 0.1758, + "step": 2955 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015909234831695379, + "loss": 0.1703, + "step": 2960 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015895109664930846, + "loss": 0.1644, + "step": 2965 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015880966448503987, + "loss": 0.1638, + "step": 2970 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015866805225718542, + "loss": 0.1659, + "step": 2975 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015852626039933383, + "loss": 0.1676, + "step": 2980 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015838428934562373, + "loss": 0.1794, + "step": 2985 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015824213953074254, + "loss": 0.1562, + "step": 2990 + }, + { + "epoch": 0.97, + "learning_rate": 0.0001580998113899249, + "loss": 0.171, + "step": 2995 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015795730535895147, + "loss": 0.1681, + "step": 3000 + } + ], + "logging_steps": 5, + "max_steps": 9255, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 1000, + "total_flos": 6.110094390603022e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/save_lora_math_r8_2e4/checkpoint-3000/training_args.bin b/save_lora_math_r8_2e4/checkpoint-3000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..175cae96ce99657f1e2b29179eee355d72511307 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-3000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0cedf67f015d3f83c601cebdd463c220d745d8e241fab282366821363eb224d +size 5880 diff --git a/save_lora_math_r8_2e4/checkpoint-3000/zero_to_fp32.py b/save_lora_math_r8_2e4/checkpoint-3000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..c98caae31534368be22b67fc4ae906836c992a8d --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-3000/zero_to_fp32.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in param_shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, args.output_file, tag=args.tag) diff --git a/save_lora_math_r8_2e4/checkpoint-4000/README.md b/save_lora_math_r8_2e4/checkpoint-4000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d380bbbf0bf0fa1f211b53410559607e5007a209 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-4000/README.md @@ -0,0 +1,202 @@ +--- +library_name: peft +base_model: meta-llama/Llama-2-7b-hf +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.9.0 \ No newline at end of file diff --git a/save_lora_math_r8_2e4/checkpoint-4000/adapter_config.json b/save_lora_math_r8_2e4/checkpoint-4000/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9c81a592f0ff08374a4f61f6003029070a8429ae --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-4000/adapter_config.json @@ -0,0 +1,35 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "meta-llama/Llama-2-7b-hf", + "bias": "none", + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "mora_type": 1, + "peft_type": "LORA", + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "down_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "use_dora": false, + "use_mora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/save_lora_math_r8_2e4/checkpoint-4000/adapter_model.safetensors b/save_lora_math_r8_2e4/checkpoint-4000/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..663cde585c70933f6dcbbf0f0994e3396d958c33 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-4000/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:244e17e5b3031bcc31c85c63c530ccebb3e084bca923162a1937fcd3ae7572b9 +size 40036488 diff --git a/save_lora_math_r8_2e4/checkpoint-4000/global_step4000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/save_lora_math_r8_2e4/checkpoint-4000/global_step4000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..643720536d2da1daeef16f7264873f152933d4f7 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-4000/global_step4000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff5fdf0cb6f1f59110bef33a71b251739a5cd5dc2cdd021bc1f8b92491f39e69 +size 119958768 diff --git a/save_lora_math_r8_2e4/checkpoint-4000/global_step4000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/save_lora_math_r8_2e4/checkpoint-4000/global_step4000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b55460b79a2b4cf6aca51387fbce0a41ac10405 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-4000/global_step4000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ae0c406ba5f418f0242cfce3cd741b343e3d1fc9f68b031e4da9ca4f30f5f53 +size 119958896 diff --git a/save_lora_math_r8_2e4/checkpoint-4000/global_step4000/mp_rank_00_model_states.pt b/save_lora_math_r8_2e4/checkpoint-4000/global_step4000/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2376325567d662080fea3e374d024b7bdcee3632 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-4000/global_step4000/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e4afaa03863f7dc68ba57f88fc966df0016389618dc5e0c96667605b799709c +size 40303404 diff --git a/save_lora_math_r8_2e4/checkpoint-4000/latest b/save_lora_math_r8_2e4/checkpoint-4000/latest new file mode 100644 index 0000000000000000000000000000000000000000..641c40ce8d5e05e04a0840d69bcc5fac045a3fbd --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-4000/latest @@ -0,0 +1 @@ +global_step4000 \ No newline at end of file diff --git a/save_lora_math_r8_2e4/checkpoint-4000/rng_state_0.pth b/save_lora_math_r8_2e4/checkpoint-4000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..ecd5a82221b897d646746dabecf536625885ed7b --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-4000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81b62990b0acd827d12ace95544582e98aaf5a0c235ca2a22d2703cecb6a0111 +size 14512 diff --git a/save_lora_math_r8_2e4/checkpoint-4000/rng_state_1.pth b/save_lora_math_r8_2e4/checkpoint-4000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..43c1f245a8d5782bf096ff0041ca3a84d8d30173 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-4000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f19aba25c63cf169d999174ce369fc4dd4919e44ffcf553e0c7ff57af0d0b285 +size 14512 diff --git a/save_lora_math_r8_2e4/checkpoint-4000/scheduler.pt b/save_lora_math_r8_2e4/checkpoint-4000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..e06f8faaf8d022bef687cd074736f823a93f5ffe --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-4000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48d8572d1c5bc07014c2505c757ef5a5c249eb50ca481df9ab0a995d53e1424b +size 1064 diff --git a/save_lora_math_r8_2e4/checkpoint-4000/trainer_state.json b/save_lora_math_r8_2e4/checkpoint-4000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..433a2db3ce7c92cdf5b5f7f38772554493813ce1 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-4000/trainer_state.json @@ -0,0 +1,4821 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.2962025316455696, + "eval_steps": 500, + "global_step": 4000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 3.5971223021582732e-06, + "loss": 0.7628, + "step": 5 + }, + { + "epoch": 0.0, + "learning_rate": 7.1942446043165465e-06, + "loss": 0.769, + "step": 10 + }, + { + "epoch": 0.0, + "learning_rate": 1.0791366906474821e-05, + "loss": 0.7619, + "step": 15 + }, + { + "epoch": 0.01, + "learning_rate": 1.4388489208633093e-05, + "loss": 0.7564, + "step": 20 + }, + { + "epoch": 0.01, + "learning_rate": 1.7985611510791367e-05, + "loss": 0.7549, + "step": 25 + }, + { + "epoch": 0.01, + "learning_rate": 2.1582733812949642e-05, + "loss": 0.7001, + "step": 30 + }, + { + "epoch": 0.01, + "learning_rate": 2.5179856115107914e-05, + "loss": 0.6289, + "step": 35 + }, + { + "epoch": 0.01, + "learning_rate": 2.8776978417266186e-05, + "loss": 0.5632, + "step": 40 + }, + { + "epoch": 0.01, + "learning_rate": 3.237410071942446e-05, + "loss": 0.4923, + "step": 45 + }, + { + "epoch": 0.02, + "learning_rate": 3.597122302158273e-05, + "loss": 0.4582, + "step": 50 + }, + { + "epoch": 0.02, + "learning_rate": 3.956834532374101e-05, + "loss": 0.4357, + "step": 55 + }, + { + "epoch": 0.02, + "learning_rate": 4.3165467625899284e-05, + "loss": 0.4079, + "step": 60 + }, + { + "epoch": 0.02, + "learning_rate": 4.676258992805755e-05, + "loss": 0.4047, + "step": 65 + }, + { + "epoch": 0.02, + "learning_rate": 5.035971223021583e-05, + "loss": 0.3916, + "step": 70 + }, + { + "epoch": 0.02, + "learning_rate": 5.39568345323741e-05, + "loss": 0.3926, + "step": 75 + }, + { + "epoch": 0.03, + "learning_rate": 5.755395683453237e-05, + "loss": 0.3725, + "step": 80 + }, + { + "epoch": 0.03, + "learning_rate": 6.115107913669065e-05, + "loss": 0.3769, + "step": 85 + }, + { + "epoch": 0.03, + "learning_rate": 6.474820143884892e-05, + "loss": 0.352, + "step": 90 + }, + { + "epoch": 0.03, + "learning_rate": 6.83453237410072e-05, + "loss": 0.3534, + "step": 95 + }, + { + "epoch": 0.03, + "learning_rate": 7.194244604316547e-05, + "loss": 0.3432, + "step": 100 + }, + { + "epoch": 0.03, + "learning_rate": 7.553956834532374e-05, + "loss": 0.3286, + "step": 105 + }, + { + "epoch": 0.04, + "learning_rate": 7.913669064748202e-05, + "loss": 0.3298, + "step": 110 + }, + { + "epoch": 0.04, + "learning_rate": 8.273381294964029e-05, + "loss": 0.3223, + "step": 115 + }, + { + "epoch": 0.04, + "learning_rate": 8.633093525179857e-05, + "loss": 0.3181, + "step": 120 + }, + { + "epoch": 0.04, + "learning_rate": 8.992805755395684e-05, + "loss": 0.3225, + "step": 125 + }, + { + "epoch": 0.04, + "learning_rate": 9.35251798561151e-05, + "loss": 0.3145, + "step": 130 + }, + { + "epoch": 0.04, + "learning_rate": 9.712230215827339e-05, + "loss": 0.3137, + "step": 135 + }, + { + "epoch": 0.05, + "learning_rate": 0.00010071942446043166, + "loss": 0.3102, + "step": 140 + }, + { + "epoch": 0.05, + "learning_rate": 0.00010431654676258992, + "loss": 0.2951, + "step": 145 + }, + { + "epoch": 0.05, + "learning_rate": 0.0001079136690647482, + "loss": 0.3021, + "step": 150 + }, + { + "epoch": 0.05, + "learning_rate": 0.00011151079136690648, + "loss": 0.2923, + "step": 155 + }, + { + "epoch": 0.05, + "learning_rate": 0.00011510791366906474, + "loss": 0.2953, + "step": 160 + }, + { + "epoch": 0.05, + "learning_rate": 0.00011870503597122303, + "loss": 0.2893, + "step": 165 + }, + { + "epoch": 0.06, + "learning_rate": 0.0001223021582733813, + "loss": 0.2953, + "step": 170 + }, + { + "epoch": 0.06, + "learning_rate": 0.00012589928057553956, + "loss": 0.2931, + "step": 175 + }, + { + "epoch": 0.06, + "learning_rate": 0.00012949640287769783, + "loss": 0.2879, + "step": 180 + }, + { + "epoch": 0.06, + "learning_rate": 0.00013309352517985613, + "loss": 0.2642, + "step": 185 + }, + { + "epoch": 0.06, + "learning_rate": 0.0001366906474820144, + "loss": 0.2772, + "step": 190 + }, + { + "epoch": 0.06, + "learning_rate": 0.00014028776978417266, + "loss": 0.2858, + "step": 195 + }, + { + "epoch": 0.06, + "learning_rate": 0.00014388489208633093, + "loss": 0.2724, + "step": 200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0001474820143884892, + "loss": 0.2726, + "step": 205 + }, + { + "epoch": 0.07, + "learning_rate": 0.00015107913669064747, + "loss": 0.2678, + "step": 210 + }, + { + "epoch": 0.07, + "learning_rate": 0.00015467625899280577, + "loss": 0.2789, + "step": 215 + }, + { + "epoch": 0.07, + "learning_rate": 0.00015827338129496403, + "loss": 0.2742, + "step": 220 + }, + { + "epoch": 0.07, + "learning_rate": 0.0001618705035971223, + "loss": 0.2732, + "step": 225 + }, + { + "epoch": 0.07, + "learning_rate": 0.00016546762589928057, + "loss": 0.2741, + "step": 230 + }, + { + "epoch": 0.08, + "learning_rate": 0.00016906474820143884, + "loss": 0.2665, + "step": 235 + }, + { + "epoch": 0.08, + "learning_rate": 0.00017266187050359714, + "loss": 0.2771, + "step": 240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0001762589928057554, + "loss": 0.2751, + "step": 245 + }, + { + "epoch": 0.08, + "learning_rate": 0.00017985611510791367, + "loss": 0.2597, + "step": 250 + }, + { + "epoch": 0.08, + "learning_rate": 0.00018345323741007194, + "loss": 0.275, + "step": 255 + }, + { + "epoch": 0.08, + "learning_rate": 0.0001870503597122302, + "loss": 0.2791, + "step": 260 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019064748201438848, + "loss": 0.2493, + "step": 265 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019424460431654677, + "loss": 0.27, + "step": 270 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019784172661870504, + "loss": 0.2676, + "step": 275 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019999997550557367, + "loss": 0.2613, + "step": 280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019999969994341502, + "loss": 0.2747, + "step": 285 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019999911820191125, + "loss": 0.2699, + "step": 290 + }, + { + "epoch": 0.1, + "learning_rate": 0.0001999982302828436, + "loss": 0.2692, + "step": 295 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999703618893072, + "loss": 0.2522, + "step": 300 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999553592382863, + "loss": 0.2541, + "step": 305 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999372949213085, + "loss": 0.2614, + "step": 310 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999161689936837, + "loss": 0.2481, + "step": 315 + }, + { + "epoch": 0.1, + "learning_rate": 0.0001999891981520095, + "loss": 0.2497, + "step": 320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019998647325745995, + "loss": 0.2545, + "step": 325 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019998344222406283, + "loss": 0.2501, + "step": 330 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019998010506109854, + "loss": 0.2577, + "step": 335 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019997646177878487, + "loss": 0.2585, + "step": 340 + }, + { + "epoch": 0.11, + "learning_rate": 0.0001999725123882768, + "loss": 0.2494, + "step": 345 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019996825690166657, + "loss": 0.2538, + "step": 350 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019996369533198364, + "loss": 0.2469, + "step": 355 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019995882769319466, + "loss": 0.2502, + "step": 360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0001999536540002034, + "loss": 0.2361, + "step": 365 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019994817426885057, + "loss": 0.2446, + "step": 370 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019994238851591417, + "loss": 0.256, + "step": 375 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019993629675910896, + "loss": 0.2352, + "step": 380 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019992989901708672, + "loss": 0.2538, + "step": 385 + }, + { + "epoch": 0.13, + "learning_rate": 0.0001999231953094361, + "loss": 0.2523, + "step": 390 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019991618565668243, + "loss": 0.2512, + "step": 395 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019990887008028798, + "loss": 0.2402, + "step": 400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019990124860265156, + "loss": 0.2423, + "step": 405 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019989332124710862, + "loss": 0.2335, + "step": 410 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019988508803793124, + "loss": 0.2284, + "step": 415 + }, + { + "epoch": 0.14, + "learning_rate": 0.00019987654900032774, + "loss": 0.2382, + "step": 420 + }, + { + "epoch": 0.14, + "learning_rate": 0.0001998677041604431, + "loss": 0.2526, + "step": 425 + }, + { + "epoch": 0.14, + "learning_rate": 0.0001998585535453584, + "loss": 0.2417, + "step": 430 + }, + { + "epoch": 0.14, + "learning_rate": 0.00019984909718309105, + "loss": 0.2381, + "step": 435 + }, + { + "epoch": 0.14, + "learning_rate": 0.0001998393351025945, + "loss": 0.2418, + "step": 440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00019982926733375843, + "loss": 0.2409, + "step": 445 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001998188939074083, + "loss": 0.2467, + "step": 450 + }, + { + "epoch": 0.15, + "learning_rate": 0.00019980821485530544, + "loss": 0.2305, + "step": 455 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001997972302101471, + "loss": 0.232, + "step": 460 + }, + { + "epoch": 0.15, + "learning_rate": 0.00019978594000556602, + "loss": 0.225, + "step": 465 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001997743442761306, + "loss": 0.2259, + "step": 470 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001997624430573447, + "loss": 0.2414, + "step": 475 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019975023638564747, + "loss": 0.238, + "step": 480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019973772429841336, + "loss": 0.224, + "step": 485 + }, + { + "epoch": 0.16, + "learning_rate": 0.0001997249068339519, + "loss": 0.2369, + "step": 490 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019971178403150762, + "loss": 0.2393, + "step": 495 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019969835593125997, + "loss": 0.2249, + "step": 500 + }, + { + "epoch": 0.16, + "learning_rate": 0.0001996846225743231, + "loss": 0.2369, + "step": 505 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019967058400274592, + "loss": 0.2267, + "step": 510 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019965624025951168, + "loss": 0.2294, + "step": 515 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019964159138853815, + "loss": 0.2255, + "step": 520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019962663743467726, + "loss": 0.22, + "step": 525 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019961137844371507, + "loss": 0.2398, + "step": 530 + }, + { + "epoch": 0.17, + "learning_rate": 0.0001995958144623716, + "loss": 0.2431, + "step": 535 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019957994553830067, + "loss": 0.2245, + "step": 540 + }, + { + "epoch": 0.18, + "learning_rate": 0.0001995637717200898, + "loss": 0.2295, + "step": 545 + }, + { + "epoch": 0.18, + "learning_rate": 0.00019954729305726008, + "loss": 0.2288, + "step": 550 + }, + { + "epoch": 0.18, + "learning_rate": 0.00019953050960026586, + "loss": 0.2333, + "step": 555 + }, + { + "epoch": 0.18, + "learning_rate": 0.0001995134214004948, + "loss": 0.2316, + "step": 560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0001994960285102676, + "loss": 0.2285, + "step": 565 + }, + { + "epoch": 0.18, + "learning_rate": 0.00019947833098283782, + "loss": 0.2273, + "step": 570 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019946032887239182, + "loss": 0.2344, + "step": 575 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019944202223404854, + "loss": 0.2251, + "step": 580 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019942341112385922, + "loss": 0.226, + "step": 585 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019940449559880752, + "loss": 0.2353, + "step": 590 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019938527571680892, + "loss": 0.226, + "step": 595 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019936575153671097, + "loss": 0.221, + "step": 600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019934592311829285, + "loss": 0.2219, + "step": 605 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019932579052226528, + "loss": 0.234, + "step": 610 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019930535381027027, + "loss": 0.2181, + "step": 615 + }, + { + "epoch": 0.2, + "learning_rate": 0.000199284613044881, + "loss": 0.2191, + "step": 620 + }, + { + "epoch": 0.2, + "learning_rate": 0.0001992635682896016, + "loss": 0.235, + "step": 625 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019924221960886698, + "loss": 0.2256, + "step": 630 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019922056706804257, + "loss": 0.2265, + "step": 635 + }, + { + "epoch": 0.21, + "learning_rate": 0.0001991986107334242, + "loss": 0.2158, + "step": 640 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019917635067223777, + "loss": 0.2238, + "step": 645 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019915378695263924, + "loss": 0.2182, + "step": 650 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019913091964371423, + "loss": 0.2185, + "step": 655 + }, + { + "epoch": 0.21, + "learning_rate": 0.000199107748815478, + "loss": 0.2063, + "step": 660 + }, + { + "epoch": 0.22, + "learning_rate": 0.000199084274538875, + "loss": 0.2162, + "step": 665 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019906049688577887, + "loss": 0.2185, + "step": 670 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019903641592899205, + "loss": 0.2186, + "step": 675 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019901203174224564, + "loss": 0.2292, + "step": 680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019898734440019933, + "loss": 0.2162, + "step": 685 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019896235397844078, + "loss": 0.219, + "step": 690 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019893706055348578, + "loss": 0.218, + "step": 695 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019891146420277777, + "loss": 0.2211, + "step": 700 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019888556500468775, + "loss": 0.2227, + "step": 705 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019885936303851395, + "loss": 0.2214, + "step": 710 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019883285838448163, + "loss": 0.2204, + "step": 715 + }, + { + "epoch": 0.23, + "learning_rate": 0.0001988060511237428, + "loss": 0.2228, + "step": 720 + }, + { + "epoch": 0.23, + "learning_rate": 0.000198778941338376, + "loss": 0.2259, + "step": 725 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019875152911138602, + "loss": 0.2062, + "step": 730 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019872381452670372, + "loss": 0.2073, + "step": 735 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019869579766918568, + "loss": 0.2172, + "step": 740 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019866747862461396, + "loss": 0.21, + "step": 745 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019863885747969598, + "loss": 0.2195, + "step": 750 + }, + { + "epoch": 0.24, + "learning_rate": 0.0001986099343220639, + "loss": 0.2129, + "step": 755 + }, + { + "epoch": 0.25, + "learning_rate": 0.0001985807092402748, + "loss": 0.215, + "step": 760 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019855118232381015, + "loss": 0.2085, + "step": 765 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019852135366307545, + "loss": 0.2196, + "step": 770 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019849122334940022, + "loss": 0.2129, + "step": 775 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019846079147503755, + "loss": 0.2221, + "step": 780 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019843005813316377, + "loss": 0.2133, + "step": 785 + }, + { + "epoch": 0.26, + "learning_rate": 0.0001983990234178784, + "loss": 0.2066, + "step": 790 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019836768742420352, + "loss": 0.2188, + "step": 795 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019833605024808382, + "loss": 0.2128, + "step": 800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019830411198638607, + "loss": 0.2109, + "step": 805 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019827187273689894, + "loss": 0.2141, + "step": 810 + }, + { + "epoch": 0.26, + "learning_rate": 0.0001982393325983326, + "loss": 0.2098, + "step": 815 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019820649167031856, + "loss": 0.2188, + "step": 820 + }, + { + "epoch": 0.27, + "learning_rate": 0.0001981733500534093, + "loss": 0.2107, + "step": 825 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019813990784907785, + "loss": 0.2162, + "step": 830 + }, + { + "epoch": 0.27, + "learning_rate": 0.0001981061651597177, + "loss": 0.2189, + "step": 835 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019807212208864227, + "loss": 0.2117, + "step": 840 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019803777874008468, + "loss": 0.209, + "step": 845 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019800313521919753, + "loss": 0.206, + "step": 850 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019796819163205246, + "loss": 0.2104, + "step": 855 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019793294808563984, + "loss": 0.2016, + "step": 860 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019789740468786841, + "loss": 0.212, + "step": 865 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019786156154756506, + "loss": 0.2051, + "step": 870 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019782541877447443, + "loss": 0.21, + "step": 875 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019778897647925855, + "loss": 0.2104, + "step": 880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0001977522347734965, + "loss": 0.2179, + "step": 885 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019771519376968416, + "loss": 0.2139, + "step": 890 + }, + { + "epoch": 0.29, + "learning_rate": 0.0001976778535812338, + "loss": 0.2034, + "step": 895 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019764021432247368, + "loss": 0.2067, + "step": 900 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019760227610864776, + "loss": 0.2189, + "step": 905 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019756403905591538, + "loss": 0.2119, + "step": 910 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019752550328135086, + "loss": 0.2004, + "step": 915 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019748666890294315, + "loss": 0.2073, + "step": 920 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019744753603959545, + "loss": 0.1995, + "step": 925 + }, + { + "epoch": 0.3, + "learning_rate": 0.0001974081048111248, + "loss": 0.2034, + "step": 930 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019736837533826192, + "loss": 0.2015, + "step": 935 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019732834774265057, + "loss": 0.2166, + "step": 940 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019728802214684734, + "loss": 0.1987, + "step": 945 + }, + { + "epoch": 0.31, + "learning_rate": 0.0001972473986743213, + "loss": 0.2051, + "step": 950 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019720647744945344, + "loss": 0.2055, + "step": 955 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019716525859753652, + "loss": 0.2005, + "step": 960 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019712374224477452, + "loss": 0.1995, + "step": 965 + }, + { + "epoch": 0.31, + "learning_rate": 0.0001970819285182823, + "loss": 0.2096, + "step": 970 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019703981754608525, + "loss": 0.2024, + "step": 975 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019699740945711884, + "loss": 0.2006, + "step": 980 + }, + { + "epoch": 0.32, + "learning_rate": 0.0001969547043812283, + "loss": 0.2077, + "step": 985 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019691170244916814, + "loss": 0.2094, + "step": 990 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019686840379260176, + "loss": 0.2051, + "step": 995 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019682480854410115, + "loss": 0.2158, + "step": 1000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0001967809168371463, + "loss": 0.2035, + "step": 1005 + }, + { + "epoch": 0.33, + "learning_rate": 0.000196736728806125, + "loss": 0.2104, + "step": 1010 + }, + { + "epoch": 0.33, + "learning_rate": 0.00019669224458633222, + "loss": 0.1969, + "step": 1015 + }, + { + "epoch": 0.33, + "learning_rate": 0.0001966474643139699, + "loss": 0.2058, + "step": 1020 + }, + { + "epoch": 0.33, + "learning_rate": 0.00019660238812614647, + "loss": 0.2045, + "step": 1025 + }, + { + "epoch": 0.33, + "learning_rate": 0.0001965570161608762, + "loss": 0.1916, + "step": 1030 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019651134855707913, + "loss": 0.2008, + "step": 1035 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019646538545458047, + "loss": 0.2095, + "step": 1040 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019641912699411016, + "loss": 0.2009, + "step": 1045 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019637257331730248, + "loss": 0.2059, + "step": 1050 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019632572456669558, + "loss": 0.2084, + "step": 1055 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019627858088573112, + "loss": 0.1997, + "step": 1060 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001962311424187537, + "loss": 0.2068, + "step": 1065 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001961834093110106, + "loss": 0.1978, + "step": 1070 + }, + { + "epoch": 0.35, + "learning_rate": 0.00019613538170865118, + "loss": 0.2028, + "step": 1075 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001960870597587264, + "loss": 0.1996, + "step": 1080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001960384436091886, + "loss": 0.2002, + "step": 1085 + }, + { + "epoch": 0.35, + "learning_rate": 0.00019598953340889082, + "loss": 0.198, + "step": 1090 + }, + { + "epoch": 0.35, + "learning_rate": 0.00019594032930758643, + "loss": 0.1974, + "step": 1095 + }, + { + "epoch": 0.36, + "learning_rate": 0.0001958908314559287, + "loss": 0.1983, + "step": 1100 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019584104000547025, + "loss": 0.1987, + "step": 1105 + }, + { + "epoch": 0.36, + "learning_rate": 0.0001957909551086627, + "loss": 0.2003, + "step": 1110 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019574057691885613, + "loss": 0.1982, + "step": 1115 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019568990559029858, + "loss": 0.2031, + "step": 1120 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019563894127813564, + "loss": 0.1969, + "step": 1125 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019558768413841006, + "loss": 0.1916, + "step": 1130 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019553613432806102, + "loss": 0.2124, + "step": 1135 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019548429200492388, + "loss": 0.2042, + "step": 1140 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019543215732772956, + "loss": 0.201, + "step": 1145 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019537973045610424, + "loss": 0.2001, + "step": 1150 + }, + { + "epoch": 0.37, + "learning_rate": 0.0001953270115505686, + "loss": 0.1978, + "step": 1155 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019527400077253755, + "loss": 0.192, + "step": 1160 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019522069828431957, + "loss": 0.1946, + "step": 1165 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019516710424911642, + "loss": 0.1848, + "step": 1170 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019511321883102244, + "loss": 0.2005, + "step": 1175 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019505904219502414, + "loss": 0.1966, + "step": 1180 + }, + { + "epoch": 0.38, + "learning_rate": 0.0001950045745069997, + "loss": 0.1863, + "step": 1185 + }, + { + "epoch": 0.39, + "learning_rate": 0.0001949498159337184, + "loss": 0.1983, + "step": 1190 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019489476664284023, + "loss": 0.2113, + "step": 1195 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019483942680291523, + "loss": 0.1956, + "step": 1200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019478379658338305, + "loss": 0.2055, + "step": 1205 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019472787615457246, + "loss": 0.1879, + "step": 1210 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019467166568770078, + "loss": 0.1924, + "step": 1215 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019461516535487338, + "loss": 0.1971, + "step": 1220 + }, + { + "epoch": 0.4, + "learning_rate": 0.0001945583753290831, + "loss": 0.1952, + "step": 1225 + }, + { + "epoch": 0.4, + "learning_rate": 0.0001945012957842098, + "loss": 0.1838, + "step": 1230 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019444392689501978, + "loss": 0.1931, + "step": 1235 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019438626883716526, + "loss": 0.2093, + "step": 1240 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019432832178718385, + "loss": 0.1992, + "step": 1245 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019427008592249795, + "loss": 0.1963, + "step": 1250 + }, + { + "epoch": 0.41, + "learning_rate": 0.0001942115614214143, + "loss": 0.2003, + "step": 1255 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019415274846312338, + "loss": 0.1914, + "step": 1260 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019409364722769882, + "loss": 0.1917, + "step": 1265 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019403425789609697, + "loss": 0.2033, + "step": 1270 + }, + { + "epoch": 0.41, + "learning_rate": 0.0001939745806501563, + "loss": 0.1975, + "step": 1275 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019391461567259662, + "loss": 0.1922, + "step": 1280 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019385436314701897, + "loss": 0.1999, + "step": 1285 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019379382325790468, + "loss": 0.1997, + "step": 1290 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019373299619061487, + "loss": 0.1872, + "step": 1295 + }, + { + "epoch": 0.42, + "learning_rate": 0.0001936718821313901, + "loss": 0.1936, + "step": 1300 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019361048126734952, + "loss": 0.1798, + "step": 1305 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019354879378649046, + "loss": 0.1959, + "step": 1310 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019348681987768783, + "loss": 0.1924, + "step": 1315 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019342455973069345, + "loss": 0.1967, + "step": 1320 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019336201353613568, + "loss": 0.1925, + "step": 1325 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019329918148551856, + "loss": 0.1878, + "step": 1330 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019323606377122146, + "loss": 0.1979, + "step": 1335 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019317266058649834, + "loss": 0.189, + "step": 1340 + }, + { + "epoch": 0.44, + "learning_rate": 0.0001931089721254773, + "loss": 0.1995, + "step": 1345 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019304499858315976, + "loss": 0.1874, + "step": 1350 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019298074015542015, + "loss": 0.1991, + "step": 1355 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019291619703900502, + "loss": 0.188, + "step": 1360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0001928513694315327, + "loss": 0.1914, + "step": 1365 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019278625753149254, + "loss": 0.1843, + "step": 1370 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019272086153824427, + "loss": 0.1959, + "step": 1375 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019265518165201758, + "loss": 0.1867, + "step": 1380 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019258921807391131, + "loss": 0.1956, + "step": 1385 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019252297100589286, + "loss": 0.1991, + "step": 1390 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019245644065079775, + "loss": 0.1934, + "step": 1395 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019238962721232876, + "loss": 0.1953, + "step": 1400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019232253089505546, + "loss": 0.1865, + "step": 1405 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019225515190441354, + "loss": 0.1864, + "step": 1410 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019218749044670413, + "loss": 0.194, + "step": 1415 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019211954672909327, + "loss": 0.1853, + "step": 1420 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019205132095961126, + "loss": 0.1967, + "step": 1425 + }, + { + "epoch": 0.46, + "learning_rate": 0.0001919828133471519, + "loss": 0.1852, + "step": 1430 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019191402410147204, + "loss": 0.1913, + "step": 1435 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019184495343319068, + "loss": 0.1911, + "step": 1440 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019177560155378865, + "loss": 0.189, + "step": 1445 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019170596867560766, + "loss": 0.1854, + "step": 1450 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019163605501184986, + "loss": 0.1974, + "step": 1455 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019156586077657712, + "loss": 0.1933, + "step": 1460 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019149538618471032, + "loss": 0.1877, + "step": 1465 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001914246314520287, + "loss": 0.1904, + "step": 1470 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001913535967951694, + "loss": 0.1841, + "step": 1475 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001912822824316264, + "loss": 0.1876, + "step": 1480 + }, + { + "epoch": 0.48, + "learning_rate": 0.00019121068857975027, + "loss": 0.1858, + "step": 1485 + }, + { + "epoch": 0.48, + "learning_rate": 0.00019113881545874722, + "loss": 0.1898, + "step": 1490 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001910666632886786, + "loss": 0.1923, + "step": 1495 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019099423229046014, + "loss": 0.1803, + "step": 1500 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019092152268586118, + "loss": 0.1945, + "step": 1505 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019084853469750424, + "loss": 0.1802, + "step": 1510 + }, + { + "epoch": 0.49, + "learning_rate": 0.0001907752685488641, + "loss": 0.197, + "step": 1515 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019070172446426724, + "loss": 0.1791, + "step": 1520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019062790266889117, + "loss": 0.1874, + "step": 1525 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019055380338876363, + "loss": 0.1806, + "step": 1530 + }, + { + "epoch": 0.5, + "learning_rate": 0.000190479426850762, + "loss": 0.1834, + "step": 1535 + }, + { + "epoch": 0.5, + "learning_rate": 0.0001904047732826126, + "loss": 0.1921, + "step": 1540 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019032984291288986, + "loss": 0.1826, + "step": 1545 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019025463597101584, + "loss": 0.1966, + "step": 1550 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019017915268725925, + "loss": 0.1867, + "step": 1555 + }, + { + "epoch": 0.51, + "learning_rate": 0.00019010339329273514, + "loss": 0.196, + "step": 1560 + }, + { + "epoch": 0.51, + "learning_rate": 0.00019002735801940373, + "loss": 0.1971, + "step": 1565 + }, + { + "epoch": 0.51, + "learning_rate": 0.00018995104710007004, + "loss": 0.187, + "step": 1570 + }, + { + "epoch": 0.51, + "learning_rate": 0.00018987446076838312, + "loss": 0.1897, + "step": 1575 + }, + { + "epoch": 0.51, + "learning_rate": 0.00018979759925883509, + "loss": 0.1887, + "step": 1580 + }, + { + "epoch": 0.51, + "learning_rate": 0.0001897204628067608, + "loss": 0.1849, + "step": 1585 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018964305164833678, + "loss": 0.1795, + "step": 1590 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018956536602058078, + "loss": 0.1984, + "step": 1595 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018948740616135085, + "loss": 0.1876, + "step": 1600 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018940917230934464, + "loss": 0.1879, + "step": 1605 + }, + { + "epoch": 0.52, + "learning_rate": 0.0001893306647040988, + "loss": 0.1906, + "step": 1610 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018925188358598813, + "loss": 0.1917, + "step": 1615 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018917282919622488, + "loss": 0.1912, + "step": 1620 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018909350177685795, + "loss": 0.1813, + "step": 1625 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018901390157077225, + "loss": 0.183, + "step": 1630 + }, + { + "epoch": 0.53, + "learning_rate": 0.0001889340288216879, + "loss": 0.1782, + "step": 1635 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018885388377415957, + "loss": 0.1866, + "step": 1640 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018877346667357548, + "loss": 0.1915, + "step": 1645 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018869277776615691, + "loss": 0.1902, + "step": 1650 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018861181729895745, + "loss": 0.1829, + "step": 1655 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018853058551986202, + "loss": 0.1855, + "step": 1660 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018844908267758633, + "loss": 0.1845, + "step": 1665 + }, + { + "epoch": 0.54, + "learning_rate": 0.000188367309021676, + "loss": 0.1794, + "step": 1670 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018828526480250575, + "loss": 0.1796, + "step": 1675 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018820295027127885, + "loss": 0.1806, + "step": 1680 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018812036568002617, + "loss": 0.1846, + "step": 1685 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018803751128160538, + "loss": 0.189, + "step": 1690 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018795438732970032, + "loss": 0.1856, + "step": 1695 + }, + { + "epoch": 0.55, + "learning_rate": 0.0001878709940788201, + "loss": 0.1797, + "step": 1700 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018778733178429844, + "loss": 0.1876, + "step": 1705 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018770340070229272, + "loss": 0.1775, + "step": 1710 + }, + { + "epoch": 0.56, + "learning_rate": 0.0001876192010897834, + "loss": 0.1862, + "step": 1715 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018753473320457306, + "loss": 0.1835, + "step": 1720 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018744999730528568, + "loss": 0.1909, + "step": 1725 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018736499365136584, + "loss": 0.189, + "step": 1730 + }, + { + "epoch": 0.56, + "learning_rate": 0.000187279722503078, + "loss": 0.1861, + "step": 1735 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018719418412150557, + "loss": 0.1809, + "step": 1740 + }, + { + "epoch": 0.57, + "learning_rate": 0.00018710837876855017, + "loss": 0.1829, + "step": 1745 + }, + { + "epoch": 0.57, + "learning_rate": 0.0001870223067069309, + "loss": 0.1886, + "step": 1750 + }, + { + "epoch": 0.57, + "learning_rate": 0.00018693596820018336, + "loss": 0.1787, + "step": 1755 + }, + { + "epoch": 0.57, + "learning_rate": 0.0001868493635126591, + "loss": 0.1816, + "step": 1760 + }, + { + "epoch": 0.57, + "learning_rate": 0.0001867624929095245, + "loss": 0.1759, + "step": 1765 + }, + { + "epoch": 0.57, + "learning_rate": 0.00018667535665676025, + "loss": 0.1702, + "step": 1770 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018658795502116035, + "loss": 0.1802, + "step": 1775 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018650028827033136, + "loss": 0.1836, + "step": 1780 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018641235667269155, + "loss": 0.1787, + "step": 1785 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018632416049747016, + "loss": 0.1811, + "step": 1790 + }, + { + "epoch": 0.58, + "learning_rate": 0.0001862357000147064, + "loss": 0.1835, + "step": 1795 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018614697549524889, + "loss": 0.1857, + "step": 1800 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018605798721075455, + "loss": 0.1867, + "step": 1805 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018596873543368797, + "loss": 0.1766, + "step": 1810 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018587922043732046, + "loss": 0.1895, + "step": 1815 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018578944249572925, + "loss": 0.1888, + "step": 1820 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018569940188379672, + "loss": 0.1871, + "step": 1825 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018560909887720944, + "loss": 0.1804, + "step": 1830 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018551853375245737, + "loss": 0.1877, + "step": 1835 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018542770678683308, + "loss": 0.182, + "step": 1840 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001853366182584308, + "loss": 0.1809, + "step": 1845 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018524526844614564, + "loss": 0.1913, + "step": 1850 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018515365762967266, + "loss": 0.185, + "step": 1855 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018506178608950613, + "loss": 0.1753, + "step": 1860 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018496965410693856, + "loss": 0.1786, + "step": 1865 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001848772619640599, + "loss": 0.1841, + "step": 1870 + }, + { + "epoch": 0.61, + "learning_rate": 0.00018478460994375667, + "loss": 0.184, + "step": 1875 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001846916983297111, + "loss": 0.179, + "step": 1880 + }, + { + "epoch": 0.61, + "learning_rate": 0.00018459852740640014, + "loss": 0.1833, + "step": 1885 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001845050974590948, + "loss": 0.186, + "step": 1890 + }, + { + "epoch": 0.61, + "learning_rate": 0.00018441140877385913, + "loss": 0.1778, + "step": 1895 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018431746163754943, + "loss": 0.1786, + "step": 1900 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018422325633781327, + "loss": 0.1826, + "step": 1905 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018412879316308866, + "loss": 0.1738, + "step": 1910 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018403407240260327, + "loss": 0.1852, + "step": 1915 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018393909434637329, + "loss": 0.1821, + "step": 1920 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001838438592852028, + "loss": 0.1773, + "step": 1925 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018374836751068282, + "loss": 0.1772, + "step": 1930 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018365261931519022, + "loss": 0.1766, + "step": 1935 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018355661499188717, + "loss": 0.1858, + "step": 1940 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018346035483471994, + "loss": 0.1786, + "step": 1945 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001833638391384181, + "loss": 0.1745, + "step": 1950 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018326706819849365, + "loss": 0.1713, + "step": 1955 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018317004231124017, + "loss": 0.1738, + "step": 1960 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018307276177373174, + "loss": 0.1807, + "step": 1965 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018297522688382218, + "loss": 0.1806, + "step": 1970 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018287743794014407, + "loss": 0.1735, + "step": 1975 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018277939524210783, + "loss": 0.1812, + "step": 1980 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001826810990899009, + "loss": 0.1734, + "step": 1985 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001825825497844867, + "loss": 0.1703, + "step": 1990 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018248374762760365, + "loss": 0.1823, + "step": 1995 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018238469292176457, + "loss": 0.1837, + "step": 2000 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018228538597025537, + "loss": 0.1807, + "step": 2005 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018218582707713434, + "loss": 0.1904, + "step": 2010 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001820860165472312, + "loss": 0.1844, + "step": 2015 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018198595468614604, + "loss": 0.1792, + "step": 2020 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018188564180024854, + "loss": 0.1845, + "step": 2025 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018178507819667698, + "loss": 0.1853, + "step": 2030 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018168426418333725, + "loss": 0.1875, + "step": 2035 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018158320006890198, + "loss": 0.1759, + "step": 2040 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018148188616280955, + "loss": 0.1816, + "step": 2045 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001813803227752631, + "loss": 0.1787, + "step": 2050 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001812785102172297, + "loss": 0.1758, + "step": 2055 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018117644880043932, + "loss": 0.1802, + "step": 2060 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018107413883738387, + "loss": 0.186, + "step": 2065 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018097158064131626, + "loss": 0.1743, + "step": 2070 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018086877452624944, + "loss": 0.1877, + "step": 2075 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018076572080695543, + "loss": 0.1742, + "step": 2080 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018066241979896444, + "loss": 0.1727, + "step": 2085 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018055887181856375, + "loss": 0.1806, + "step": 2090 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018045507718279677, + "loss": 0.175, + "step": 2095 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018035103620946225, + "loss": 0.1727, + "step": 2100 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001802467492171131, + "loss": 0.1779, + "step": 2105 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018014221652505556, + "loss": 0.1823, + "step": 2110 + }, + { + "epoch": 0.69, + "learning_rate": 0.000180037438453348, + "loss": 0.1814, + "step": 2115 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001799324153228003, + "loss": 0.1707, + "step": 2120 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001798271474549725, + "loss": 0.1778, + "step": 2125 + }, + { + "epoch": 0.69, + "learning_rate": 0.000179721635172174, + "loss": 0.1888, + "step": 2130 + }, + { + "epoch": 0.69, + "learning_rate": 0.00017961587879746266, + "loss": 0.1785, + "step": 2135 + }, + { + "epoch": 0.69, + "learning_rate": 0.00017950987865464365, + "loss": 0.1661, + "step": 2140 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017940363506826836, + "loss": 0.1792, + "step": 2145 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001792971483636338, + "loss": 0.1732, + "step": 2150 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017919041886678118, + "loss": 0.1771, + "step": 2155 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001790834469044952, + "loss": 0.173, + "step": 2160 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017897623280430284, + "loss": 0.1768, + "step": 2165 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017886877689447258, + "loss": 0.175, + "step": 2170 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001787610795040132, + "loss": 0.1691, + "step": 2175 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001786531409626728, + "loss": 0.1774, + "step": 2180 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001785449616009379, + "loss": 0.1709, + "step": 2185 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001784365417500324, + "loss": 0.1809, + "step": 2190 + }, + { + "epoch": 0.71, + "learning_rate": 0.00017832788174191645, + "loss": 0.1764, + "step": 2195 + }, + { + "epoch": 0.71, + "learning_rate": 0.00017821898190928558, + "loss": 0.1739, + "step": 2200 + }, + { + "epoch": 0.71, + "learning_rate": 0.00017810984258556957, + "loss": 0.1772, + "step": 2205 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017800046410493145, + "loss": 0.1779, + "step": 2210 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001778908468022666, + "loss": 0.1828, + "step": 2215 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017778099101320156, + "loss": 0.1722, + "step": 2220 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017767089707409304, + "loss": 0.1737, + "step": 2225 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017756056532202696, + "loss": 0.1688, + "step": 2230 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017744999609481738, + "loss": 0.1784, + "step": 2235 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017733918973100545, + "loss": 0.1747, + "step": 2240 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017722814656985841, + "loss": 0.1667, + "step": 2245 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001771168669513685, + "loss": 0.1648, + "step": 2250 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017700535121625187, + "loss": 0.1777, + "step": 2255 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017689359970594779, + "loss": 0.175, + "step": 2260 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001767816127626173, + "loss": 0.1715, + "step": 2265 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001766693907291423, + "loss": 0.1742, + "step": 2270 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017655693394912452, + "loss": 0.1868, + "step": 2275 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017644424276688441, + "loss": 0.1757, + "step": 2280 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017633131752746017, + "loss": 0.1803, + "step": 2285 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017621815857660656, + "loss": 0.1812, + "step": 2290 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017610476626079395, + "loss": 0.1691, + "step": 2295 + }, + { + "epoch": 0.75, + "learning_rate": 0.00017599114092720726, + "loss": 0.1662, + "step": 2300 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001758772829237448, + "loss": 0.1709, + "step": 2305 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001757631925990174, + "loss": 0.1822, + "step": 2310 + }, + { + "epoch": 0.75, + "learning_rate": 0.00017564887030234696, + "loss": 0.1732, + "step": 2315 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001755343163837659, + "loss": 0.1854, + "step": 2320 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001754195311940157, + "loss": 0.1768, + "step": 2325 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017530451508454594, + "loss": 0.1785, + "step": 2330 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017518926840751325, + "loss": 0.1724, + "step": 2335 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017507379151578017, + "loss": 0.1812, + "step": 2340 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017495808476291423, + "loss": 0.1754, + "step": 2345 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017484214850318664, + "loss": 0.1799, + "step": 2350 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001747259830915713, + "loss": 0.1769, + "step": 2355 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017460958888374385, + "loss": 0.1764, + "step": 2360 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001744929662360803, + "loss": 0.1716, + "step": 2365 + }, + { + "epoch": 0.77, + "learning_rate": 0.00017437611550565624, + "loss": 0.1699, + "step": 2370 + }, + { + "epoch": 0.77, + "learning_rate": 0.00017425903705024553, + "loss": 0.1661, + "step": 2375 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001741417312283193, + "loss": 0.1699, + "step": 2380 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001740241983990448, + "loss": 0.1682, + "step": 2385 + }, + { + "epoch": 0.77, + "learning_rate": 0.00017390643892228438, + "loss": 0.173, + "step": 2390 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001737884531585943, + "loss": 0.1852, + "step": 2395 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017367024146922372, + "loss": 0.1681, + "step": 2400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017355180421611347, + "loss": 0.1798, + "step": 2405 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017343314176189502, + "loss": 0.1705, + "step": 2410 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017331425446988946, + "loss": 0.177, + "step": 2415 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017319514270410614, + "loss": 0.1754, + "step": 2420 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017307580682924178, + "loss": 0.1679, + "step": 2425 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017295624721067932, + "loss": 0.1763, + "step": 2430 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017283646421448673, + "loss": 0.1769, + "step": 2435 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017271645820741583, + "loss": 0.1753, + "step": 2440 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001725962295569014, + "loss": 0.1715, + "step": 2445 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017247577863105983, + "loss": 0.1784, + "step": 2450 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017235510579868802, + "loss": 0.17, + "step": 2455 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017223421142926245, + "loss": 0.1686, + "step": 2460 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017211309589293774, + "loss": 0.166, + "step": 2465 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017199175956054584, + "loss": 0.1652, + "step": 2470 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017187020280359455, + "loss": 0.1732, + "step": 2475 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001717484259942668, + "loss": 0.1754, + "step": 2480 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017162642950541902, + "loss": 0.1748, + "step": 2485 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017150421371058049, + "loss": 0.1733, + "step": 2490 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017138177898395177, + "loss": 0.1737, + "step": 2495 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017125912570040383, + "loss": 0.175, + "step": 2500 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001711362542354769, + "loss": 0.173, + "step": 2505 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017101316496537905, + "loss": 0.1796, + "step": 2510 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017088985826698542, + "loss": 0.1722, + "step": 2515 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017076633451783675, + "loss": 0.1738, + "step": 2520 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017064259409613834, + "loss": 0.1743, + "step": 2525 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017051863738075902, + "loss": 0.1653, + "step": 2530 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001703944647512297, + "loss": 0.1761, + "step": 2535 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017027007658774256, + "loss": 0.1649, + "step": 2540 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017014547327114956, + "loss": 0.1738, + "step": 2545 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001700206551829614, + "loss": 0.1585, + "step": 2550 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016989562270534655, + "loss": 0.183, + "step": 2555 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001697703762211297, + "loss": 0.1693, + "step": 2560 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001696449161137909, + "loss": 0.173, + "step": 2565 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016951924276746425, + "loss": 0.1691, + "step": 2570 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016939335656693667, + "loss": 0.1698, + "step": 2575 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016926725789764691, + "loss": 0.1706, + "step": 2580 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016914094714568417, + "loss": 0.1691, + "step": 2585 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016901442469778713, + "loss": 0.1665, + "step": 2590 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016888769094134247, + "loss": 0.1831, + "step": 2595 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016876074626438395, + "loss": 0.1637, + "step": 2600 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016863359105559117, + "loss": 0.1757, + "step": 2605 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016850622570428826, + "loss": 0.1721, + "step": 2610 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016837865060044272, + "loss": 0.1647, + "step": 2615 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016825086613466442, + "loss": 0.1734, + "step": 2620 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016812287269820416, + "loss": 0.1847, + "step": 2625 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016799467068295257, + "loss": 0.1682, + "step": 2630 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001678662604814389, + "loss": 0.1741, + "step": 2635 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016773764248682988, + "loss": 0.1692, + "step": 2640 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016760881709292842, + "loss": 0.1805, + "step": 2645 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001674797846941724, + "loss": 0.1685, + "step": 2650 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016735054568563364, + "loss": 0.1673, + "step": 2655 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001672211004630164, + "loss": 0.1686, + "step": 2660 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016709144942265644, + "loss": 0.1749, + "step": 2665 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001669615929615197, + "loss": 0.1644, + "step": 2670 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016683153147720097, + "loss": 0.1673, + "step": 2675 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016670126536792287, + "loss": 0.1722, + "step": 2680 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001665707950325345, + "loss": 0.1588, + "step": 2685 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001664401208705103, + "loss": 0.164, + "step": 2690 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016630924328194877, + "loss": 0.1708, + "step": 2695 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016617816266757123, + "loss": 0.1711, + "step": 2700 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016604687942872067, + "loss": 0.1701, + "step": 2705 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016591539396736046, + "loss": 0.1708, + "step": 2710 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016578370668607307, + "loss": 0.1669, + "step": 2715 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016565181798805907, + "loss": 0.1601, + "step": 2720 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016551972827713555, + "loss": 0.1733, + "step": 2725 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016538743795773515, + "loss": 0.1641, + "step": 2730 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016525494743490474, + "loss": 0.1723, + "step": 2735 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016512225711430415, + "loss": 0.1655, + "step": 2740 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016498936740220492, + "loss": 0.172, + "step": 2745 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016485627870548917, + "loss": 0.1686, + "step": 2750 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016472299143164816, + "loss": 0.1709, + "step": 2755 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016458950598878126, + "loss": 0.1648, + "step": 2760 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016445582278559458, + "loss": 0.1641, + "step": 2765 + }, + { + "epoch": 0.9, + "learning_rate": 0.0001643219422313996, + "loss": 0.1662, + "step": 2770 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016418786473611227, + "loss": 0.1763, + "step": 2775 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016405359071025138, + "loss": 0.1773, + "step": 2780 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016391912056493743, + "loss": 0.1678, + "step": 2785 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016378445471189156, + "loss": 0.1632, + "step": 2790 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016364959356343398, + "loss": 0.1671, + "step": 2795 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016351453753248296, + "loss": 0.1739, + "step": 2800 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016337928703255336, + "loss": 0.1727, + "step": 2805 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016324384247775554, + "loss": 0.1756, + "step": 2810 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016310820428279397, + "loss": 0.1671, + "step": 2815 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016297237286296606, + "loss": 0.169, + "step": 2820 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016283634863416082, + "loss": 0.1693, + "step": 2825 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016270013201285757, + "loss": 0.164, + "step": 2830 + }, + { + "epoch": 0.92, + "learning_rate": 0.0001625637234161247, + "loss": 0.1663, + "step": 2835 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016242712326161845, + "loss": 0.1742, + "step": 2840 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016229033196758154, + "loss": 0.169, + "step": 2845 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016215334995284188, + "loss": 0.1703, + "step": 2850 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001620161776368114, + "loss": 0.1669, + "step": 2855 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001618788154394846, + "loss": 0.1634, + "step": 2860 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001617412637814375, + "loss": 0.163, + "step": 2865 + }, + { + "epoch": 0.93, + "learning_rate": 0.00016160352308382608, + "loss": 0.1651, + "step": 2870 + }, + { + "epoch": 0.93, + "learning_rate": 0.00016146559376838514, + "loss": 0.1657, + "step": 2875 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001613274762574271, + "loss": 0.1638, + "step": 2880 + }, + { + "epoch": 0.93, + "learning_rate": 0.00016118917097384048, + "loss": 0.1691, + "step": 2885 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016105067834108874, + "loss": 0.1627, + "step": 2890 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016091199878320902, + "loss": 0.1608, + "step": 2895 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016077313272481076, + "loss": 0.1617, + "step": 2900 + }, + { + "epoch": 0.94, + "learning_rate": 0.0001606340805910744, + "loss": 0.1696, + "step": 2905 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016049484280775012, + "loss": 0.1604, + "step": 2910 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016035541980115653, + "loss": 0.161, + "step": 2915 + }, + { + "epoch": 0.95, + "learning_rate": 0.00016021581199817943, + "loss": 0.1703, + "step": 2920 + }, + { + "epoch": 0.95, + "learning_rate": 0.0001600760198262703, + "loss": 0.1671, + "step": 2925 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015993604371344517, + "loss": 0.1603, + "step": 2930 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015979588408828333, + "loss": 0.1655, + "step": 2935 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015965554137992584, + "loss": 0.1675, + "step": 2940 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015951501601807437, + "loss": 0.1641, + "step": 2945 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015937430843298986, + "loss": 0.165, + "step": 2950 + }, + { + "epoch": 0.96, + "learning_rate": 0.0001592334190554912, + "loss": 0.1758, + "step": 2955 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015909234831695379, + "loss": 0.1703, + "step": 2960 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015895109664930846, + "loss": 0.1644, + "step": 2965 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015880966448503987, + "loss": 0.1638, + "step": 2970 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015866805225718542, + "loss": 0.1659, + "step": 2975 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015852626039933383, + "loss": 0.1676, + "step": 2980 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015838428934562373, + "loss": 0.1794, + "step": 2985 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015824213953074254, + "loss": 0.1562, + "step": 2990 + }, + { + "epoch": 0.97, + "learning_rate": 0.0001580998113899249, + "loss": 0.171, + "step": 2995 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015795730535895147, + "loss": 0.1681, + "step": 3000 + }, + { + "epoch": 0.97, + "learning_rate": 0.0001578146218741476, + "loss": 0.1617, + "step": 3005 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015767176137238196, + "loss": 0.1708, + "step": 3010 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015752872429106523, + "loss": 0.1636, + "step": 3015 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015738551106814872, + "loss": 0.1678, + "step": 3020 + }, + { + "epoch": 0.98, + "learning_rate": 0.000157242122142123, + "loss": 0.1612, + "step": 3025 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015709855795201675, + "loss": 0.1658, + "step": 3030 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015695481893739514, + "loss": 0.1621, + "step": 3035 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015681090553835864, + "loss": 0.1718, + "step": 3040 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015666681819554175, + "loss": 0.1712, + "step": 3045 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015652255735011135, + "loss": 0.1693, + "step": 3050 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015637812344376583, + "loss": 0.1714, + "step": 3055 + }, + { + "epoch": 0.99, + "learning_rate": 0.0001562335169187332, + "loss": 0.1661, + "step": 3060 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015608873821777012, + "loss": 0.1676, + "step": 3065 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015594378778416044, + "loss": 0.1654, + "step": 3070 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015579866606171372, + "loss": 0.1644, + "step": 3075 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015565337349476404, + "loss": 0.1744, + "step": 3080 + }, + { + "epoch": 1.0, + "learning_rate": 0.0001555079105281686, + "loss": 0.1606, + "step": 3085 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015536227760730632, + "loss": 0.1659, + "step": 3090 + }, + { + "epoch": 1.0, + "learning_rate": 0.0001552164751780764, + "loss": 0.1602, + "step": 3095 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015507050368689714, + "loss": 0.1636, + "step": 3100 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015492436358070438, + "loss": 0.1555, + "step": 3105 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015477805530695035, + "loss": 0.1661, + "step": 3110 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015463157931360207, + "loss": 0.1628, + "step": 3115 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015448493604914014, + "loss": 0.1696, + "step": 3120 + }, + { + "epoch": 1.01, + "learning_rate": 0.0001543381259625573, + "loss": 0.1685, + "step": 3125 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015419114950335704, + "loss": 0.1635, + "step": 3130 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015404400712155228, + "loss": 0.1631, + "step": 3135 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015389669926766395, + "loss": 0.1664, + "step": 3140 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015374922639271963, + "loss": 0.1617, + "step": 3145 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015360158894825216, + "loss": 0.174, + "step": 3150 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015345378738629826, + "loss": 0.1591, + "step": 3155 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015330582215939713, + "loss": 0.1729, + "step": 3160 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015315769372058907, + "loss": 0.1732, + "step": 3165 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015300940252341416, + "loss": 0.1608, + "step": 3170 + }, + { + "epoch": 1.03, + "learning_rate": 0.0001528609490219108, + "loss": 0.168, + "step": 3175 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015271233367061425, + "loss": 0.1731, + "step": 3180 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015256355692455542, + "loss": 0.1656, + "step": 3185 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015241461923925934, + "loss": 0.1591, + "step": 3190 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015226552107074385, + "loss": 0.1635, + "step": 3195 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015211626287551803, + "loss": 0.1622, + "step": 3200 + }, + { + "epoch": 1.04, + "learning_rate": 0.0001519668451105811, + "loss": 0.162, + "step": 3205 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015181726823342072, + "loss": 0.1614, + "step": 3210 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015166753270201176, + "loss": 0.1657, + "step": 3215 + }, + { + "epoch": 1.04, + "learning_rate": 0.0001515176389748149, + "loss": 0.1623, + "step": 3220 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015136758751077504, + "loss": 0.1629, + "step": 3225 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015121737876932034, + "loss": 0.1556, + "step": 3230 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015106701321036017, + "loss": 0.1651, + "step": 3235 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015091649129428424, + "loss": 0.1575, + "step": 3240 + }, + { + "epoch": 1.05, + "learning_rate": 0.000150765813481961, + "loss": 0.1594, + "step": 3245 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015061498023473608, + "loss": 0.1624, + "step": 3250 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015046399201443114, + "loss": 0.1616, + "step": 3255 + }, + { + "epoch": 1.06, + "learning_rate": 0.00015031284928334238, + "loss": 0.1693, + "step": 3260 + }, + { + "epoch": 1.06, + "learning_rate": 0.00015016155250423892, + "loss": 0.1592, + "step": 3265 + }, + { + "epoch": 1.06, + "learning_rate": 0.00015001010214036174, + "loss": 0.1542, + "step": 3270 + }, + { + "epoch": 1.06, + "learning_rate": 0.0001498584986554219, + "loss": 0.1631, + "step": 3275 + }, + { + "epoch": 1.06, + "learning_rate": 0.0001497067425135993, + "loss": 0.1612, + "step": 3280 + }, + { + "epoch": 1.06, + "learning_rate": 0.00014955483417954137, + "loss": 0.1551, + "step": 3285 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014940277411836146, + "loss": 0.155, + "step": 3290 + }, + { + "epoch": 1.07, + "learning_rate": 0.0001492505627956374, + "loss": 0.1593, + "step": 3295 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014909820067741028, + "loss": 0.1622, + "step": 3300 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014894568823018277, + "loss": 0.1637, + "step": 3305 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014879302592091798, + "loss": 0.1643, + "step": 3310 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014864021421703768, + "loss": 0.1642, + "step": 3315 + }, + { + "epoch": 1.08, + "learning_rate": 0.00014848725358642114, + "loss": 0.1607, + "step": 3320 + }, + { + "epoch": 1.08, + "learning_rate": 0.00014833414449740373, + "loss": 0.1591, + "step": 3325 + }, + { + "epoch": 1.08, + "learning_rate": 0.0001481808874187752, + "loss": 0.1682, + "step": 3330 + }, + { + "epoch": 1.08, + "learning_rate": 0.00014802748281977842, + "loss": 0.1572, + "step": 3335 + }, + { + "epoch": 1.08, + "learning_rate": 0.0001478739311701081, + "loss": 0.1595, + "step": 3340 + }, + { + "epoch": 1.08, + "learning_rate": 0.000147720232939909, + "loss": 0.1683, + "step": 3345 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014756638859977483, + "loss": 0.1539, + "step": 3350 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014741239862074655, + "loss": 0.1594, + "step": 3355 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014725826347431113, + "loss": 0.1658, + "step": 3360 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014710398363239994, + "loss": 0.1621, + "step": 3365 + }, + { + "epoch": 1.09, + "learning_rate": 0.0001469495595673874, + "loss": 0.1651, + "step": 3370 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014679499175208953, + "loss": 0.1655, + "step": 3375 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014664028065976246, + "loss": 0.1669, + "step": 3380 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014648542676410105, + "loss": 0.1593, + "step": 3385 + }, + { + "epoch": 1.1, + "learning_rate": 0.0001463304305392373, + "loss": 0.157, + "step": 3390 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014617529245973917, + "loss": 0.1593, + "step": 3395 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014602001300060877, + "loss": 0.1516, + "step": 3400 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014586459263728115, + "loss": 0.156, + "step": 3405 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014570903184562282, + "loss": 0.1637, + "step": 3410 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014555333110193022, + "loss": 0.1552, + "step": 3415 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014539749088292826, + "loss": 0.1624, + "step": 3420 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014524151166576897, + "loss": 0.1568, + "step": 3425 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014508539392802994, + "loss": 0.1583, + "step": 3430 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014492913814771283, + "loss": 0.1625, + "step": 3435 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014477274480324202, + "loss": 0.1563, + "step": 3440 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014461621437346307, + "loss": 0.1634, + "step": 3445 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014445954733764127, + "loss": 0.1452, + "step": 3450 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014430274417546012, + "loss": 0.1539, + "step": 3455 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014414580536702, + "loss": 0.1636, + "step": 3460 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014398873139283657, + "loss": 0.1509, + "step": 3465 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014383152273383934, + "loss": 0.1636, + "step": 3470 + }, + { + "epoch": 1.13, + "learning_rate": 0.00014367417987137015, + "loss": 0.1619, + "step": 3475 + }, + { + "epoch": 1.13, + "learning_rate": 0.00014351670328718183, + "loss": 0.1648, + "step": 3480 + }, + { + "epoch": 1.13, + "learning_rate": 0.00014335909346343658, + "loss": 0.1508, + "step": 3485 + }, + { + "epoch": 1.13, + "learning_rate": 0.0001432013508827046, + "loss": 0.1546, + "step": 3490 + }, + { + "epoch": 1.13, + "learning_rate": 0.0001430434760279625, + "loss": 0.1549, + "step": 3495 + }, + { + "epoch": 1.13, + "learning_rate": 0.000142885469382592, + "loss": 0.1412, + "step": 3500 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001427273314303782, + "loss": 0.1574, + "step": 3505 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001425690626555083, + "loss": 0.16, + "step": 3510 + }, + { + "epoch": 1.14, + "learning_rate": 0.00014241066354257008, + "loss": 0.1633, + "step": 3515 + }, + { + "epoch": 1.14, + "learning_rate": 0.00014225213457655034, + "loss": 0.1499, + "step": 3520 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001420934762428335, + "loss": 0.159, + "step": 3525 + }, + { + "epoch": 1.14, + "learning_rate": 0.00014193468902720007, + "loss": 0.1561, + "step": 3530 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014177577341582516, + "loss": 0.1619, + "step": 3535 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014161672989527698, + "loss": 0.16, + "step": 3540 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014145755895251546, + "loss": 0.148, + "step": 3545 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014129826107489054, + "loss": 0.1502, + "step": 3550 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014113883675014098, + "loss": 0.1427, + "step": 3555 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001409792864663925, + "loss": 0.1613, + "step": 3560 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001408196107121566, + "loss": 0.1597, + "step": 3565 + }, + { + "epoch": 1.16, + "learning_rate": 0.00014065980997632896, + "loss": 0.1501, + "step": 3570 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001404998847481879, + "loss": 0.1537, + "step": 3575 + }, + { + "epoch": 1.16, + "learning_rate": 0.00014033983551739286, + "loss": 0.1608, + "step": 3580 + }, + { + "epoch": 1.16, + "learning_rate": 0.00014017966277398304, + "loss": 0.1488, + "step": 3585 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001400193670083757, + "loss": 0.1593, + "step": 3590 + }, + { + "epoch": 1.16, + "learning_rate": 0.00013985894871136492, + "loss": 0.1525, + "step": 3595 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013969840837411982, + "loss": 0.1529, + "step": 3600 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013953774648818324, + "loss": 0.1489, + "step": 3605 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013937696354547022, + "loss": 0.144, + "step": 3610 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013921606003826627, + "loss": 0.1584, + "step": 3615 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013905503645922628, + "loss": 0.1657, + "step": 3620 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013889389330137264, + "loss": 0.1557, + "step": 3625 + }, + { + "epoch": 1.18, + "learning_rate": 0.00013873263105809388, + "loss": 0.1551, + "step": 3630 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001385712502231432, + "loss": 0.157, + "step": 3635 + }, + { + "epoch": 1.18, + "learning_rate": 0.00013840975129063684, + "loss": 0.1548, + "step": 3640 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001382481347550526, + "loss": 0.1565, + "step": 3645 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001380864011112285, + "loss": 0.1497, + "step": 3650 + }, + { + "epoch": 1.18, + "learning_rate": 0.000137924550854361, + "loss": 0.1569, + "step": 3655 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001377625844800037, + "loss": 0.1582, + "step": 3660 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013760050248406557, + "loss": 0.1571, + "step": 3665 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001374383053628098, + "loss": 0.1524, + "step": 3670 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013727599361285187, + "loss": 0.1567, + "step": 3675 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013711356773115838, + "loss": 0.1548, + "step": 3680 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013695102821504533, + "loss": 0.1557, + "step": 3685 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001367883755621766, + "loss": 0.148, + "step": 3690 + }, + { + "epoch": 1.2, + "learning_rate": 0.00013662561027056252, + "loss": 0.1604, + "step": 3695 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001364627328385583, + "loss": 0.15, + "step": 3700 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001362997437648625, + "loss": 0.1476, + "step": 3705 + }, + { + "epoch": 1.2, + "learning_rate": 0.00013613664354851547, + "loss": 0.1586, + "step": 3710 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001359734326888979, + "loss": 0.1536, + "step": 3715 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013581011168572923, + "loss": 0.1564, + "step": 3720 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013564668103906616, + "loss": 0.1534, + "step": 3725 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013548314124930104, + "loss": 0.1521, + "step": 3730 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013531949281716044, + "loss": 0.1489, + "step": 3735 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013515573624370357, + "loss": 0.1528, + "step": 3740 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001349918720303207, + "loss": 0.1405, + "step": 3745 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013482790067873178, + "loss": 0.1457, + "step": 3750 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013466382269098468, + "loss": 0.1554, + "step": 3755 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013449963856945383, + "loss": 0.1478, + "step": 3760 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013433534881683858, + "loss": 0.1633, + "step": 3765 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013417095393616178, + "loss": 0.15, + "step": 3770 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013400645443076804, + "loss": 0.1552, + "step": 3775 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013384185080432244, + "loss": 0.1485, + "step": 3780 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001336771435608088, + "loss": 0.1531, + "step": 3785 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013351233320452812, + "loss": 0.158, + "step": 3790 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013334742024009725, + "loss": 0.1554, + "step": 3795 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013318240517244713, + "loss": 0.1537, + "step": 3800 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001330172885068213, + "loss": 0.1584, + "step": 3805 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013285207074877447, + "loss": 0.1612, + "step": 3810 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013268675240417073, + "loss": 0.1449, + "step": 3815 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013252133397918226, + "loss": 0.1456, + "step": 3820 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001323558159802876, + "loss": 0.1541, + "step": 3825 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013219019891427025, + "loss": 0.1492, + "step": 3830 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013202448328821694, + "loss": 0.1528, + "step": 3835 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013185866960951622, + "loss": 0.1516, + "step": 3840 + }, + { + "epoch": 1.25, + "learning_rate": 0.0001316927583858569, + "loss": 0.1484, + "step": 3845 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013152675012522629, + "loss": 0.1509, + "step": 3850 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013136064533590897, + "loss": 0.1516, + "step": 3855 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013119444452648504, + "loss": 0.15, + "step": 3860 + }, + { + "epoch": 1.25, + "learning_rate": 0.0001310281482058286, + "loss": 0.1571, + "step": 3865 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013086175688310612, + "loss": 0.1526, + "step": 3870 + }, + { + "epoch": 1.26, + "learning_rate": 0.000130695271067775, + "loss": 0.1445, + "step": 3875 + }, + { + "epoch": 1.26, + "learning_rate": 0.00013052869126958198, + "loss": 0.1598, + "step": 3880 + }, + { + "epoch": 1.26, + "learning_rate": 0.00013036201799856149, + "loss": 0.1499, + "step": 3885 + }, + { + "epoch": 1.26, + "learning_rate": 0.0001301952517650342, + "loss": 0.1507, + "step": 3890 + }, + { + "epoch": 1.26, + "learning_rate": 0.0001300283930796054, + "loss": 0.1489, + "step": 3895 + }, + { + "epoch": 1.26, + "learning_rate": 0.00012986144245316346, + "loss": 0.1477, + "step": 3900 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012969440039687822, + "loss": 0.1528, + "step": 3905 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012952726742219953, + "loss": 0.1514, + "step": 3910 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012936004404085557, + "loss": 0.1482, + "step": 3915 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012919273076485133, + "loss": 0.1558, + "step": 3920 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012902532810646697, + "loss": 0.1539, + "step": 3925 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012885783657825647, + "loss": 0.1479, + "step": 3930 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012869025669304585, + "loss": 0.1449, + "step": 3935 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012852258896393158, + "loss": 0.1489, + "step": 3940 + }, + { + "epoch": 1.28, + "learning_rate": 0.0001283548339042792, + "loss": 0.1479, + "step": 3945 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012818699202772152, + "loss": 0.149, + "step": 3950 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012801906384815733, + "loss": 0.1472, + "step": 3955 + }, + { + "epoch": 1.28, + "learning_rate": 0.0001278510498797495, + "loss": 0.1526, + "step": 3960 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012768295063692376, + "loss": 0.1469, + "step": 3965 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012751476663436668, + "loss": 0.1576, + "step": 3970 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012734649838702455, + "loss": 0.1583, + "step": 3975 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012717814641010153, + "loss": 0.1434, + "step": 3980 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012700971121905812, + "loss": 0.1463, + "step": 3985 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012684119332960963, + "loss": 0.1543, + "step": 3990 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012667259325772464, + "loss": 0.1501, + "step": 3995 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012650391151962324, + "loss": 0.1475, + "step": 4000 + } + ], + "logging_steps": 5, + "max_steps": 9255, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 1000, + "total_flos": 8.151063543299441e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/save_lora_math_r8_2e4/checkpoint-4000/training_args.bin b/save_lora_math_r8_2e4/checkpoint-4000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..175cae96ce99657f1e2b29179eee355d72511307 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-4000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0cedf67f015d3f83c601cebdd463c220d745d8e241fab282366821363eb224d +size 5880 diff --git a/save_lora_math_r8_2e4/checkpoint-4000/zero_to_fp32.py b/save_lora_math_r8_2e4/checkpoint-4000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..c98caae31534368be22b67fc4ae906836c992a8d --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-4000/zero_to_fp32.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in param_shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, args.output_file, tag=args.tag) diff --git a/save_lora_math_r8_2e4/checkpoint-5000/README.md b/save_lora_math_r8_2e4/checkpoint-5000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d380bbbf0bf0fa1f211b53410559607e5007a209 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-5000/README.md @@ -0,0 +1,202 @@ +--- +library_name: peft +base_model: meta-llama/Llama-2-7b-hf +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.9.0 \ No newline at end of file diff --git a/save_lora_math_r8_2e4/checkpoint-5000/adapter_config.json b/save_lora_math_r8_2e4/checkpoint-5000/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9c81a592f0ff08374a4f61f6003029070a8429ae --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-5000/adapter_config.json @@ -0,0 +1,35 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "meta-llama/Llama-2-7b-hf", + "bias": "none", + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "mora_type": 1, + "peft_type": "LORA", + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "down_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "use_dora": false, + "use_mora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/save_lora_math_r8_2e4/checkpoint-5000/adapter_model.safetensors b/save_lora_math_r8_2e4/checkpoint-5000/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c029180c8d4b08924e717756b57f2ed151b52c59 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-5000/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45125977a85a639b320366474d174adff40e7e0c36bd118d2f356f6e9edeb710 +size 40036488 diff --git a/save_lora_math_r8_2e4/checkpoint-5000/global_step5000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/save_lora_math_r8_2e4/checkpoint-5000/global_step5000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c800254f0dab6815e15a40d4b1d7e7cf0900ca4 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-5000/global_step5000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b76f47795b19cbe31b07cab8590b8488c3f6811590c8f286c189ac2fcf08bdd2 +size 119958768 diff --git a/save_lora_math_r8_2e4/checkpoint-5000/global_step5000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/save_lora_math_r8_2e4/checkpoint-5000/global_step5000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a9974a8624f07acb2923354a496a889a1379d6a3 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-5000/global_step5000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1897f6e2f92d84eae51ca5c2a17e2ba144befc925db3c9e256e9b99461114d2d +size 119958896 diff --git a/save_lora_math_r8_2e4/checkpoint-5000/global_step5000/mp_rank_00_model_states.pt b/save_lora_math_r8_2e4/checkpoint-5000/global_step5000/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9abe9f4192fbe451501503c0fdb984a7270a7437 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-5000/global_step5000/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f347ce71710007c241724d1bd9f5162e6381ea91bd8cce751d6bfa948ded9ff1 +size 40303404 diff --git a/save_lora_math_r8_2e4/checkpoint-5000/latest b/save_lora_math_r8_2e4/checkpoint-5000/latest new file mode 100644 index 0000000000000000000000000000000000000000..f805186fa43374540c3fa51dfd3cca9ac06e56a5 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-5000/latest @@ -0,0 +1 @@ +global_step5000 \ No newline at end of file diff --git a/save_lora_math_r8_2e4/checkpoint-5000/rng_state_0.pth b/save_lora_math_r8_2e4/checkpoint-5000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..9bb507f4de7b0fa57da99e2278d50001ab1afea1 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-5000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:551fbf0e804cc089b106b7e0e662f145157d84151648f1b0ed0f742e014cc70a +size 14512 diff --git a/save_lora_math_r8_2e4/checkpoint-5000/rng_state_1.pth b/save_lora_math_r8_2e4/checkpoint-5000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..78aa8b0e44b3eb3d89b6f21c450b0120ff50e286 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-5000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f89ef5b8eb355c34a5d89e00c8f42e1a779753f313a379ed301e4e6509f4d718 +size 14512 diff --git a/save_lora_math_r8_2e4/checkpoint-5000/scheduler.pt b/save_lora_math_r8_2e4/checkpoint-5000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce8c642773e9efddf5d90c0e0d11d41c98f2a1dc --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-5000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75c9fc699a32795404b1be2a515e92adfbb1f5a61895b9b956a273e5bd9e3872 +size 1064 diff --git a/save_lora_math_r8_2e4/checkpoint-5000/trainer_state.json b/save_lora_math_r8_2e4/checkpoint-5000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..11fe9252efbea9d37b7f292b1940b705d5a8561d --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-5000/trainer_state.json @@ -0,0 +1,6021 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.620253164556962, + "eval_steps": 500, + "global_step": 5000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 3.5971223021582732e-06, + "loss": 0.7628, + "step": 5 + }, + { + "epoch": 0.0, + "learning_rate": 7.1942446043165465e-06, + "loss": 0.769, + "step": 10 + }, + { + "epoch": 0.0, + "learning_rate": 1.0791366906474821e-05, + "loss": 0.7619, + "step": 15 + }, + { + "epoch": 0.01, + "learning_rate": 1.4388489208633093e-05, + "loss": 0.7564, + "step": 20 + }, + { + "epoch": 0.01, + "learning_rate": 1.7985611510791367e-05, + "loss": 0.7549, + "step": 25 + }, + { + "epoch": 0.01, + "learning_rate": 2.1582733812949642e-05, + "loss": 0.7001, + "step": 30 + }, + { + "epoch": 0.01, + "learning_rate": 2.5179856115107914e-05, + "loss": 0.6289, + "step": 35 + }, + { + "epoch": 0.01, + "learning_rate": 2.8776978417266186e-05, + "loss": 0.5632, + "step": 40 + }, + { + "epoch": 0.01, + "learning_rate": 3.237410071942446e-05, + "loss": 0.4923, + "step": 45 + }, + { + "epoch": 0.02, + "learning_rate": 3.597122302158273e-05, + "loss": 0.4582, + "step": 50 + }, + { + "epoch": 0.02, + "learning_rate": 3.956834532374101e-05, + "loss": 0.4357, + "step": 55 + }, + { + "epoch": 0.02, + "learning_rate": 4.3165467625899284e-05, + "loss": 0.4079, + "step": 60 + }, + { + "epoch": 0.02, + "learning_rate": 4.676258992805755e-05, + "loss": 0.4047, + "step": 65 + }, + { + "epoch": 0.02, + "learning_rate": 5.035971223021583e-05, + "loss": 0.3916, + "step": 70 + }, + { + "epoch": 0.02, + "learning_rate": 5.39568345323741e-05, + "loss": 0.3926, + "step": 75 + }, + { + "epoch": 0.03, + "learning_rate": 5.755395683453237e-05, + "loss": 0.3725, + "step": 80 + }, + { + "epoch": 0.03, + "learning_rate": 6.115107913669065e-05, + "loss": 0.3769, + "step": 85 + }, + { + "epoch": 0.03, + "learning_rate": 6.474820143884892e-05, + "loss": 0.352, + "step": 90 + }, + { + "epoch": 0.03, + "learning_rate": 6.83453237410072e-05, + "loss": 0.3534, + "step": 95 + }, + { + "epoch": 0.03, + "learning_rate": 7.194244604316547e-05, + "loss": 0.3432, + "step": 100 + }, + { + "epoch": 0.03, + "learning_rate": 7.553956834532374e-05, + "loss": 0.3286, + "step": 105 + }, + { + "epoch": 0.04, + "learning_rate": 7.913669064748202e-05, + "loss": 0.3298, + "step": 110 + }, + { + "epoch": 0.04, + "learning_rate": 8.273381294964029e-05, + "loss": 0.3223, + "step": 115 + }, + { + "epoch": 0.04, + "learning_rate": 8.633093525179857e-05, + "loss": 0.3181, + "step": 120 + }, + { + "epoch": 0.04, + "learning_rate": 8.992805755395684e-05, + "loss": 0.3225, + "step": 125 + }, + { + "epoch": 0.04, + "learning_rate": 9.35251798561151e-05, + "loss": 0.3145, + "step": 130 + }, + { + "epoch": 0.04, + "learning_rate": 9.712230215827339e-05, + "loss": 0.3137, + "step": 135 + }, + { + "epoch": 0.05, + "learning_rate": 0.00010071942446043166, + "loss": 0.3102, + "step": 140 + }, + { + "epoch": 0.05, + "learning_rate": 0.00010431654676258992, + "loss": 0.2951, + "step": 145 + }, + { + "epoch": 0.05, + "learning_rate": 0.0001079136690647482, + "loss": 0.3021, + "step": 150 + }, + { + "epoch": 0.05, + "learning_rate": 0.00011151079136690648, + "loss": 0.2923, + "step": 155 + }, + { + "epoch": 0.05, + "learning_rate": 0.00011510791366906474, + "loss": 0.2953, + "step": 160 + }, + { + "epoch": 0.05, + "learning_rate": 0.00011870503597122303, + "loss": 0.2893, + "step": 165 + }, + { + "epoch": 0.06, + "learning_rate": 0.0001223021582733813, + "loss": 0.2953, + "step": 170 + }, + { + "epoch": 0.06, + "learning_rate": 0.00012589928057553956, + "loss": 0.2931, + "step": 175 + }, + { + "epoch": 0.06, + "learning_rate": 0.00012949640287769783, + "loss": 0.2879, + "step": 180 + }, + { + "epoch": 0.06, + "learning_rate": 0.00013309352517985613, + "loss": 0.2642, + "step": 185 + }, + { + "epoch": 0.06, + "learning_rate": 0.0001366906474820144, + "loss": 0.2772, + "step": 190 + }, + { + "epoch": 0.06, + "learning_rate": 0.00014028776978417266, + "loss": 0.2858, + "step": 195 + }, + { + "epoch": 0.06, + "learning_rate": 0.00014388489208633093, + "loss": 0.2724, + "step": 200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0001474820143884892, + "loss": 0.2726, + "step": 205 + }, + { + "epoch": 0.07, + "learning_rate": 0.00015107913669064747, + "loss": 0.2678, + "step": 210 + }, + { + "epoch": 0.07, + "learning_rate": 0.00015467625899280577, + "loss": 0.2789, + "step": 215 + }, + { + "epoch": 0.07, + "learning_rate": 0.00015827338129496403, + "loss": 0.2742, + "step": 220 + }, + { + "epoch": 0.07, + "learning_rate": 0.0001618705035971223, + "loss": 0.2732, + "step": 225 + }, + { + "epoch": 0.07, + "learning_rate": 0.00016546762589928057, + "loss": 0.2741, + "step": 230 + }, + { + "epoch": 0.08, + "learning_rate": 0.00016906474820143884, + "loss": 0.2665, + "step": 235 + }, + { + "epoch": 0.08, + "learning_rate": 0.00017266187050359714, + "loss": 0.2771, + "step": 240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0001762589928057554, + "loss": 0.2751, + "step": 245 + }, + { + "epoch": 0.08, + "learning_rate": 0.00017985611510791367, + "loss": 0.2597, + "step": 250 + }, + { + "epoch": 0.08, + "learning_rate": 0.00018345323741007194, + "loss": 0.275, + "step": 255 + }, + { + "epoch": 0.08, + "learning_rate": 0.0001870503597122302, + "loss": 0.2791, + "step": 260 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019064748201438848, + "loss": 0.2493, + "step": 265 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019424460431654677, + "loss": 0.27, + "step": 270 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019784172661870504, + "loss": 0.2676, + "step": 275 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019999997550557367, + "loss": 0.2613, + "step": 280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019999969994341502, + "loss": 0.2747, + "step": 285 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019999911820191125, + "loss": 0.2699, + "step": 290 + }, + { + "epoch": 0.1, + "learning_rate": 0.0001999982302828436, + "loss": 0.2692, + "step": 295 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999703618893072, + "loss": 0.2522, + "step": 300 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999553592382863, + "loss": 0.2541, + "step": 305 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999372949213085, + "loss": 0.2614, + "step": 310 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999161689936837, + "loss": 0.2481, + "step": 315 + }, + { + "epoch": 0.1, + "learning_rate": 0.0001999891981520095, + "loss": 0.2497, + "step": 320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019998647325745995, + "loss": 0.2545, + "step": 325 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019998344222406283, + "loss": 0.2501, + "step": 330 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019998010506109854, + "loss": 0.2577, + "step": 335 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019997646177878487, + "loss": 0.2585, + "step": 340 + }, + { + "epoch": 0.11, + "learning_rate": 0.0001999725123882768, + "loss": 0.2494, + "step": 345 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019996825690166657, + "loss": 0.2538, + "step": 350 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019996369533198364, + "loss": 0.2469, + "step": 355 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019995882769319466, + "loss": 0.2502, + "step": 360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0001999536540002034, + "loss": 0.2361, + "step": 365 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019994817426885057, + "loss": 0.2446, + "step": 370 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019994238851591417, + "loss": 0.256, + "step": 375 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019993629675910896, + "loss": 0.2352, + "step": 380 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019992989901708672, + "loss": 0.2538, + "step": 385 + }, + { + "epoch": 0.13, + "learning_rate": 0.0001999231953094361, + "loss": 0.2523, + "step": 390 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019991618565668243, + "loss": 0.2512, + "step": 395 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019990887008028798, + "loss": 0.2402, + "step": 400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019990124860265156, + "loss": 0.2423, + "step": 405 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019989332124710862, + "loss": 0.2335, + "step": 410 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019988508803793124, + "loss": 0.2284, + "step": 415 + }, + { + "epoch": 0.14, + "learning_rate": 0.00019987654900032774, + "loss": 0.2382, + "step": 420 + }, + { + "epoch": 0.14, + "learning_rate": 0.0001998677041604431, + "loss": 0.2526, + "step": 425 + }, + { + "epoch": 0.14, + "learning_rate": 0.0001998585535453584, + "loss": 0.2417, + "step": 430 + }, + { + "epoch": 0.14, + "learning_rate": 0.00019984909718309105, + "loss": 0.2381, + "step": 435 + }, + { + "epoch": 0.14, + "learning_rate": 0.0001998393351025945, + "loss": 0.2418, + "step": 440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00019982926733375843, + "loss": 0.2409, + "step": 445 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001998188939074083, + "loss": 0.2467, + "step": 450 + }, + { + "epoch": 0.15, + "learning_rate": 0.00019980821485530544, + "loss": 0.2305, + "step": 455 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001997972302101471, + "loss": 0.232, + "step": 460 + }, + { + "epoch": 0.15, + "learning_rate": 0.00019978594000556602, + "loss": 0.225, + "step": 465 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001997743442761306, + "loss": 0.2259, + "step": 470 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001997624430573447, + "loss": 0.2414, + "step": 475 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019975023638564747, + "loss": 0.238, + "step": 480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019973772429841336, + "loss": 0.224, + "step": 485 + }, + { + "epoch": 0.16, + "learning_rate": 0.0001997249068339519, + "loss": 0.2369, + "step": 490 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019971178403150762, + "loss": 0.2393, + "step": 495 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019969835593125997, + "loss": 0.2249, + "step": 500 + }, + { + "epoch": 0.16, + "learning_rate": 0.0001996846225743231, + "loss": 0.2369, + "step": 505 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019967058400274592, + "loss": 0.2267, + "step": 510 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019965624025951168, + "loss": 0.2294, + "step": 515 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019964159138853815, + "loss": 0.2255, + "step": 520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019962663743467726, + "loss": 0.22, + "step": 525 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019961137844371507, + "loss": 0.2398, + "step": 530 + }, + { + "epoch": 0.17, + "learning_rate": 0.0001995958144623716, + "loss": 0.2431, + "step": 535 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019957994553830067, + "loss": 0.2245, + "step": 540 + }, + { + "epoch": 0.18, + "learning_rate": 0.0001995637717200898, + "loss": 0.2295, + "step": 545 + }, + { + "epoch": 0.18, + "learning_rate": 0.00019954729305726008, + "loss": 0.2288, + "step": 550 + }, + { + "epoch": 0.18, + "learning_rate": 0.00019953050960026586, + "loss": 0.2333, + "step": 555 + }, + { + "epoch": 0.18, + "learning_rate": 0.0001995134214004948, + "loss": 0.2316, + "step": 560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0001994960285102676, + "loss": 0.2285, + "step": 565 + }, + { + "epoch": 0.18, + "learning_rate": 0.00019947833098283782, + "loss": 0.2273, + "step": 570 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019946032887239182, + "loss": 0.2344, + "step": 575 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019944202223404854, + "loss": 0.2251, + "step": 580 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019942341112385922, + "loss": 0.226, + "step": 585 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019940449559880752, + "loss": 0.2353, + "step": 590 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019938527571680892, + "loss": 0.226, + "step": 595 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019936575153671097, + "loss": 0.221, + "step": 600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019934592311829285, + "loss": 0.2219, + "step": 605 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019932579052226528, + "loss": 0.234, + "step": 610 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019930535381027027, + "loss": 0.2181, + "step": 615 + }, + { + "epoch": 0.2, + "learning_rate": 0.000199284613044881, + "loss": 0.2191, + "step": 620 + }, + { + "epoch": 0.2, + "learning_rate": 0.0001992635682896016, + "loss": 0.235, + "step": 625 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019924221960886698, + "loss": 0.2256, + "step": 630 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019922056706804257, + "loss": 0.2265, + "step": 635 + }, + { + "epoch": 0.21, + "learning_rate": 0.0001991986107334242, + "loss": 0.2158, + "step": 640 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019917635067223777, + "loss": 0.2238, + "step": 645 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019915378695263924, + "loss": 0.2182, + "step": 650 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019913091964371423, + "loss": 0.2185, + "step": 655 + }, + { + "epoch": 0.21, + "learning_rate": 0.000199107748815478, + "loss": 0.2063, + "step": 660 + }, + { + "epoch": 0.22, + "learning_rate": 0.000199084274538875, + "loss": 0.2162, + "step": 665 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019906049688577887, + "loss": 0.2185, + "step": 670 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019903641592899205, + "loss": 0.2186, + "step": 675 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019901203174224564, + "loss": 0.2292, + "step": 680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019898734440019933, + "loss": 0.2162, + "step": 685 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019896235397844078, + "loss": 0.219, + "step": 690 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019893706055348578, + "loss": 0.218, + "step": 695 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019891146420277777, + "loss": 0.2211, + "step": 700 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019888556500468775, + "loss": 0.2227, + "step": 705 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019885936303851395, + "loss": 0.2214, + "step": 710 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019883285838448163, + "loss": 0.2204, + "step": 715 + }, + { + "epoch": 0.23, + "learning_rate": 0.0001988060511237428, + "loss": 0.2228, + "step": 720 + }, + { + "epoch": 0.23, + "learning_rate": 0.000198778941338376, + "loss": 0.2259, + "step": 725 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019875152911138602, + "loss": 0.2062, + "step": 730 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019872381452670372, + "loss": 0.2073, + "step": 735 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019869579766918568, + "loss": 0.2172, + "step": 740 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019866747862461396, + "loss": 0.21, + "step": 745 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019863885747969598, + "loss": 0.2195, + "step": 750 + }, + { + "epoch": 0.24, + "learning_rate": 0.0001986099343220639, + "loss": 0.2129, + "step": 755 + }, + { + "epoch": 0.25, + "learning_rate": 0.0001985807092402748, + "loss": 0.215, + "step": 760 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019855118232381015, + "loss": 0.2085, + "step": 765 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019852135366307545, + "loss": 0.2196, + "step": 770 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019849122334940022, + "loss": 0.2129, + "step": 775 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019846079147503755, + "loss": 0.2221, + "step": 780 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019843005813316377, + "loss": 0.2133, + "step": 785 + }, + { + "epoch": 0.26, + "learning_rate": 0.0001983990234178784, + "loss": 0.2066, + "step": 790 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019836768742420352, + "loss": 0.2188, + "step": 795 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019833605024808382, + "loss": 0.2128, + "step": 800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019830411198638607, + "loss": 0.2109, + "step": 805 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019827187273689894, + "loss": 0.2141, + "step": 810 + }, + { + "epoch": 0.26, + "learning_rate": 0.0001982393325983326, + "loss": 0.2098, + "step": 815 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019820649167031856, + "loss": 0.2188, + "step": 820 + }, + { + "epoch": 0.27, + "learning_rate": 0.0001981733500534093, + "loss": 0.2107, + "step": 825 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019813990784907785, + "loss": 0.2162, + "step": 830 + }, + { + "epoch": 0.27, + "learning_rate": 0.0001981061651597177, + "loss": 0.2189, + "step": 835 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019807212208864227, + "loss": 0.2117, + "step": 840 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019803777874008468, + "loss": 0.209, + "step": 845 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019800313521919753, + "loss": 0.206, + "step": 850 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019796819163205246, + "loss": 0.2104, + "step": 855 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019793294808563984, + "loss": 0.2016, + "step": 860 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019789740468786841, + "loss": 0.212, + "step": 865 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019786156154756506, + "loss": 0.2051, + "step": 870 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019782541877447443, + "loss": 0.21, + "step": 875 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019778897647925855, + "loss": 0.2104, + "step": 880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0001977522347734965, + "loss": 0.2179, + "step": 885 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019771519376968416, + "loss": 0.2139, + "step": 890 + }, + { + "epoch": 0.29, + "learning_rate": 0.0001976778535812338, + "loss": 0.2034, + "step": 895 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019764021432247368, + "loss": 0.2067, + "step": 900 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019760227610864776, + "loss": 0.2189, + "step": 905 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019756403905591538, + "loss": 0.2119, + "step": 910 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019752550328135086, + "loss": 0.2004, + "step": 915 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019748666890294315, + "loss": 0.2073, + "step": 920 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019744753603959545, + "loss": 0.1995, + "step": 925 + }, + { + "epoch": 0.3, + "learning_rate": 0.0001974081048111248, + "loss": 0.2034, + "step": 930 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019736837533826192, + "loss": 0.2015, + "step": 935 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019732834774265057, + "loss": 0.2166, + "step": 940 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019728802214684734, + "loss": 0.1987, + "step": 945 + }, + { + "epoch": 0.31, + "learning_rate": 0.0001972473986743213, + "loss": 0.2051, + "step": 950 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019720647744945344, + "loss": 0.2055, + "step": 955 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019716525859753652, + "loss": 0.2005, + "step": 960 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019712374224477452, + "loss": 0.1995, + "step": 965 + }, + { + "epoch": 0.31, + "learning_rate": 0.0001970819285182823, + "loss": 0.2096, + "step": 970 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019703981754608525, + "loss": 0.2024, + "step": 975 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019699740945711884, + "loss": 0.2006, + "step": 980 + }, + { + "epoch": 0.32, + "learning_rate": 0.0001969547043812283, + "loss": 0.2077, + "step": 985 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019691170244916814, + "loss": 0.2094, + "step": 990 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019686840379260176, + "loss": 0.2051, + "step": 995 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019682480854410115, + "loss": 0.2158, + "step": 1000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0001967809168371463, + "loss": 0.2035, + "step": 1005 + }, + { + "epoch": 0.33, + "learning_rate": 0.000196736728806125, + "loss": 0.2104, + "step": 1010 + }, + { + "epoch": 0.33, + "learning_rate": 0.00019669224458633222, + "loss": 0.1969, + "step": 1015 + }, + { + "epoch": 0.33, + "learning_rate": 0.0001966474643139699, + "loss": 0.2058, + "step": 1020 + }, + { + "epoch": 0.33, + "learning_rate": 0.00019660238812614647, + "loss": 0.2045, + "step": 1025 + }, + { + "epoch": 0.33, + "learning_rate": 0.0001965570161608762, + "loss": 0.1916, + "step": 1030 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019651134855707913, + "loss": 0.2008, + "step": 1035 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019646538545458047, + "loss": 0.2095, + "step": 1040 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019641912699411016, + "loss": 0.2009, + "step": 1045 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019637257331730248, + "loss": 0.2059, + "step": 1050 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019632572456669558, + "loss": 0.2084, + "step": 1055 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019627858088573112, + "loss": 0.1997, + "step": 1060 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001962311424187537, + "loss": 0.2068, + "step": 1065 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001961834093110106, + "loss": 0.1978, + "step": 1070 + }, + { + "epoch": 0.35, + "learning_rate": 0.00019613538170865118, + "loss": 0.2028, + "step": 1075 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001960870597587264, + "loss": 0.1996, + "step": 1080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001960384436091886, + "loss": 0.2002, + "step": 1085 + }, + { + "epoch": 0.35, + "learning_rate": 0.00019598953340889082, + "loss": 0.198, + "step": 1090 + }, + { + "epoch": 0.35, + "learning_rate": 0.00019594032930758643, + "loss": 0.1974, + "step": 1095 + }, + { + "epoch": 0.36, + "learning_rate": 0.0001958908314559287, + "loss": 0.1983, + "step": 1100 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019584104000547025, + "loss": 0.1987, + "step": 1105 + }, + { + "epoch": 0.36, + "learning_rate": 0.0001957909551086627, + "loss": 0.2003, + "step": 1110 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019574057691885613, + "loss": 0.1982, + "step": 1115 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019568990559029858, + "loss": 0.2031, + "step": 1120 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019563894127813564, + "loss": 0.1969, + "step": 1125 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019558768413841006, + "loss": 0.1916, + "step": 1130 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019553613432806102, + "loss": 0.2124, + "step": 1135 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019548429200492388, + "loss": 0.2042, + "step": 1140 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019543215732772956, + "loss": 0.201, + "step": 1145 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019537973045610424, + "loss": 0.2001, + "step": 1150 + }, + { + "epoch": 0.37, + "learning_rate": 0.0001953270115505686, + "loss": 0.1978, + "step": 1155 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019527400077253755, + "loss": 0.192, + "step": 1160 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019522069828431957, + "loss": 0.1946, + "step": 1165 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019516710424911642, + "loss": 0.1848, + "step": 1170 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019511321883102244, + "loss": 0.2005, + "step": 1175 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019505904219502414, + "loss": 0.1966, + "step": 1180 + }, + { + "epoch": 0.38, + "learning_rate": 0.0001950045745069997, + "loss": 0.1863, + "step": 1185 + }, + { + "epoch": 0.39, + "learning_rate": 0.0001949498159337184, + "loss": 0.1983, + "step": 1190 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019489476664284023, + "loss": 0.2113, + "step": 1195 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019483942680291523, + "loss": 0.1956, + "step": 1200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019478379658338305, + "loss": 0.2055, + "step": 1205 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019472787615457246, + "loss": 0.1879, + "step": 1210 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019467166568770078, + "loss": 0.1924, + "step": 1215 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019461516535487338, + "loss": 0.1971, + "step": 1220 + }, + { + "epoch": 0.4, + "learning_rate": 0.0001945583753290831, + "loss": 0.1952, + "step": 1225 + }, + { + "epoch": 0.4, + "learning_rate": 0.0001945012957842098, + "loss": 0.1838, + "step": 1230 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019444392689501978, + "loss": 0.1931, + "step": 1235 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019438626883716526, + "loss": 0.2093, + "step": 1240 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019432832178718385, + "loss": 0.1992, + "step": 1245 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019427008592249795, + "loss": 0.1963, + "step": 1250 + }, + { + "epoch": 0.41, + "learning_rate": 0.0001942115614214143, + "loss": 0.2003, + "step": 1255 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019415274846312338, + "loss": 0.1914, + "step": 1260 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019409364722769882, + "loss": 0.1917, + "step": 1265 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019403425789609697, + "loss": 0.2033, + "step": 1270 + }, + { + "epoch": 0.41, + "learning_rate": 0.0001939745806501563, + "loss": 0.1975, + "step": 1275 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019391461567259662, + "loss": 0.1922, + "step": 1280 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019385436314701897, + "loss": 0.1999, + "step": 1285 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019379382325790468, + "loss": 0.1997, + "step": 1290 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019373299619061487, + "loss": 0.1872, + "step": 1295 + }, + { + "epoch": 0.42, + "learning_rate": 0.0001936718821313901, + "loss": 0.1936, + "step": 1300 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019361048126734952, + "loss": 0.1798, + "step": 1305 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019354879378649046, + "loss": 0.1959, + "step": 1310 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019348681987768783, + "loss": 0.1924, + "step": 1315 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019342455973069345, + "loss": 0.1967, + "step": 1320 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019336201353613568, + "loss": 0.1925, + "step": 1325 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019329918148551856, + "loss": 0.1878, + "step": 1330 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019323606377122146, + "loss": 0.1979, + "step": 1335 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019317266058649834, + "loss": 0.189, + "step": 1340 + }, + { + "epoch": 0.44, + "learning_rate": 0.0001931089721254773, + "loss": 0.1995, + "step": 1345 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019304499858315976, + "loss": 0.1874, + "step": 1350 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019298074015542015, + "loss": 0.1991, + "step": 1355 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019291619703900502, + "loss": 0.188, + "step": 1360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0001928513694315327, + "loss": 0.1914, + "step": 1365 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019278625753149254, + "loss": 0.1843, + "step": 1370 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019272086153824427, + "loss": 0.1959, + "step": 1375 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019265518165201758, + "loss": 0.1867, + "step": 1380 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019258921807391131, + "loss": 0.1956, + "step": 1385 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019252297100589286, + "loss": 0.1991, + "step": 1390 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019245644065079775, + "loss": 0.1934, + "step": 1395 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019238962721232876, + "loss": 0.1953, + "step": 1400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019232253089505546, + "loss": 0.1865, + "step": 1405 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019225515190441354, + "loss": 0.1864, + "step": 1410 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019218749044670413, + "loss": 0.194, + "step": 1415 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019211954672909327, + "loss": 0.1853, + "step": 1420 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019205132095961126, + "loss": 0.1967, + "step": 1425 + }, + { + "epoch": 0.46, + "learning_rate": 0.0001919828133471519, + "loss": 0.1852, + "step": 1430 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019191402410147204, + "loss": 0.1913, + "step": 1435 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019184495343319068, + "loss": 0.1911, + "step": 1440 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019177560155378865, + "loss": 0.189, + "step": 1445 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019170596867560766, + "loss": 0.1854, + "step": 1450 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019163605501184986, + "loss": 0.1974, + "step": 1455 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019156586077657712, + "loss": 0.1933, + "step": 1460 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019149538618471032, + "loss": 0.1877, + "step": 1465 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001914246314520287, + "loss": 0.1904, + "step": 1470 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001913535967951694, + "loss": 0.1841, + "step": 1475 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001912822824316264, + "loss": 0.1876, + "step": 1480 + }, + { + "epoch": 0.48, + "learning_rate": 0.00019121068857975027, + "loss": 0.1858, + "step": 1485 + }, + { + "epoch": 0.48, + "learning_rate": 0.00019113881545874722, + "loss": 0.1898, + "step": 1490 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001910666632886786, + "loss": 0.1923, + "step": 1495 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019099423229046014, + "loss": 0.1803, + "step": 1500 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019092152268586118, + "loss": 0.1945, + "step": 1505 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019084853469750424, + "loss": 0.1802, + "step": 1510 + }, + { + "epoch": 0.49, + "learning_rate": 0.0001907752685488641, + "loss": 0.197, + "step": 1515 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019070172446426724, + "loss": 0.1791, + "step": 1520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019062790266889117, + "loss": 0.1874, + "step": 1525 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019055380338876363, + "loss": 0.1806, + "step": 1530 + }, + { + "epoch": 0.5, + "learning_rate": 0.000190479426850762, + "loss": 0.1834, + "step": 1535 + }, + { + "epoch": 0.5, + "learning_rate": 0.0001904047732826126, + "loss": 0.1921, + "step": 1540 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019032984291288986, + "loss": 0.1826, + "step": 1545 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019025463597101584, + "loss": 0.1966, + "step": 1550 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019017915268725925, + "loss": 0.1867, + "step": 1555 + }, + { + "epoch": 0.51, + "learning_rate": 0.00019010339329273514, + "loss": 0.196, + "step": 1560 + }, + { + "epoch": 0.51, + "learning_rate": 0.00019002735801940373, + "loss": 0.1971, + "step": 1565 + }, + { + "epoch": 0.51, + "learning_rate": 0.00018995104710007004, + "loss": 0.187, + "step": 1570 + }, + { + "epoch": 0.51, + "learning_rate": 0.00018987446076838312, + "loss": 0.1897, + "step": 1575 + }, + { + "epoch": 0.51, + "learning_rate": 0.00018979759925883509, + "loss": 0.1887, + "step": 1580 + }, + { + "epoch": 0.51, + "learning_rate": 0.0001897204628067608, + "loss": 0.1849, + "step": 1585 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018964305164833678, + "loss": 0.1795, + "step": 1590 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018956536602058078, + "loss": 0.1984, + "step": 1595 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018948740616135085, + "loss": 0.1876, + "step": 1600 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018940917230934464, + "loss": 0.1879, + "step": 1605 + }, + { + "epoch": 0.52, + "learning_rate": 0.0001893306647040988, + "loss": 0.1906, + "step": 1610 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018925188358598813, + "loss": 0.1917, + "step": 1615 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018917282919622488, + "loss": 0.1912, + "step": 1620 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018909350177685795, + "loss": 0.1813, + "step": 1625 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018901390157077225, + "loss": 0.183, + "step": 1630 + }, + { + "epoch": 0.53, + "learning_rate": 0.0001889340288216879, + "loss": 0.1782, + "step": 1635 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018885388377415957, + "loss": 0.1866, + "step": 1640 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018877346667357548, + "loss": 0.1915, + "step": 1645 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018869277776615691, + "loss": 0.1902, + "step": 1650 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018861181729895745, + "loss": 0.1829, + "step": 1655 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018853058551986202, + "loss": 0.1855, + "step": 1660 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018844908267758633, + "loss": 0.1845, + "step": 1665 + }, + { + "epoch": 0.54, + "learning_rate": 0.000188367309021676, + "loss": 0.1794, + "step": 1670 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018828526480250575, + "loss": 0.1796, + "step": 1675 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018820295027127885, + "loss": 0.1806, + "step": 1680 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018812036568002617, + "loss": 0.1846, + "step": 1685 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018803751128160538, + "loss": 0.189, + "step": 1690 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018795438732970032, + "loss": 0.1856, + "step": 1695 + }, + { + "epoch": 0.55, + "learning_rate": 0.0001878709940788201, + "loss": 0.1797, + "step": 1700 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018778733178429844, + "loss": 0.1876, + "step": 1705 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018770340070229272, + "loss": 0.1775, + "step": 1710 + }, + { + "epoch": 0.56, + "learning_rate": 0.0001876192010897834, + "loss": 0.1862, + "step": 1715 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018753473320457306, + "loss": 0.1835, + "step": 1720 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018744999730528568, + "loss": 0.1909, + "step": 1725 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018736499365136584, + "loss": 0.189, + "step": 1730 + }, + { + "epoch": 0.56, + "learning_rate": 0.000187279722503078, + "loss": 0.1861, + "step": 1735 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018719418412150557, + "loss": 0.1809, + "step": 1740 + }, + { + "epoch": 0.57, + "learning_rate": 0.00018710837876855017, + "loss": 0.1829, + "step": 1745 + }, + { + "epoch": 0.57, + "learning_rate": 0.0001870223067069309, + "loss": 0.1886, + "step": 1750 + }, + { + "epoch": 0.57, + "learning_rate": 0.00018693596820018336, + "loss": 0.1787, + "step": 1755 + }, + { + "epoch": 0.57, + "learning_rate": 0.0001868493635126591, + "loss": 0.1816, + "step": 1760 + }, + { + "epoch": 0.57, + "learning_rate": 0.0001867624929095245, + "loss": 0.1759, + "step": 1765 + }, + { + "epoch": 0.57, + "learning_rate": 0.00018667535665676025, + "loss": 0.1702, + "step": 1770 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018658795502116035, + "loss": 0.1802, + "step": 1775 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018650028827033136, + "loss": 0.1836, + "step": 1780 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018641235667269155, + "loss": 0.1787, + "step": 1785 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018632416049747016, + "loss": 0.1811, + "step": 1790 + }, + { + "epoch": 0.58, + "learning_rate": 0.0001862357000147064, + "loss": 0.1835, + "step": 1795 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018614697549524889, + "loss": 0.1857, + "step": 1800 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018605798721075455, + "loss": 0.1867, + "step": 1805 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018596873543368797, + "loss": 0.1766, + "step": 1810 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018587922043732046, + "loss": 0.1895, + "step": 1815 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018578944249572925, + "loss": 0.1888, + "step": 1820 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018569940188379672, + "loss": 0.1871, + "step": 1825 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018560909887720944, + "loss": 0.1804, + "step": 1830 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018551853375245737, + "loss": 0.1877, + "step": 1835 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018542770678683308, + "loss": 0.182, + "step": 1840 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001853366182584308, + "loss": 0.1809, + "step": 1845 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018524526844614564, + "loss": 0.1913, + "step": 1850 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018515365762967266, + "loss": 0.185, + "step": 1855 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018506178608950613, + "loss": 0.1753, + "step": 1860 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018496965410693856, + "loss": 0.1786, + "step": 1865 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001848772619640599, + "loss": 0.1841, + "step": 1870 + }, + { + "epoch": 0.61, + "learning_rate": 0.00018478460994375667, + "loss": 0.184, + "step": 1875 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001846916983297111, + "loss": 0.179, + "step": 1880 + }, + { + "epoch": 0.61, + "learning_rate": 0.00018459852740640014, + "loss": 0.1833, + "step": 1885 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001845050974590948, + "loss": 0.186, + "step": 1890 + }, + { + "epoch": 0.61, + "learning_rate": 0.00018441140877385913, + "loss": 0.1778, + "step": 1895 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018431746163754943, + "loss": 0.1786, + "step": 1900 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018422325633781327, + "loss": 0.1826, + "step": 1905 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018412879316308866, + "loss": 0.1738, + "step": 1910 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018403407240260327, + "loss": 0.1852, + "step": 1915 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018393909434637329, + "loss": 0.1821, + "step": 1920 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001838438592852028, + "loss": 0.1773, + "step": 1925 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018374836751068282, + "loss": 0.1772, + "step": 1930 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018365261931519022, + "loss": 0.1766, + "step": 1935 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018355661499188717, + "loss": 0.1858, + "step": 1940 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018346035483471994, + "loss": 0.1786, + "step": 1945 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001833638391384181, + "loss": 0.1745, + "step": 1950 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018326706819849365, + "loss": 0.1713, + "step": 1955 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018317004231124017, + "loss": 0.1738, + "step": 1960 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018307276177373174, + "loss": 0.1807, + "step": 1965 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018297522688382218, + "loss": 0.1806, + "step": 1970 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018287743794014407, + "loss": 0.1735, + "step": 1975 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018277939524210783, + "loss": 0.1812, + "step": 1980 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001826810990899009, + "loss": 0.1734, + "step": 1985 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001825825497844867, + "loss": 0.1703, + "step": 1990 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018248374762760365, + "loss": 0.1823, + "step": 1995 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018238469292176457, + "loss": 0.1837, + "step": 2000 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018228538597025537, + "loss": 0.1807, + "step": 2005 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018218582707713434, + "loss": 0.1904, + "step": 2010 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001820860165472312, + "loss": 0.1844, + "step": 2015 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018198595468614604, + "loss": 0.1792, + "step": 2020 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018188564180024854, + "loss": 0.1845, + "step": 2025 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018178507819667698, + "loss": 0.1853, + "step": 2030 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018168426418333725, + "loss": 0.1875, + "step": 2035 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018158320006890198, + "loss": 0.1759, + "step": 2040 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018148188616280955, + "loss": 0.1816, + "step": 2045 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001813803227752631, + "loss": 0.1787, + "step": 2050 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001812785102172297, + "loss": 0.1758, + "step": 2055 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018117644880043932, + "loss": 0.1802, + "step": 2060 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018107413883738387, + "loss": 0.186, + "step": 2065 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018097158064131626, + "loss": 0.1743, + "step": 2070 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018086877452624944, + "loss": 0.1877, + "step": 2075 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018076572080695543, + "loss": 0.1742, + "step": 2080 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018066241979896444, + "loss": 0.1727, + "step": 2085 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018055887181856375, + "loss": 0.1806, + "step": 2090 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018045507718279677, + "loss": 0.175, + "step": 2095 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018035103620946225, + "loss": 0.1727, + "step": 2100 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001802467492171131, + "loss": 0.1779, + "step": 2105 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018014221652505556, + "loss": 0.1823, + "step": 2110 + }, + { + "epoch": 0.69, + "learning_rate": 0.000180037438453348, + "loss": 0.1814, + "step": 2115 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001799324153228003, + "loss": 0.1707, + "step": 2120 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001798271474549725, + "loss": 0.1778, + "step": 2125 + }, + { + "epoch": 0.69, + "learning_rate": 0.000179721635172174, + "loss": 0.1888, + "step": 2130 + }, + { + "epoch": 0.69, + "learning_rate": 0.00017961587879746266, + "loss": 0.1785, + "step": 2135 + }, + { + "epoch": 0.69, + "learning_rate": 0.00017950987865464365, + "loss": 0.1661, + "step": 2140 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017940363506826836, + "loss": 0.1792, + "step": 2145 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001792971483636338, + "loss": 0.1732, + "step": 2150 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017919041886678118, + "loss": 0.1771, + "step": 2155 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001790834469044952, + "loss": 0.173, + "step": 2160 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017897623280430284, + "loss": 0.1768, + "step": 2165 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017886877689447258, + "loss": 0.175, + "step": 2170 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001787610795040132, + "loss": 0.1691, + "step": 2175 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001786531409626728, + "loss": 0.1774, + "step": 2180 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001785449616009379, + "loss": 0.1709, + "step": 2185 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001784365417500324, + "loss": 0.1809, + "step": 2190 + }, + { + "epoch": 0.71, + "learning_rate": 0.00017832788174191645, + "loss": 0.1764, + "step": 2195 + }, + { + "epoch": 0.71, + "learning_rate": 0.00017821898190928558, + "loss": 0.1739, + "step": 2200 + }, + { + "epoch": 0.71, + "learning_rate": 0.00017810984258556957, + "loss": 0.1772, + "step": 2205 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017800046410493145, + "loss": 0.1779, + "step": 2210 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001778908468022666, + "loss": 0.1828, + "step": 2215 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017778099101320156, + "loss": 0.1722, + "step": 2220 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017767089707409304, + "loss": 0.1737, + "step": 2225 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017756056532202696, + "loss": 0.1688, + "step": 2230 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017744999609481738, + "loss": 0.1784, + "step": 2235 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017733918973100545, + "loss": 0.1747, + "step": 2240 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017722814656985841, + "loss": 0.1667, + "step": 2245 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001771168669513685, + "loss": 0.1648, + "step": 2250 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017700535121625187, + "loss": 0.1777, + "step": 2255 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017689359970594779, + "loss": 0.175, + "step": 2260 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001767816127626173, + "loss": 0.1715, + "step": 2265 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001766693907291423, + "loss": 0.1742, + "step": 2270 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017655693394912452, + "loss": 0.1868, + "step": 2275 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017644424276688441, + "loss": 0.1757, + "step": 2280 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017633131752746017, + "loss": 0.1803, + "step": 2285 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017621815857660656, + "loss": 0.1812, + "step": 2290 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017610476626079395, + "loss": 0.1691, + "step": 2295 + }, + { + "epoch": 0.75, + "learning_rate": 0.00017599114092720726, + "loss": 0.1662, + "step": 2300 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001758772829237448, + "loss": 0.1709, + "step": 2305 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001757631925990174, + "loss": 0.1822, + "step": 2310 + }, + { + "epoch": 0.75, + "learning_rate": 0.00017564887030234696, + "loss": 0.1732, + "step": 2315 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001755343163837659, + "loss": 0.1854, + "step": 2320 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001754195311940157, + "loss": 0.1768, + "step": 2325 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017530451508454594, + "loss": 0.1785, + "step": 2330 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017518926840751325, + "loss": 0.1724, + "step": 2335 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017507379151578017, + "loss": 0.1812, + "step": 2340 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017495808476291423, + "loss": 0.1754, + "step": 2345 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017484214850318664, + "loss": 0.1799, + "step": 2350 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001747259830915713, + "loss": 0.1769, + "step": 2355 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017460958888374385, + "loss": 0.1764, + "step": 2360 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001744929662360803, + "loss": 0.1716, + "step": 2365 + }, + { + "epoch": 0.77, + "learning_rate": 0.00017437611550565624, + "loss": 0.1699, + "step": 2370 + }, + { + "epoch": 0.77, + "learning_rate": 0.00017425903705024553, + "loss": 0.1661, + "step": 2375 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001741417312283193, + "loss": 0.1699, + "step": 2380 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001740241983990448, + "loss": 0.1682, + "step": 2385 + }, + { + "epoch": 0.77, + "learning_rate": 0.00017390643892228438, + "loss": 0.173, + "step": 2390 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001737884531585943, + "loss": 0.1852, + "step": 2395 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017367024146922372, + "loss": 0.1681, + "step": 2400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017355180421611347, + "loss": 0.1798, + "step": 2405 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017343314176189502, + "loss": 0.1705, + "step": 2410 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017331425446988946, + "loss": 0.177, + "step": 2415 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017319514270410614, + "loss": 0.1754, + "step": 2420 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017307580682924178, + "loss": 0.1679, + "step": 2425 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017295624721067932, + "loss": 0.1763, + "step": 2430 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017283646421448673, + "loss": 0.1769, + "step": 2435 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017271645820741583, + "loss": 0.1753, + "step": 2440 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001725962295569014, + "loss": 0.1715, + "step": 2445 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017247577863105983, + "loss": 0.1784, + "step": 2450 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017235510579868802, + "loss": 0.17, + "step": 2455 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017223421142926245, + "loss": 0.1686, + "step": 2460 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017211309589293774, + "loss": 0.166, + "step": 2465 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017199175956054584, + "loss": 0.1652, + "step": 2470 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017187020280359455, + "loss": 0.1732, + "step": 2475 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001717484259942668, + "loss": 0.1754, + "step": 2480 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017162642950541902, + "loss": 0.1748, + "step": 2485 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017150421371058049, + "loss": 0.1733, + "step": 2490 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017138177898395177, + "loss": 0.1737, + "step": 2495 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017125912570040383, + "loss": 0.175, + "step": 2500 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001711362542354769, + "loss": 0.173, + "step": 2505 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017101316496537905, + "loss": 0.1796, + "step": 2510 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017088985826698542, + "loss": 0.1722, + "step": 2515 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017076633451783675, + "loss": 0.1738, + "step": 2520 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017064259409613834, + "loss": 0.1743, + "step": 2525 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017051863738075902, + "loss": 0.1653, + "step": 2530 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001703944647512297, + "loss": 0.1761, + "step": 2535 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017027007658774256, + "loss": 0.1649, + "step": 2540 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017014547327114956, + "loss": 0.1738, + "step": 2545 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001700206551829614, + "loss": 0.1585, + "step": 2550 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016989562270534655, + "loss": 0.183, + "step": 2555 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001697703762211297, + "loss": 0.1693, + "step": 2560 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001696449161137909, + "loss": 0.173, + "step": 2565 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016951924276746425, + "loss": 0.1691, + "step": 2570 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016939335656693667, + "loss": 0.1698, + "step": 2575 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016926725789764691, + "loss": 0.1706, + "step": 2580 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016914094714568417, + "loss": 0.1691, + "step": 2585 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016901442469778713, + "loss": 0.1665, + "step": 2590 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016888769094134247, + "loss": 0.1831, + "step": 2595 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016876074626438395, + "loss": 0.1637, + "step": 2600 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016863359105559117, + "loss": 0.1757, + "step": 2605 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016850622570428826, + "loss": 0.1721, + "step": 2610 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016837865060044272, + "loss": 0.1647, + "step": 2615 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016825086613466442, + "loss": 0.1734, + "step": 2620 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016812287269820416, + "loss": 0.1847, + "step": 2625 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016799467068295257, + "loss": 0.1682, + "step": 2630 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001678662604814389, + "loss": 0.1741, + "step": 2635 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016773764248682988, + "loss": 0.1692, + "step": 2640 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016760881709292842, + "loss": 0.1805, + "step": 2645 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001674797846941724, + "loss": 0.1685, + "step": 2650 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016735054568563364, + "loss": 0.1673, + "step": 2655 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001672211004630164, + "loss": 0.1686, + "step": 2660 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016709144942265644, + "loss": 0.1749, + "step": 2665 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001669615929615197, + "loss": 0.1644, + "step": 2670 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016683153147720097, + "loss": 0.1673, + "step": 2675 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016670126536792287, + "loss": 0.1722, + "step": 2680 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001665707950325345, + "loss": 0.1588, + "step": 2685 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001664401208705103, + "loss": 0.164, + "step": 2690 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016630924328194877, + "loss": 0.1708, + "step": 2695 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016617816266757123, + "loss": 0.1711, + "step": 2700 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016604687942872067, + "loss": 0.1701, + "step": 2705 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016591539396736046, + "loss": 0.1708, + "step": 2710 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016578370668607307, + "loss": 0.1669, + "step": 2715 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016565181798805907, + "loss": 0.1601, + "step": 2720 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016551972827713555, + "loss": 0.1733, + "step": 2725 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016538743795773515, + "loss": 0.1641, + "step": 2730 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016525494743490474, + "loss": 0.1723, + "step": 2735 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016512225711430415, + "loss": 0.1655, + "step": 2740 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016498936740220492, + "loss": 0.172, + "step": 2745 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016485627870548917, + "loss": 0.1686, + "step": 2750 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016472299143164816, + "loss": 0.1709, + "step": 2755 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016458950598878126, + "loss": 0.1648, + "step": 2760 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016445582278559458, + "loss": 0.1641, + "step": 2765 + }, + { + "epoch": 0.9, + "learning_rate": 0.0001643219422313996, + "loss": 0.1662, + "step": 2770 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016418786473611227, + "loss": 0.1763, + "step": 2775 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016405359071025138, + "loss": 0.1773, + "step": 2780 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016391912056493743, + "loss": 0.1678, + "step": 2785 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016378445471189156, + "loss": 0.1632, + "step": 2790 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016364959356343398, + "loss": 0.1671, + "step": 2795 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016351453753248296, + "loss": 0.1739, + "step": 2800 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016337928703255336, + "loss": 0.1727, + "step": 2805 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016324384247775554, + "loss": 0.1756, + "step": 2810 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016310820428279397, + "loss": 0.1671, + "step": 2815 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016297237286296606, + "loss": 0.169, + "step": 2820 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016283634863416082, + "loss": 0.1693, + "step": 2825 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016270013201285757, + "loss": 0.164, + "step": 2830 + }, + { + "epoch": 0.92, + "learning_rate": 0.0001625637234161247, + "loss": 0.1663, + "step": 2835 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016242712326161845, + "loss": 0.1742, + "step": 2840 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016229033196758154, + "loss": 0.169, + "step": 2845 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016215334995284188, + "loss": 0.1703, + "step": 2850 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001620161776368114, + "loss": 0.1669, + "step": 2855 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001618788154394846, + "loss": 0.1634, + "step": 2860 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001617412637814375, + "loss": 0.163, + "step": 2865 + }, + { + "epoch": 0.93, + "learning_rate": 0.00016160352308382608, + "loss": 0.1651, + "step": 2870 + }, + { + "epoch": 0.93, + "learning_rate": 0.00016146559376838514, + "loss": 0.1657, + "step": 2875 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001613274762574271, + "loss": 0.1638, + "step": 2880 + }, + { + "epoch": 0.93, + "learning_rate": 0.00016118917097384048, + "loss": 0.1691, + "step": 2885 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016105067834108874, + "loss": 0.1627, + "step": 2890 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016091199878320902, + "loss": 0.1608, + "step": 2895 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016077313272481076, + "loss": 0.1617, + "step": 2900 + }, + { + "epoch": 0.94, + "learning_rate": 0.0001606340805910744, + "loss": 0.1696, + "step": 2905 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016049484280775012, + "loss": 0.1604, + "step": 2910 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016035541980115653, + "loss": 0.161, + "step": 2915 + }, + { + "epoch": 0.95, + "learning_rate": 0.00016021581199817943, + "loss": 0.1703, + "step": 2920 + }, + { + "epoch": 0.95, + "learning_rate": 0.0001600760198262703, + "loss": 0.1671, + "step": 2925 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015993604371344517, + "loss": 0.1603, + "step": 2930 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015979588408828333, + "loss": 0.1655, + "step": 2935 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015965554137992584, + "loss": 0.1675, + "step": 2940 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015951501601807437, + "loss": 0.1641, + "step": 2945 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015937430843298986, + "loss": 0.165, + "step": 2950 + }, + { + "epoch": 0.96, + "learning_rate": 0.0001592334190554912, + "loss": 0.1758, + "step": 2955 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015909234831695379, + "loss": 0.1703, + "step": 2960 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015895109664930846, + "loss": 0.1644, + "step": 2965 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015880966448503987, + "loss": 0.1638, + "step": 2970 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015866805225718542, + "loss": 0.1659, + "step": 2975 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015852626039933383, + "loss": 0.1676, + "step": 2980 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015838428934562373, + "loss": 0.1794, + "step": 2985 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015824213953074254, + "loss": 0.1562, + "step": 2990 + }, + { + "epoch": 0.97, + "learning_rate": 0.0001580998113899249, + "loss": 0.171, + "step": 2995 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015795730535895147, + "loss": 0.1681, + "step": 3000 + }, + { + "epoch": 0.97, + "learning_rate": 0.0001578146218741476, + "loss": 0.1617, + "step": 3005 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015767176137238196, + "loss": 0.1708, + "step": 3010 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015752872429106523, + "loss": 0.1636, + "step": 3015 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015738551106814872, + "loss": 0.1678, + "step": 3020 + }, + { + "epoch": 0.98, + "learning_rate": 0.000157242122142123, + "loss": 0.1612, + "step": 3025 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015709855795201675, + "loss": 0.1658, + "step": 3030 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015695481893739514, + "loss": 0.1621, + "step": 3035 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015681090553835864, + "loss": 0.1718, + "step": 3040 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015666681819554175, + "loss": 0.1712, + "step": 3045 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015652255735011135, + "loss": 0.1693, + "step": 3050 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015637812344376583, + "loss": 0.1714, + "step": 3055 + }, + { + "epoch": 0.99, + "learning_rate": 0.0001562335169187332, + "loss": 0.1661, + "step": 3060 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015608873821777012, + "loss": 0.1676, + "step": 3065 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015594378778416044, + "loss": 0.1654, + "step": 3070 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015579866606171372, + "loss": 0.1644, + "step": 3075 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015565337349476404, + "loss": 0.1744, + "step": 3080 + }, + { + "epoch": 1.0, + "learning_rate": 0.0001555079105281686, + "loss": 0.1606, + "step": 3085 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015536227760730632, + "loss": 0.1659, + "step": 3090 + }, + { + "epoch": 1.0, + "learning_rate": 0.0001552164751780764, + "loss": 0.1602, + "step": 3095 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015507050368689714, + "loss": 0.1636, + "step": 3100 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015492436358070438, + "loss": 0.1555, + "step": 3105 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015477805530695035, + "loss": 0.1661, + "step": 3110 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015463157931360207, + "loss": 0.1628, + "step": 3115 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015448493604914014, + "loss": 0.1696, + "step": 3120 + }, + { + "epoch": 1.01, + "learning_rate": 0.0001543381259625573, + "loss": 0.1685, + "step": 3125 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015419114950335704, + "loss": 0.1635, + "step": 3130 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015404400712155228, + "loss": 0.1631, + "step": 3135 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015389669926766395, + "loss": 0.1664, + "step": 3140 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015374922639271963, + "loss": 0.1617, + "step": 3145 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015360158894825216, + "loss": 0.174, + "step": 3150 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015345378738629826, + "loss": 0.1591, + "step": 3155 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015330582215939713, + "loss": 0.1729, + "step": 3160 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015315769372058907, + "loss": 0.1732, + "step": 3165 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015300940252341416, + "loss": 0.1608, + "step": 3170 + }, + { + "epoch": 1.03, + "learning_rate": 0.0001528609490219108, + "loss": 0.168, + "step": 3175 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015271233367061425, + "loss": 0.1731, + "step": 3180 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015256355692455542, + "loss": 0.1656, + "step": 3185 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015241461923925934, + "loss": 0.1591, + "step": 3190 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015226552107074385, + "loss": 0.1635, + "step": 3195 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015211626287551803, + "loss": 0.1622, + "step": 3200 + }, + { + "epoch": 1.04, + "learning_rate": 0.0001519668451105811, + "loss": 0.162, + "step": 3205 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015181726823342072, + "loss": 0.1614, + "step": 3210 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015166753270201176, + "loss": 0.1657, + "step": 3215 + }, + { + "epoch": 1.04, + "learning_rate": 0.0001515176389748149, + "loss": 0.1623, + "step": 3220 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015136758751077504, + "loss": 0.1629, + "step": 3225 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015121737876932034, + "loss": 0.1556, + "step": 3230 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015106701321036017, + "loss": 0.1651, + "step": 3235 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015091649129428424, + "loss": 0.1575, + "step": 3240 + }, + { + "epoch": 1.05, + "learning_rate": 0.000150765813481961, + "loss": 0.1594, + "step": 3245 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015061498023473608, + "loss": 0.1624, + "step": 3250 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015046399201443114, + "loss": 0.1616, + "step": 3255 + }, + { + "epoch": 1.06, + "learning_rate": 0.00015031284928334238, + "loss": 0.1693, + "step": 3260 + }, + { + "epoch": 1.06, + "learning_rate": 0.00015016155250423892, + "loss": 0.1592, + "step": 3265 + }, + { + "epoch": 1.06, + "learning_rate": 0.00015001010214036174, + "loss": 0.1542, + "step": 3270 + }, + { + "epoch": 1.06, + "learning_rate": 0.0001498584986554219, + "loss": 0.1631, + "step": 3275 + }, + { + "epoch": 1.06, + "learning_rate": 0.0001497067425135993, + "loss": 0.1612, + "step": 3280 + }, + { + "epoch": 1.06, + "learning_rate": 0.00014955483417954137, + "loss": 0.1551, + "step": 3285 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014940277411836146, + "loss": 0.155, + "step": 3290 + }, + { + "epoch": 1.07, + "learning_rate": 0.0001492505627956374, + "loss": 0.1593, + "step": 3295 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014909820067741028, + "loss": 0.1622, + "step": 3300 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014894568823018277, + "loss": 0.1637, + "step": 3305 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014879302592091798, + "loss": 0.1643, + "step": 3310 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014864021421703768, + "loss": 0.1642, + "step": 3315 + }, + { + "epoch": 1.08, + "learning_rate": 0.00014848725358642114, + "loss": 0.1607, + "step": 3320 + }, + { + "epoch": 1.08, + "learning_rate": 0.00014833414449740373, + "loss": 0.1591, + "step": 3325 + }, + { + "epoch": 1.08, + "learning_rate": 0.0001481808874187752, + "loss": 0.1682, + "step": 3330 + }, + { + "epoch": 1.08, + "learning_rate": 0.00014802748281977842, + "loss": 0.1572, + "step": 3335 + }, + { + "epoch": 1.08, + "learning_rate": 0.0001478739311701081, + "loss": 0.1595, + "step": 3340 + }, + { + "epoch": 1.08, + "learning_rate": 0.000147720232939909, + "loss": 0.1683, + "step": 3345 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014756638859977483, + "loss": 0.1539, + "step": 3350 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014741239862074655, + "loss": 0.1594, + "step": 3355 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014725826347431113, + "loss": 0.1658, + "step": 3360 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014710398363239994, + "loss": 0.1621, + "step": 3365 + }, + { + "epoch": 1.09, + "learning_rate": 0.0001469495595673874, + "loss": 0.1651, + "step": 3370 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014679499175208953, + "loss": 0.1655, + "step": 3375 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014664028065976246, + "loss": 0.1669, + "step": 3380 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014648542676410105, + "loss": 0.1593, + "step": 3385 + }, + { + "epoch": 1.1, + "learning_rate": 0.0001463304305392373, + "loss": 0.157, + "step": 3390 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014617529245973917, + "loss": 0.1593, + "step": 3395 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014602001300060877, + "loss": 0.1516, + "step": 3400 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014586459263728115, + "loss": 0.156, + "step": 3405 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014570903184562282, + "loss": 0.1637, + "step": 3410 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014555333110193022, + "loss": 0.1552, + "step": 3415 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014539749088292826, + "loss": 0.1624, + "step": 3420 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014524151166576897, + "loss": 0.1568, + "step": 3425 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014508539392802994, + "loss": 0.1583, + "step": 3430 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014492913814771283, + "loss": 0.1625, + "step": 3435 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014477274480324202, + "loss": 0.1563, + "step": 3440 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014461621437346307, + "loss": 0.1634, + "step": 3445 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014445954733764127, + "loss": 0.1452, + "step": 3450 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014430274417546012, + "loss": 0.1539, + "step": 3455 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014414580536702, + "loss": 0.1636, + "step": 3460 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014398873139283657, + "loss": 0.1509, + "step": 3465 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014383152273383934, + "loss": 0.1636, + "step": 3470 + }, + { + "epoch": 1.13, + "learning_rate": 0.00014367417987137015, + "loss": 0.1619, + "step": 3475 + }, + { + "epoch": 1.13, + "learning_rate": 0.00014351670328718183, + "loss": 0.1648, + "step": 3480 + }, + { + "epoch": 1.13, + "learning_rate": 0.00014335909346343658, + "loss": 0.1508, + "step": 3485 + }, + { + "epoch": 1.13, + "learning_rate": 0.0001432013508827046, + "loss": 0.1546, + "step": 3490 + }, + { + "epoch": 1.13, + "learning_rate": 0.0001430434760279625, + "loss": 0.1549, + "step": 3495 + }, + { + "epoch": 1.13, + "learning_rate": 0.000142885469382592, + "loss": 0.1412, + "step": 3500 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001427273314303782, + "loss": 0.1574, + "step": 3505 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001425690626555083, + "loss": 0.16, + "step": 3510 + }, + { + "epoch": 1.14, + "learning_rate": 0.00014241066354257008, + "loss": 0.1633, + "step": 3515 + }, + { + "epoch": 1.14, + "learning_rate": 0.00014225213457655034, + "loss": 0.1499, + "step": 3520 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001420934762428335, + "loss": 0.159, + "step": 3525 + }, + { + "epoch": 1.14, + "learning_rate": 0.00014193468902720007, + "loss": 0.1561, + "step": 3530 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014177577341582516, + "loss": 0.1619, + "step": 3535 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014161672989527698, + "loss": 0.16, + "step": 3540 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014145755895251546, + "loss": 0.148, + "step": 3545 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014129826107489054, + "loss": 0.1502, + "step": 3550 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014113883675014098, + "loss": 0.1427, + "step": 3555 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001409792864663925, + "loss": 0.1613, + "step": 3560 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001408196107121566, + "loss": 0.1597, + "step": 3565 + }, + { + "epoch": 1.16, + "learning_rate": 0.00014065980997632896, + "loss": 0.1501, + "step": 3570 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001404998847481879, + "loss": 0.1537, + "step": 3575 + }, + { + "epoch": 1.16, + "learning_rate": 0.00014033983551739286, + "loss": 0.1608, + "step": 3580 + }, + { + "epoch": 1.16, + "learning_rate": 0.00014017966277398304, + "loss": 0.1488, + "step": 3585 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001400193670083757, + "loss": 0.1593, + "step": 3590 + }, + { + "epoch": 1.16, + "learning_rate": 0.00013985894871136492, + "loss": 0.1525, + "step": 3595 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013969840837411982, + "loss": 0.1529, + "step": 3600 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013953774648818324, + "loss": 0.1489, + "step": 3605 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013937696354547022, + "loss": 0.144, + "step": 3610 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013921606003826627, + "loss": 0.1584, + "step": 3615 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013905503645922628, + "loss": 0.1657, + "step": 3620 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013889389330137264, + "loss": 0.1557, + "step": 3625 + }, + { + "epoch": 1.18, + "learning_rate": 0.00013873263105809388, + "loss": 0.1551, + "step": 3630 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001385712502231432, + "loss": 0.157, + "step": 3635 + }, + { + "epoch": 1.18, + "learning_rate": 0.00013840975129063684, + "loss": 0.1548, + "step": 3640 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001382481347550526, + "loss": 0.1565, + "step": 3645 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001380864011112285, + "loss": 0.1497, + "step": 3650 + }, + { + "epoch": 1.18, + "learning_rate": 0.000137924550854361, + "loss": 0.1569, + "step": 3655 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001377625844800037, + "loss": 0.1582, + "step": 3660 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013760050248406557, + "loss": 0.1571, + "step": 3665 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001374383053628098, + "loss": 0.1524, + "step": 3670 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013727599361285187, + "loss": 0.1567, + "step": 3675 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013711356773115838, + "loss": 0.1548, + "step": 3680 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013695102821504533, + "loss": 0.1557, + "step": 3685 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001367883755621766, + "loss": 0.148, + "step": 3690 + }, + { + "epoch": 1.2, + "learning_rate": 0.00013662561027056252, + "loss": 0.1604, + "step": 3695 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001364627328385583, + "loss": 0.15, + "step": 3700 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001362997437648625, + "loss": 0.1476, + "step": 3705 + }, + { + "epoch": 1.2, + "learning_rate": 0.00013613664354851547, + "loss": 0.1586, + "step": 3710 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001359734326888979, + "loss": 0.1536, + "step": 3715 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013581011168572923, + "loss": 0.1564, + "step": 3720 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013564668103906616, + "loss": 0.1534, + "step": 3725 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013548314124930104, + "loss": 0.1521, + "step": 3730 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013531949281716044, + "loss": 0.1489, + "step": 3735 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013515573624370357, + "loss": 0.1528, + "step": 3740 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001349918720303207, + "loss": 0.1405, + "step": 3745 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013482790067873178, + "loss": 0.1457, + "step": 3750 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013466382269098468, + "loss": 0.1554, + "step": 3755 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013449963856945383, + "loss": 0.1478, + "step": 3760 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013433534881683858, + "loss": 0.1633, + "step": 3765 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013417095393616178, + "loss": 0.15, + "step": 3770 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013400645443076804, + "loss": 0.1552, + "step": 3775 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013384185080432244, + "loss": 0.1485, + "step": 3780 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001336771435608088, + "loss": 0.1531, + "step": 3785 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013351233320452812, + "loss": 0.158, + "step": 3790 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013334742024009725, + "loss": 0.1554, + "step": 3795 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013318240517244713, + "loss": 0.1537, + "step": 3800 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001330172885068213, + "loss": 0.1584, + "step": 3805 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013285207074877447, + "loss": 0.1612, + "step": 3810 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013268675240417073, + "loss": 0.1449, + "step": 3815 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013252133397918226, + "loss": 0.1456, + "step": 3820 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001323558159802876, + "loss": 0.1541, + "step": 3825 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013219019891427025, + "loss": 0.1492, + "step": 3830 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013202448328821694, + "loss": 0.1528, + "step": 3835 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013185866960951622, + "loss": 0.1516, + "step": 3840 + }, + { + "epoch": 1.25, + "learning_rate": 0.0001316927583858569, + "loss": 0.1484, + "step": 3845 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013152675012522629, + "loss": 0.1509, + "step": 3850 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013136064533590897, + "loss": 0.1516, + "step": 3855 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013119444452648504, + "loss": 0.15, + "step": 3860 + }, + { + "epoch": 1.25, + "learning_rate": 0.0001310281482058286, + "loss": 0.1571, + "step": 3865 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013086175688310612, + "loss": 0.1526, + "step": 3870 + }, + { + "epoch": 1.26, + "learning_rate": 0.000130695271067775, + "loss": 0.1445, + "step": 3875 + }, + { + "epoch": 1.26, + "learning_rate": 0.00013052869126958198, + "loss": 0.1598, + "step": 3880 + }, + { + "epoch": 1.26, + "learning_rate": 0.00013036201799856149, + "loss": 0.1499, + "step": 3885 + }, + { + "epoch": 1.26, + "learning_rate": 0.0001301952517650342, + "loss": 0.1507, + "step": 3890 + }, + { + "epoch": 1.26, + "learning_rate": 0.0001300283930796054, + "loss": 0.1489, + "step": 3895 + }, + { + "epoch": 1.26, + "learning_rate": 0.00012986144245316346, + "loss": 0.1477, + "step": 3900 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012969440039687822, + "loss": 0.1528, + "step": 3905 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012952726742219953, + "loss": 0.1514, + "step": 3910 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012936004404085557, + "loss": 0.1482, + "step": 3915 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012919273076485133, + "loss": 0.1558, + "step": 3920 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012902532810646697, + "loss": 0.1539, + "step": 3925 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012885783657825647, + "loss": 0.1479, + "step": 3930 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012869025669304585, + "loss": 0.1449, + "step": 3935 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012852258896393158, + "loss": 0.1489, + "step": 3940 + }, + { + "epoch": 1.28, + "learning_rate": 0.0001283548339042792, + "loss": 0.1479, + "step": 3945 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012818699202772152, + "loss": 0.149, + "step": 3950 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012801906384815733, + "loss": 0.1472, + "step": 3955 + }, + { + "epoch": 1.28, + "learning_rate": 0.0001278510498797495, + "loss": 0.1526, + "step": 3960 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012768295063692376, + "loss": 0.1469, + "step": 3965 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012751476663436668, + "loss": 0.1576, + "step": 3970 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012734649838702455, + "loss": 0.1583, + "step": 3975 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012717814641010153, + "loss": 0.1434, + "step": 3980 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012700971121905812, + "loss": 0.1463, + "step": 3985 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012684119332960963, + "loss": 0.1543, + "step": 3990 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012667259325772464, + "loss": 0.1501, + "step": 3995 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012650391151962324, + "loss": 0.1475, + "step": 4000 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012633514863177566, + "loss": 0.147, + "step": 4005 + }, + { + "epoch": 1.3, + "learning_rate": 0.0001261663051109005, + "loss": 0.1454, + "step": 4010 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012599738147396332, + "loss": 0.1459, + "step": 4015 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012582837823817497, + "loss": 0.1479, + "step": 4020 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012565929592099002, + "loss": 0.1541, + "step": 4025 + }, + { + "epoch": 1.31, + "learning_rate": 0.00012549013504010512, + "loss": 0.145, + "step": 4030 + }, + { + "epoch": 1.31, + "learning_rate": 0.0001253208961134575, + "loss": 0.1467, + "step": 4035 + }, + { + "epoch": 1.31, + "learning_rate": 0.0001251515796592234, + "loss": 0.1497, + "step": 4040 + }, + { + "epoch": 1.31, + "learning_rate": 0.00012498218619581632, + "loss": 0.1439, + "step": 4045 + }, + { + "epoch": 1.31, + "learning_rate": 0.00012481271624188568, + "loss": 0.1419, + "step": 4050 + }, + { + "epoch": 1.31, + "learning_rate": 0.00012464317031631494, + "loss": 0.1518, + "step": 4055 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012447354893822034, + "loss": 0.1462, + "step": 4060 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012430385262694906, + "loss": 0.1473, + "step": 4065 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012413408190207768, + "loss": 0.1495, + "step": 4070 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012396423728341067, + "loss": 0.1566, + "step": 4075 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012379431929097873, + "loss": 0.1507, + "step": 4080 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012362432844503725, + "loss": 0.1544, + "step": 4085 + }, + { + "epoch": 1.33, + "learning_rate": 0.0001234542652660646, + "loss": 0.1528, + "step": 4090 + }, + { + "epoch": 1.33, + "learning_rate": 0.00012328413027476066, + "loss": 0.153, + "step": 4095 + }, + { + "epoch": 1.33, + "learning_rate": 0.00012311392399204526, + "loss": 0.1438, + "step": 4100 + }, + { + "epoch": 1.33, + "learning_rate": 0.0001229436469390564, + "loss": 0.1497, + "step": 4105 + }, + { + "epoch": 1.33, + "learning_rate": 0.00012277329963714882, + "loss": 0.1471, + "step": 4110 + }, + { + "epoch": 1.33, + "learning_rate": 0.0001226028826078923, + "loss": 0.1442, + "step": 4115 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012243239637307023, + "loss": 0.1455, + "step": 4120 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012226184145467775, + "loss": 0.1509, + "step": 4125 + }, + { + "epoch": 1.34, + "learning_rate": 0.0001220912183749204, + "loss": 0.1504, + "step": 4130 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012192052765621234, + "loss": 0.1524, + "step": 4135 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012174976982117488, + "loss": 0.1526, + "step": 4140 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012157894539263482, + "loss": 0.1501, + "step": 4145 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012140805489362281, + "loss": 0.1464, + "step": 4150 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012123709884737189, + "loss": 0.1481, + "step": 4155 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012106607777731567, + "loss": 0.1483, + "step": 4160 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012089499220708696, + "loss": 0.1456, + "step": 4165 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012072384266051595, + "loss": 0.1479, + "step": 4170 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012055262966162883, + "loss": 0.1488, + "step": 4175 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012038135373464602, + "loss": 0.1468, + "step": 4180 + }, + { + "epoch": 1.36, + "learning_rate": 0.00012021001540398058, + "loss": 0.1412, + "step": 4185 + }, + { + "epoch": 1.36, + "learning_rate": 0.00012003861519423662, + "loss": 0.1503, + "step": 4190 + }, + { + "epoch": 1.36, + "learning_rate": 0.00011986715363020784, + "loss": 0.1497, + "step": 4195 + }, + { + "epoch": 1.36, + "learning_rate": 0.00011969563123687566, + "loss": 0.1457, + "step": 4200 + }, + { + "epoch": 1.36, + "learning_rate": 0.00011952404853940782, + "loss": 0.1469, + "step": 4205 + }, + { + "epoch": 1.36, + "learning_rate": 0.00011935240606315666, + "loss": 0.1485, + "step": 4210 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011918070433365754, + "loss": 0.1413, + "step": 4215 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011900894387662731, + "loss": 0.1565, + "step": 4220 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011883712521796253, + "loss": 0.1516, + "step": 4225 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011866524888373811, + "loss": 0.146, + "step": 4230 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011849331540020543, + "loss": 0.1507, + "step": 4235 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011832132529379088, + "loss": 0.1493, + "step": 4240 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011814927909109419, + "loss": 0.1419, + "step": 4245 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011797717731888691, + "loss": 0.1414, + "step": 4250 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011780502050411069, + "loss": 0.1385, + "step": 4255 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011763280917387572, + "loss": 0.1473, + "step": 4260 + }, + { + "epoch": 1.38, + "learning_rate": 0.0001174605438554591, + "loss": 0.1496, + "step": 4265 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011728822507630326, + "loss": 0.1369, + "step": 4270 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011711585336401429, + "loss": 0.1437, + "step": 4275 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011694342924636031, + "loss": 0.1564, + "step": 4280 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011677095325126996, + "loss": 0.1443, + "step": 4285 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011659842590683074, + "loss": 0.1577, + "step": 4290 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011642584774128728, + "loss": 0.1416, + "step": 4295 + }, + { + "epoch": 1.39, + "learning_rate": 0.0001162532192830399, + "loss": 0.145, + "step": 4300 + }, + { + "epoch": 1.4, + "learning_rate": 0.00011608054106064281, + "loss": 0.1458, + "step": 4305 + }, + { + "epoch": 1.4, + "learning_rate": 0.0001159078136028027, + "loss": 0.1445, + "step": 4310 + }, + { + "epoch": 1.4, + "learning_rate": 0.00011573503743837693, + "loss": 0.139, + "step": 4315 + }, + { + "epoch": 1.4, + "learning_rate": 0.00011556221309637203, + "loss": 0.1441, + "step": 4320 + }, + { + "epoch": 1.4, + "learning_rate": 0.000115389341105942, + "loss": 0.1567, + "step": 4325 + }, + { + "epoch": 1.4, + "learning_rate": 0.0001152164219963868, + "loss": 0.1499, + "step": 4330 + }, + { + "epoch": 1.4, + "learning_rate": 0.00011504345629715057, + "loss": 0.15, + "step": 4335 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011487044453782016, + "loss": 0.1449, + "step": 4340 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011469738724812346, + "loss": 0.1423, + "step": 4345 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011452428495792773, + "loss": 0.1468, + "step": 4350 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011435113819723797, + "loss": 0.1519, + "step": 4355 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011417794749619544, + "loss": 0.1498, + "step": 4360 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011400471338507585, + "loss": 0.1406, + "step": 4365 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011383143639428791, + "loss": 0.1485, + "step": 4370 + }, + { + "epoch": 1.42, + "learning_rate": 0.0001136581170543715, + "loss": 0.1502, + "step": 4375 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011348475589599625, + "loss": 0.139, + "step": 4380 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011331135344995985, + "loss": 0.1485, + "step": 4385 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011313791024718634, + "loss": 0.133, + "step": 4390 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011296442681872457, + "loss": 0.1473, + "step": 4395 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011279090369574658, + "loss": 0.1401, + "step": 4400 + }, + { + "epoch": 1.43, + "learning_rate": 0.0001126173414095459, + "loss": 0.1483, + "step": 4405 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011244374049153603, + "loss": 0.1436, + "step": 4410 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011227010147324868, + "loss": 0.1445, + "step": 4415 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011209642488633227, + "loss": 0.1451, + "step": 4420 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011192271126255024, + "loss": 0.1438, + "step": 4425 + }, + { + "epoch": 1.44, + "learning_rate": 0.00011174896113377945, + "loss": 0.1474, + "step": 4430 + }, + { + "epoch": 1.44, + "learning_rate": 0.0001115751750320084, + "loss": 0.1444, + "step": 4435 + }, + { + "epoch": 1.44, + "learning_rate": 0.00011140135348933593, + "loss": 0.147, + "step": 4440 + }, + { + "epoch": 1.44, + "learning_rate": 0.00011122749703796929, + "loss": 0.1447, + "step": 4445 + }, + { + "epoch": 1.44, + "learning_rate": 0.0001110536062102226, + "loss": 0.1423, + "step": 4450 + }, + { + "epoch": 1.44, + "learning_rate": 0.00011087968153851525, + "loss": 0.1405, + "step": 4455 + }, + { + "epoch": 1.45, + "learning_rate": 0.00011070572355537026, + "loss": 0.1468, + "step": 4460 + }, + { + "epoch": 1.45, + "learning_rate": 0.00011053173279341265, + "loss": 0.1442, + "step": 4465 + }, + { + "epoch": 1.45, + "learning_rate": 0.0001103577097853678, + "loss": 0.1447, + "step": 4470 + }, + { + "epoch": 1.45, + "learning_rate": 0.00011018365506405978, + "loss": 0.1514, + "step": 4475 + }, + { + "epoch": 1.45, + "learning_rate": 0.00011000956916240985, + "loss": 0.148, + "step": 4480 + }, + { + "epoch": 1.45, + "learning_rate": 0.00010983545261343464, + "loss": 0.1499, + "step": 4485 + }, + { + "epoch": 1.45, + "learning_rate": 0.00010966130595024466, + "loss": 0.1398, + "step": 4490 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010948712970604266, + "loss": 0.1379, + "step": 4495 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010931292441412191, + "loss": 0.1438, + "step": 4500 + }, + { + "epoch": 1.46, + "learning_rate": 0.0001091386906078646, + "loss": 0.1453, + "step": 4505 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010896442882074029, + "loss": 0.1439, + "step": 4510 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010879013958630418, + "loss": 0.1436, + "step": 4515 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010861582343819548, + "loss": 0.144, + "step": 4520 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010844148091013581, + "loss": 0.1439, + "step": 4525 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010826711253592768, + "loss": 0.1441, + "step": 4530 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010809271884945256, + "loss": 0.1421, + "step": 4535 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010791830038466952, + "loss": 0.1476, + "step": 4540 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010774385767561347, + "loss": 0.1484, + "step": 4545 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010756939125639358, + "loss": 0.1437, + "step": 4550 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010739490166119155, + "loss": 0.1403, + "step": 4555 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010722038942426014, + "loss": 0.1437, + "step": 4560 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010704585507992131, + "loss": 0.1429, + "step": 4565 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010687129916256482, + "loss": 0.1401, + "step": 4570 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010669672220664644, + "loss": 0.1448, + "step": 4575 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010652212474668635, + "loss": 0.1449, + "step": 4580 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010634750731726748, + "loss": 0.1431, + "step": 4585 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010617287045303395, + "loss": 0.1467, + "step": 4590 + }, + { + "epoch": 1.49, + "learning_rate": 0.0001059982146886894, + "loss": 0.1403, + "step": 4595 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010582354055899528, + "loss": 0.1467, + "step": 4600 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010564884859876934, + "loss": 0.1399, + "step": 4605 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010547413934288386, + "loss": 0.1408, + "step": 4610 + }, + { + "epoch": 1.5, + "learning_rate": 0.0001052994133262641, + "loss": 0.1411, + "step": 4615 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010512467108388661, + "loss": 0.1398, + "step": 4620 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010494991315077772, + "loss": 0.1444, + "step": 4625 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010477514006201166, + "loss": 0.1438, + "step": 4630 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010460035235270918, + "loss": 0.1499, + "step": 4635 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010442555055803572, + "loss": 0.1451, + "step": 4640 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010425073521319986, + "loss": 0.15, + "step": 4645 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010407590685345174, + "loss": 0.1471, + "step": 4650 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010390106601408125, + "loss": 0.1454, + "step": 4655 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010372621323041656, + "loss": 0.1476, + "step": 4660 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010355134903782235, + "loss": 0.142, + "step": 4665 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010337647397169832, + "loss": 0.1391, + "step": 4670 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010320158856747737, + "loss": 0.1398, + "step": 4675 + }, + { + "epoch": 1.52, + "learning_rate": 0.0001030266933606241, + "loss": 0.15, + "step": 4680 + }, + { + "epoch": 1.52, + "learning_rate": 0.0001028517888866331, + "loss": 0.1387, + "step": 4685 + }, + { + "epoch": 1.52, + "learning_rate": 0.00010267687568102737, + "loss": 0.1484, + "step": 4690 + }, + { + "epoch": 1.52, + "learning_rate": 0.00010250195427935663, + "loss": 0.1483, + "step": 4695 + }, + { + "epoch": 1.52, + "learning_rate": 0.00010232702521719564, + "loss": 0.1452, + "step": 4700 + }, + { + "epoch": 1.52, + "learning_rate": 0.00010215208903014272, + "loss": 0.1481, + "step": 4705 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010197714625381795, + "loss": 0.1406, + "step": 4710 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010180219742386157, + "loss": 0.139, + "step": 4715 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010162724307593234, + "loss": 0.1354, + "step": 4720 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010145228374570599, + "loss": 0.1379, + "step": 4725 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010127731996887339, + "loss": 0.1481, + "step": 4730 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010110235228113918, + "loss": 0.1464, + "step": 4735 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010092738121821985, + "loss": 0.1426, + "step": 4740 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010075240731584223, + "loss": 0.1415, + "step": 4745 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010057743110974194, + "loss": 0.1393, + "step": 4750 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010040245313566153, + "loss": 0.1408, + "step": 4755 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010022747392934905, + "loss": 0.1358, + "step": 4760 + }, + { + "epoch": 1.54, + "learning_rate": 0.0001000524940265563, + "loss": 0.1401, + "step": 4765 + }, + { + "epoch": 1.55, + "learning_rate": 9.98775139630372e-05, + "loss": 0.1379, + "step": 4770 + }, + { + "epoch": 1.55, + "learning_rate": 9.970253427454616e-05, + "loss": 0.1425, + "step": 4775 + }, + { + "epoch": 1.55, + "learning_rate": 9.95275554968365e-05, + "loss": 0.1475, + "step": 4780 + }, + { + "epoch": 1.55, + "learning_rate": 9.935257816565861e-05, + "loss": 0.1364, + "step": 4785 + }, + { + "epoch": 1.55, + "learning_rate": 9.917760281675866e-05, + "loss": 0.1443, + "step": 4790 + }, + { + "epoch": 1.55, + "learning_rate": 9.900262998587653e-05, + "loss": 0.1388, + "step": 4795 + }, + { + "epoch": 1.56, + "learning_rate": 9.882766020874456e-05, + "loss": 0.1375, + "step": 4800 + }, + { + "epoch": 1.56, + "learning_rate": 9.865269402108565e-05, + "loss": 0.1413, + "step": 4805 + }, + { + "epoch": 1.56, + "learning_rate": 9.847773195861175e-05, + "loss": 0.1499, + "step": 4810 + }, + { + "epoch": 1.56, + "learning_rate": 9.830277455702213e-05, + "loss": 0.1421, + "step": 4815 + }, + { + "epoch": 1.56, + "learning_rate": 9.812782235200189e-05, + "loss": 0.146, + "step": 4820 + }, + { + "epoch": 1.56, + "learning_rate": 9.795287587922006e-05, + "loss": 0.1343, + "step": 4825 + }, + { + "epoch": 1.57, + "learning_rate": 9.777793567432828e-05, + "loss": 0.1451, + "step": 4830 + }, + { + "epoch": 1.57, + "learning_rate": 9.760300227295894e-05, + "loss": 0.1438, + "step": 4835 + }, + { + "epoch": 1.57, + "learning_rate": 9.742807621072353e-05, + "loss": 0.1436, + "step": 4840 + }, + { + "epoch": 1.57, + "learning_rate": 9.725315802321121e-05, + "loss": 0.1384, + "step": 4845 + }, + { + "epoch": 1.57, + "learning_rate": 9.707824824598684e-05, + "loss": 0.1368, + "step": 4850 + }, + { + "epoch": 1.57, + "learning_rate": 9.690334741458974e-05, + "loss": 0.1309, + "step": 4855 + }, + { + "epoch": 1.57, + "learning_rate": 9.672845606453168e-05, + "loss": 0.1384, + "step": 4860 + }, + { + "epoch": 1.58, + "learning_rate": 9.655357473129548e-05, + "loss": 0.1387, + "step": 4865 + }, + { + "epoch": 1.58, + "learning_rate": 9.637870395033324e-05, + "loss": 0.1426, + "step": 4870 + }, + { + "epoch": 1.58, + "learning_rate": 9.620384425706484e-05, + "loss": 0.1407, + "step": 4875 + }, + { + "epoch": 1.58, + "learning_rate": 9.602899618687606e-05, + "loss": 0.1413, + "step": 4880 + }, + { + "epoch": 1.58, + "learning_rate": 9.585416027511727e-05, + "loss": 0.1434, + "step": 4885 + }, + { + "epoch": 1.58, + "learning_rate": 9.567933705710143e-05, + "loss": 0.146, + "step": 4890 + }, + { + "epoch": 1.59, + "learning_rate": 9.550452706810281e-05, + "loss": 0.1384, + "step": 4895 + }, + { + "epoch": 1.59, + "learning_rate": 9.532973084335511e-05, + "loss": 0.1447, + "step": 4900 + }, + { + "epoch": 1.59, + "learning_rate": 9.515494891804979e-05, + "loss": 0.1418, + "step": 4905 + }, + { + "epoch": 1.59, + "learning_rate": 9.498018182733466e-05, + "loss": 0.144, + "step": 4910 + }, + { + "epoch": 1.59, + "learning_rate": 9.480543010631207e-05, + "loss": 0.1449, + "step": 4915 + }, + { + "epoch": 1.59, + "learning_rate": 9.463069429003728e-05, + "loss": 0.1391, + "step": 4920 + }, + { + "epoch": 1.6, + "learning_rate": 9.445597491351685e-05, + "loss": 0.1484, + "step": 4925 + }, + { + "epoch": 1.6, + "learning_rate": 9.42812725117071e-05, + "loss": 0.1402, + "step": 4930 + }, + { + "epoch": 1.6, + "learning_rate": 9.410658761951221e-05, + "loss": 0.1478, + "step": 4935 + }, + { + "epoch": 1.6, + "learning_rate": 9.393192077178295e-05, + "loss": 0.1431, + "step": 4940 + }, + { + "epoch": 1.6, + "learning_rate": 9.375727250331462e-05, + "loss": 0.1405, + "step": 4945 + }, + { + "epoch": 1.6, + "learning_rate": 9.358264334884585e-05, + "loss": 0.1359, + "step": 4950 + }, + { + "epoch": 1.61, + "learning_rate": 9.340803384305657e-05, + "loss": 0.1464, + "step": 4955 + }, + { + "epoch": 1.61, + "learning_rate": 9.323344452056668e-05, + "loss": 0.1408, + "step": 4960 + }, + { + "epoch": 1.61, + "learning_rate": 9.305887591593419e-05, + "loss": 0.1435, + "step": 4965 + }, + { + "epoch": 1.61, + "learning_rate": 9.288432856365374e-05, + "loss": 0.1449, + "step": 4970 + }, + { + "epoch": 1.61, + "learning_rate": 9.270980299815486e-05, + "loss": 0.1403, + "step": 4975 + }, + { + "epoch": 1.61, + "learning_rate": 9.253529975380039e-05, + "loss": 0.1434, + "step": 4980 + }, + { + "epoch": 1.62, + "learning_rate": 9.236081936488485e-05, + "loss": 0.1405, + "step": 4985 + }, + { + "epoch": 1.62, + "learning_rate": 9.21863623656327e-05, + "loss": 0.1394, + "step": 4990 + }, + { + "epoch": 1.62, + "learning_rate": 9.20119292901969e-05, + "loss": 0.1352, + "step": 4995 + }, + { + "epoch": 1.62, + "learning_rate": 9.183752067265703e-05, + "loss": 0.1452, + "step": 5000 + } + ], + "logging_steps": 5, + "max_steps": 9255, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 1000, + "total_flos": 1.0184015317051638e+19, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/save_lora_math_r8_2e4/checkpoint-5000/training_args.bin b/save_lora_math_r8_2e4/checkpoint-5000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..175cae96ce99657f1e2b29179eee355d72511307 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-5000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0cedf67f015d3f83c601cebdd463c220d745d8e241fab282366821363eb224d +size 5880 diff --git a/save_lora_math_r8_2e4/checkpoint-5000/zero_to_fp32.py b/save_lora_math_r8_2e4/checkpoint-5000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..c98caae31534368be22b67fc4ae906836c992a8d --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-5000/zero_to_fp32.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in param_shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, args.output_file, tag=args.tag) diff --git a/save_lora_math_r8_2e4/checkpoint-6000/README.md b/save_lora_math_r8_2e4/checkpoint-6000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d380bbbf0bf0fa1f211b53410559607e5007a209 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-6000/README.md @@ -0,0 +1,202 @@ +--- +library_name: peft +base_model: meta-llama/Llama-2-7b-hf +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.9.0 \ No newline at end of file diff --git a/save_lora_math_r8_2e4/checkpoint-6000/adapter_config.json b/save_lora_math_r8_2e4/checkpoint-6000/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9c81a592f0ff08374a4f61f6003029070a8429ae --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-6000/adapter_config.json @@ -0,0 +1,35 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "meta-llama/Llama-2-7b-hf", + "bias": "none", + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "mora_type": 1, + "peft_type": "LORA", + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "down_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "use_dora": false, + "use_mora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/save_lora_math_r8_2e4/checkpoint-6000/adapter_model.safetensors b/save_lora_math_r8_2e4/checkpoint-6000/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d1cb0704e41cf0b7510406db19f4d4cb07f8e1b9 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-6000/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25b3f7f51254be9e71ee1640cb6587ccb98688a38803a07002350bcb0b7eb10a +size 40036488 diff --git a/save_lora_math_r8_2e4/checkpoint-6000/global_step6000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/save_lora_math_r8_2e4/checkpoint-6000/global_step6000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..467753132e08445730c593a0ae897f233477ff62 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-6000/global_step6000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a172a25ac6b268d6b45e680ec5e099b44ef62c6dc739e2f702989db45da0189 +size 119958768 diff --git a/save_lora_math_r8_2e4/checkpoint-6000/global_step6000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/save_lora_math_r8_2e4/checkpoint-6000/global_step6000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c450d24819cd42441006b57e2b378d89137ad91 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-6000/global_step6000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3fae0fdaf57233168956ab542bab8b4016bc681003ea3375e95798acf73b332 +size 119958896 diff --git a/save_lora_math_r8_2e4/checkpoint-6000/global_step6000/mp_rank_00_model_states.pt b/save_lora_math_r8_2e4/checkpoint-6000/global_step6000/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aaf468fb00dec7bd545d6f710d2d26994b7e14e6 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-6000/global_step6000/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a863df5b8dcba31066748b757b4dc61e726aba2ffb68c3544b372bc1c57b1cb1 +size 40303404 diff --git a/save_lora_math_r8_2e4/checkpoint-6000/latest b/save_lora_math_r8_2e4/checkpoint-6000/latest new file mode 100644 index 0000000000000000000000000000000000000000..3ccb5110452b4a1eef79a1c432a18944dc25a985 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-6000/latest @@ -0,0 +1 @@ +global_step6000 \ No newline at end of file diff --git a/save_lora_math_r8_2e4/checkpoint-6000/rng_state_0.pth b/save_lora_math_r8_2e4/checkpoint-6000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..a977155348013274e252e05ac14db8e2d87579d1 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-6000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1e4ef6c5127e4f44883c7bdc4494274b1721f82660cad96c2257e79626a005c +size 14512 diff --git a/save_lora_math_r8_2e4/checkpoint-6000/rng_state_1.pth b/save_lora_math_r8_2e4/checkpoint-6000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..4b80d3df973e9c311727f3310f41d479ed170271 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-6000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53f7838a2d49d86b8fdc01282b740b7d4f83dedd5096ccaf640d465268856266 +size 14512 diff --git a/save_lora_math_r8_2e4/checkpoint-6000/scheduler.pt b/save_lora_math_r8_2e4/checkpoint-6000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..16647692525669b3e39e684ed63e97ee8639c0b1 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-6000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6be789b0c1507ab448acf122e0ecf8322e28fbc0f574bd25ddb02075f1edc229 +size 1064 diff --git a/save_lora_math_r8_2e4/checkpoint-6000/trainer_state.json b/save_lora_math_r8_2e4/checkpoint-6000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6ee50d83d9894873fb3dec677f60bb2aec68c0bd --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-6000/trainer_state.json @@ -0,0 +1,7221 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.9443037974683546, + "eval_steps": 500, + "global_step": 6000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 3.5971223021582732e-06, + "loss": 0.7628, + "step": 5 + }, + { + "epoch": 0.0, + "learning_rate": 7.1942446043165465e-06, + "loss": 0.769, + "step": 10 + }, + { + "epoch": 0.0, + "learning_rate": 1.0791366906474821e-05, + "loss": 0.7619, + "step": 15 + }, + { + "epoch": 0.01, + "learning_rate": 1.4388489208633093e-05, + "loss": 0.7564, + "step": 20 + }, + { + "epoch": 0.01, + "learning_rate": 1.7985611510791367e-05, + "loss": 0.7549, + "step": 25 + }, + { + "epoch": 0.01, + "learning_rate": 2.1582733812949642e-05, + "loss": 0.7001, + "step": 30 + }, + { + "epoch": 0.01, + "learning_rate": 2.5179856115107914e-05, + "loss": 0.6289, + "step": 35 + }, + { + "epoch": 0.01, + "learning_rate": 2.8776978417266186e-05, + "loss": 0.5632, + "step": 40 + }, + { + "epoch": 0.01, + "learning_rate": 3.237410071942446e-05, + "loss": 0.4923, + "step": 45 + }, + { + "epoch": 0.02, + "learning_rate": 3.597122302158273e-05, + "loss": 0.4582, + "step": 50 + }, + { + "epoch": 0.02, + "learning_rate": 3.956834532374101e-05, + "loss": 0.4357, + "step": 55 + }, + { + "epoch": 0.02, + "learning_rate": 4.3165467625899284e-05, + "loss": 0.4079, + "step": 60 + }, + { + "epoch": 0.02, + "learning_rate": 4.676258992805755e-05, + "loss": 0.4047, + "step": 65 + }, + { + "epoch": 0.02, + "learning_rate": 5.035971223021583e-05, + "loss": 0.3916, + "step": 70 + }, + { + "epoch": 0.02, + "learning_rate": 5.39568345323741e-05, + "loss": 0.3926, + "step": 75 + }, + { + "epoch": 0.03, + "learning_rate": 5.755395683453237e-05, + "loss": 0.3725, + "step": 80 + }, + { + "epoch": 0.03, + "learning_rate": 6.115107913669065e-05, + "loss": 0.3769, + "step": 85 + }, + { + "epoch": 0.03, + "learning_rate": 6.474820143884892e-05, + "loss": 0.352, + "step": 90 + }, + { + "epoch": 0.03, + "learning_rate": 6.83453237410072e-05, + "loss": 0.3534, + "step": 95 + }, + { + "epoch": 0.03, + "learning_rate": 7.194244604316547e-05, + "loss": 0.3432, + "step": 100 + }, + { + "epoch": 0.03, + "learning_rate": 7.553956834532374e-05, + "loss": 0.3286, + "step": 105 + }, + { + "epoch": 0.04, + "learning_rate": 7.913669064748202e-05, + "loss": 0.3298, + "step": 110 + }, + { + "epoch": 0.04, + "learning_rate": 8.273381294964029e-05, + "loss": 0.3223, + "step": 115 + }, + { + "epoch": 0.04, + "learning_rate": 8.633093525179857e-05, + "loss": 0.3181, + "step": 120 + }, + { + "epoch": 0.04, + "learning_rate": 8.992805755395684e-05, + "loss": 0.3225, + "step": 125 + }, + { + "epoch": 0.04, + "learning_rate": 9.35251798561151e-05, + "loss": 0.3145, + "step": 130 + }, + { + "epoch": 0.04, + "learning_rate": 9.712230215827339e-05, + "loss": 0.3137, + "step": 135 + }, + { + "epoch": 0.05, + "learning_rate": 0.00010071942446043166, + "loss": 0.3102, + "step": 140 + }, + { + "epoch": 0.05, + "learning_rate": 0.00010431654676258992, + "loss": 0.2951, + "step": 145 + }, + { + "epoch": 0.05, + "learning_rate": 0.0001079136690647482, + "loss": 0.3021, + "step": 150 + }, + { + "epoch": 0.05, + "learning_rate": 0.00011151079136690648, + "loss": 0.2923, + "step": 155 + }, + { + "epoch": 0.05, + "learning_rate": 0.00011510791366906474, + "loss": 0.2953, + "step": 160 + }, + { + "epoch": 0.05, + "learning_rate": 0.00011870503597122303, + "loss": 0.2893, + "step": 165 + }, + { + "epoch": 0.06, + "learning_rate": 0.0001223021582733813, + "loss": 0.2953, + "step": 170 + }, + { + "epoch": 0.06, + "learning_rate": 0.00012589928057553956, + "loss": 0.2931, + "step": 175 + }, + { + "epoch": 0.06, + "learning_rate": 0.00012949640287769783, + "loss": 0.2879, + "step": 180 + }, + { + "epoch": 0.06, + "learning_rate": 0.00013309352517985613, + "loss": 0.2642, + "step": 185 + }, + { + "epoch": 0.06, + "learning_rate": 0.0001366906474820144, + "loss": 0.2772, + "step": 190 + }, + { + "epoch": 0.06, + "learning_rate": 0.00014028776978417266, + "loss": 0.2858, + "step": 195 + }, + { + "epoch": 0.06, + "learning_rate": 0.00014388489208633093, + "loss": 0.2724, + "step": 200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0001474820143884892, + "loss": 0.2726, + "step": 205 + }, + { + "epoch": 0.07, + "learning_rate": 0.00015107913669064747, + "loss": 0.2678, + "step": 210 + }, + { + "epoch": 0.07, + "learning_rate": 0.00015467625899280577, + "loss": 0.2789, + "step": 215 + }, + { + "epoch": 0.07, + "learning_rate": 0.00015827338129496403, + "loss": 0.2742, + "step": 220 + }, + { + "epoch": 0.07, + "learning_rate": 0.0001618705035971223, + "loss": 0.2732, + "step": 225 + }, + { + "epoch": 0.07, + "learning_rate": 0.00016546762589928057, + "loss": 0.2741, + "step": 230 + }, + { + "epoch": 0.08, + "learning_rate": 0.00016906474820143884, + "loss": 0.2665, + "step": 235 + }, + { + "epoch": 0.08, + "learning_rate": 0.00017266187050359714, + "loss": 0.2771, + "step": 240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0001762589928057554, + "loss": 0.2751, + "step": 245 + }, + { + "epoch": 0.08, + "learning_rate": 0.00017985611510791367, + "loss": 0.2597, + "step": 250 + }, + { + "epoch": 0.08, + "learning_rate": 0.00018345323741007194, + "loss": 0.275, + "step": 255 + }, + { + "epoch": 0.08, + "learning_rate": 0.0001870503597122302, + "loss": 0.2791, + "step": 260 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019064748201438848, + "loss": 0.2493, + "step": 265 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019424460431654677, + "loss": 0.27, + "step": 270 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019784172661870504, + "loss": 0.2676, + "step": 275 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019999997550557367, + "loss": 0.2613, + "step": 280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019999969994341502, + "loss": 0.2747, + "step": 285 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019999911820191125, + "loss": 0.2699, + "step": 290 + }, + { + "epoch": 0.1, + "learning_rate": 0.0001999982302828436, + "loss": 0.2692, + "step": 295 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999703618893072, + "loss": 0.2522, + "step": 300 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999553592382863, + "loss": 0.2541, + "step": 305 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999372949213085, + "loss": 0.2614, + "step": 310 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999161689936837, + "loss": 0.2481, + "step": 315 + }, + { + "epoch": 0.1, + "learning_rate": 0.0001999891981520095, + "loss": 0.2497, + "step": 320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019998647325745995, + "loss": 0.2545, + "step": 325 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019998344222406283, + "loss": 0.2501, + "step": 330 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019998010506109854, + "loss": 0.2577, + "step": 335 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019997646177878487, + "loss": 0.2585, + "step": 340 + }, + { + "epoch": 0.11, + "learning_rate": 0.0001999725123882768, + "loss": 0.2494, + "step": 345 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019996825690166657, + "loss": 0.2538, + "step": 350 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019996369533198364, + "loss": 0.2469, + "step": 355 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019995882769319466, + "loss": 0.2502, + "step": 360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0001999536540002034, + "loss": 0.2361, + "step": 365 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019994817426885057, + "loss": 0.2446, + "step": 370 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019994238851591417, + "loss": 0.256, + "step": 375 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019993629675910896, + "loss": 0.2352, + "step": 380 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019992989901708672, + "loss": 0.2538, + "step": 385 + }, + { + "epoch": 0.13, + "learning_rate": 0.0001999231953094361, + "loss": 0.2523, + "step": 390 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019991618565668243, + "loss": 0.2512, + "step": 395 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019990887008028798, + "loss": 0.2402, + "step": 400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019990124860265156, + "loss": 0.2423, + "step": 405 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019989332124710862, + "loss": 0.2335, + "step": 410 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019988508803793124, + "loss": 0.2284, + "step": 415 + }, + { + "epoch": 0.14, + "learning_rate": 0.00019987654900032774, + "loss": 0.2382, + "step": 420 + }, + { + "epoch": 0.14, + "learning_rate": 0.0001998677041604431, + "loss": 0.2526, + "step": 425 + }, + { + "epoch": 0.14, + "learning_rate": 0.0001998585535453584, + "loss": 0.2417, + "step": 430 + }, + { + "epoch": 0.14, + "learning_rate": 0.00019984909718309105, + "loss": 0.2381, + "step": 435 + }, + { + "epoch": 0.14, + "learning_rate": 0.0001998393351025945, + "loss": 0.2418, + "step": 440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00019982926733375843, + "loss": 0.2409, + "step": 445 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001998188939074083, + "loss": 0.2467, + "step": 450 + }, + { + "epoch": 0.15, + "learning_rate": 0.00019980821485530544, + "loss": 0.2305, + "step": 455 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001997972302101471, + "loss": 0.232, + "step": 460 + }, + { + "epoch": 0.15, + "learning_rate": 0.00019978594000556602, + "loss": 0.225, + "step": 465 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001997743442761306, + "loss": 0.2259, + "step": 470 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001997624430573447, + "loss": 0.2414, + "step": 475 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019975023638564747, + "loss": 0.238, + "step": 480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019973772429841336, + "loss": 0.224, + "step": 485 + }, + { + "epoch": 0.16, + "learning_rate": 0.0001997249068339519, + "loss": 0.2369, + "step": 490 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019971178403150762, + "loss": 0.2393, + "step": 495 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019969835593125997, + "loss": 0.2249, + "step": 500 + }, + { + "epoch": 0.16, + "learning_rate": 0.0001996846225743231, + "loss": 0.2369, + "step": 505 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019967058400274592, + "loss": 0.2267, + "step": 510 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019965624025951168, + "loss": 0.2294, + "step": 515 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019964159138853815, + "loss": 0.2255, + "step": 520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019962663743467726, + "loss": 0.22, + "step": 525 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019961137844371507, + "loss": 0.2398, + "step": 530 + }, + { + "epoch": 0.17, + "learning_rate": 0.0001995958144623716, + "loss": 0.2431, + "step": 535 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019957994553830067, + "loss": 0.2245, + "step": 540 + }, + { + "epoch": 0.18, + "learning_rate": 0.0001995637717200898, + "loss": 0.2295, + "step": 545 + }, + { + "epoch": 0.18, + "learning_rate": 0.00019954729305726008, + "loss": 0.2288, + "step": 550 + }, + { + "epoch": 0.18, + "learning_rate": 0.00019953050960026586, + "loss": 0.2333, + "step": 555 + }, + { + "epoch": 0.18, + "learning_rate": 0.0001995134214004948, + "loss": 0.2316, + "step": 560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0001994960285102676, + "loss": 0.2285, + "step": 565 + }, + { + "epoch": 0.18, + "learning_rate": 0.00019947833098283782, + "loss": 0.2273, + "step": 570 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019946032887239182, + "loss": 0.2344, + "step": 575 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019944202223404854, + "loss": 0.2251, + "step": 580 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019942341112385922, + "loss": 0.226, + "step": 585 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019940449559880752, + "loss": 0.2353, + "step": 590 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019938527571680892, + "loss": 0.226, + "step": 595 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019936575153671097, + "loss": 0.221, + "step": 600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019934592311829285, + "loss": 0.2219, + "step": 605 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019932579052226528, + "loss": 0.234, + "step": 610 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019930535381027027, + "loss": 0.2181, + "step": 615 + }, + { + "epoch": 0.2, + "learning_rate": 0.000199284613044881, + "loss": 0.2191, + "step": 620 + }, + { + "epoch": 0.2, + "learning_rate": 0.0001992635682896016, + "loss": 0.235, + "step": 625 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019924221960886698, + "loss": 0.2256, + "step": 630 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019922056706804257, + "loss": 0.2265, + "step": 635 + }, + { + "epoch": 0.21, + "learning_rate": 0.0001991986107334242, + "loss": 0.2158, + "step": 640 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019917635067223777, + "loss": 0.2238, + "step": 645 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019915378695263924, + "loss": 0.2182, + "step": 650 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019913091964371423, + "loss": 0.2185, + "step": 655 + }, + { + "epoch": 0.21, + "learning_rate": 0.000199107748815478, + "loss": 0.2063, + "step": 660 + }, + { + "epoch": 0.22, + "learning_rate": 0.000199084274538875, + "loss": 0.2162, + "step": 665 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019906049688577887, + "loss": 0.2185, + "step": 670 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019903641592899205, + "loss": 0.2186, + "step": 675 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019901203174224564, + "loss": 0.2292, + "step": 680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019898734440019933, + "loss": 0.2162, + "step": 685 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019896235397844078, + "loss": 0.219, + "step": 690 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019893706055348578, + "loss": 0.218, + "step": 695 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019891146420277777, + "loss": 0.2211, + "step": 700 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019888556500468775, + "loss": 0.2227, + "step": 705 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019885936303851395, + "loss": 0.2214, + "step": 710 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019883285838448163, + "loss": 0.2204, + "step": 715 + }, + { + "epoch": 0.23, + "learning_rate": 0.0001988060511237428, + "loss": 0.2228, + "step": 720 + }, + { + "epoch": 0.23, + "learning_rate": 0.000198778941338376, + "loss": 0.2259, + "step": 725 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019875152911138602, + "loss": 0.2062, + "step": 730 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019872381452670372, + "loss": 0.2073, + "step": 735 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019869579766918568, + "loss": 0.2172, + "step": 740 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019866747862461396, + "loss": 0.21, + "step": 745 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019863885747969598, + "loss": 0.2195, + "step": 750 + }, + { + "epoch": 0.24, + "learning_rate": 0.0001986099343220639, + "loss": 0.2129, + "step": 755 + }, + { + "epoch": 0.25, + "learning_rate": 0.0001985807092402748, + "loss": 0.215, + "step": 760 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019855118232381015, + "loss": 0.2085, + "step": 765 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019852135366307545, + "loss": 0.2196, + "step": 770 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019849122334940022, + "loss": 0.2129, + "step": 775 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019846079147503755, + "loss": 0.2221, + "step": 780 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019843005813316377, + "loss": 0.2133, + "step": 785 + }, + { + "epoch": 0.26, + "learning_rate": 0.0001983990234178784, + "loss": 0.2066, + "step": 790 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019836768742420352, + "loss": 0.2188, + "step": 795 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019833605024808382, + "loss": 0.2128, + "step": 800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019830411198638607, + "loss": 0.2109, + "step": 805 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019827187273689894, + "loss": 0.2141, + "step": 810 + }, + { + "epoch": 0.26, + "learning_rate": 0.0001982393325983326, + "loss": 0.2098, + "step": 815 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019820649167031856, + "loss": 0.2188, + "step": 820 + }, + { + "epoch": 0.27, + "learning_rate": 0.0001981733500534093, + "loss": 0.2107, + "step": 825 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019813990784907785, + "loss": 0.2162, + "step": 830 + }, + { + "epoch": 0.27, + "learning_rate": 0.0001981061651597177, + "loss": 0.2189, + "step": 835 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019807212208864227, + "loss": 0.2117, + "step": 840 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019803777874008468, + "loss": 0.209, + "step": 845 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019800313521919753, + "loss": 0.206, + "step": 850 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019796819163205246, + "loss": 0.2104, + "step": 855 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019793294808563984, + "loss": 0.2016, + "step": 860 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019789740468786841, + "loss": 0.212, + "step": 865 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019786156154756506, + "loss": 0.2051, + "step": 870 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019782541877447443, + "loss": 0.21, + "step": 875 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019778897647925855, + "loss": 0.2104, + "step": 880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0001977522347734965, + "loss": 0.2179, + "step": 885 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019771519376968416, + "loss": 0.2139, + "step": 890 + }, + { + "epoch": 0.29, + "learning_rate": 0.0001976778535812338, + "loss": 0.2034, + "step": 895 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019764021432247368, + "loss": 0.2067, + "step": 900 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019760227610864776, + "loss": 0.2189, + "step": 905 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019756403905591538, + "loss": 0.2119, + "step": 910 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019752550328135086, + "loss": 0.2004, + "step": 915 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019748666890294315, + "loss": 0.2073, + "step": 920 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019744753603959545, + "loss": 0.1995, + "step": 925 + }, + { + "epoch": 0.3, + "learning_rate": 0.0001974081048111248, + "loss": 0.2034, + "step": 930 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019736837533826192, + "loss": 0.2015, + "step": 935 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019732834774265057, + "loss": 0.2166, + "step": 940 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019728802214684734, + "loss": 0.1987, + "step": 945 + }, + { + "epoch": 0.31, + "learning_rate": 0.0001972473986743213, + "loss": 0.2051, + "step": 950 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019720647744945344, + "loss": 0.2055, + "step": 955 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019716525859753652, + "loss": 0.2005, + "step": 960 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019712374224477452, + "loss": 0.1995, + "step": 965 + }, + { + "epoch": 0.31, + "learning_rate": 0.0001970819285182823, + "loss": 0.2096, + "step": 970 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019703981754608525, + "loss": 0.2024, + "step": 975 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019699740945711884, + "loss": 0.2006, + "step": 980 + }, + { + "epoch": 0.32, + "learning_rate": 0.0001969547043812283, + "loss": 0.2077, + "step": 985 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019691170244916814, + "loss": 0.2094, + "step": 990 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019686840379260176, + "loss": 0.2051, + "step": 995 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019682480854410115, + "loss": 0.2158, + "step": 1000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0001967809168371463, + "loss": 0.2035, + "step": 1005 + }, + { + "epoch": 0.33, + "learning_rate": 0.000196736728806125, + "loss": 0.2104, + "step": 1010 + }, + { + "epoch": 0.33, + "learning_rate": 0.00019669224458633222, + "loss": 0.1969, + "step": 1015 + }, + { + "epoch": 0.33, + "learning_rate": 0.0001966474643139699, + "loss": 0.2058, + "step": 1020 + }, + { + "epoch": 0.33, + "learning_rate": 0.00019660238812614647, + "loss": 0.2045, + "step": 1025 + }, + { + "epoch": 0.33, + "learning_rate": 0.0001965570161608762, + "loss": 0.1916, + "step": 1030 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019651134855707913, + "loss": 0.2008, + "step": 1035 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019646538545458047, + "loss": 0.2095, + "step": 1040 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019641912699411016, + "loss": 0.2009, + "step": 1045 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019637257331730248, + "loss": 0.2059, + "step": 1050 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019632572456669558, + "loss": 0.2084, + "step": 1055 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019627858088573112, + "loss": 0.1997, + "step": 1060 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001962311424187537, + "loss": 0.2068, + "step": 1065 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001961834093110106, + "loss": 0.1978, + "step": 1070 + }, + { + "epoch": 0.35, + "learning_rate": 0.00019613538170865118, + "loss": 0.2028, + "step": 1075 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001960870597587264, + "loss": 0.1996, + "step": 1080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001960384436091886, + "loss": 0.2002, + "step": 1085 + }, + { + "epoch": 0.35, + "learning_rate": 0.00019598953340889082, + "loss": 0.198, + "step": 1090 + }, + { + "epoch": 0.35, + "learning_rate": 0.00019594032930758643, + "loss": 0.1974, + "step": 1095 + }, + { + "epoch": 0.36, + "learning_rate": 0.0001958908314559287, + "loss": 0.1983, + "step": 1100 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019584104000547025, + "loss": 0.1987, + "step": 1105 + }, + { + "epoch": 0.36, + "learning_rate": 0.0001957909551086627, + "loss": 0.2003, + "step": 1110 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019574057691885613, + "loss": 0.1982, + "step": 1115 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019568990559029858, + "loss": 0.2031, + "step": 1120 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019563894127813564, + "loss": 0.1969, + "step": 1125 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019558768413841006, + "loss": 0.1916, + "step": 1130 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019553613432806102, + "loss": 0.2124, + "step": 1135 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019548429200492388, + "loss": 0.2042, + "step": 1140 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019543215732772956, + "loss": 0.201, + "step": 1145 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019537973045610424, + "loss": 0.2001, + "step": 1150 + }, + { + "epoch": 0.37, + "learning_rate": 0.0001953270115505686, + "loss": 0.1978, + "step": 1155 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019527400077253755, + "loss": 0.192, + "step": 1160 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019522069828431957, + "loss": 0.1946, + "step": 1165 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019516710424911642, + "loss": 0.1848, + "step": 1170 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019511321883102244, + "loss": 0.2005, + "step": 1175 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019505904219502414, + "loss": 0.1966, + "step": 1180 + }, + { + "epoch": 0.38, + "learning_rate": 0.0001950045745069997, + "loss": 0.1863, + "step": 1185 + }, + { + "epoch": 0.39, + "learning_rate": 0.0001949498159337184, + "loss": 0.1983, + "step": 1190 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019489476664284023, + "loss": 0.2113, + "step": 1195 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019483942680291523, + "loss": 0.1956, + "step": 1200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019478379658338305, + "loss": 0.2055, + "step": 1205 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019472787615457246, + "loss": 0.1879, + "step": 1210 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019467166568770078, + "loss": 0.1924, + "step": 1215 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019461516535487338, + "loss": 0.1971, + "step": 1220 + }, + { + "epoch": 0.4, + "learning_rate": 0.0001945583753290831, + "loss": 0.1952, + "step": 1225 + }, + { + "epoch": 0.4, + "learning_rate": 0.0001945012957842098, + "loss": 0.1838, + "step": 1230 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019444392689501978, + "loss": 0.1931, + "step": 1235 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019438626883716526, + "loss": 0.2093, + "step": 1240 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019432832178718385, + "loss": 0.1992, + "step": 1245 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019427008592249795, + "loss": 0.1963, + "step": 1250 + }, + { + "epoch": 0.41, + "learning_rate": 0.0001942115614214143, + "loss": 0.2003, + "step": 1255 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019415274846312338, + "loss": 0.1914, + "step": 1260 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019409364722769882, + "loss": 0.1917, + "step": 1265 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019403425789609697, + "loss": 0.2033, + "step": 1270 + }, + { + "epoch": 0.41, + "learning_rate": 0.0001939745806501563, + "loss": 0.1975, + "step": 1275 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019391461567259662, + "loss": 0.1922, + "step": 1280 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019385436314701897, + "loss": 0.1999, + "step": 1285 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019379382325790468, + "loss": 0.1997, + "step": 1290 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019373299619061487, + "loss": 0.1872, + "step": 1295 + }, + { + "epoch": 0.42, + "learning_rate": 0.0001936718821313901, + "loss": 0.1936, + "step": 1300 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019361048126734952, + "loss": 0.1798, + "step": 1305 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019354879378649046, + "loss": 0.1959, + "step": 1310 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019348681987768783, + "loss": 0.1924, + "step": 1315 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019342455973069345, + "loss": 0.1967, + "step": 1320 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019336201353613568, + "loss": 0.1925, + "step": 1325 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019329918148551856, + "loss": 0.1878, + "step": 1330 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019323606377122146, + "loss": 0.1979, + "step": 1335 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019317266058649834, + "loss": 0.189, + "step": 1340 + }, + { + "epoch": 0.44, + "learning_rate": 0.0001931089721254773, + "loss": 0.1995, + "step": 1345 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019304499858315976, + "loss": 0.1874, + "step": 1350 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019298074015542015, + "loss": 0.1991, + "step": 1355 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019291619703900502, + "loss": 0.188, + "step": 1360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0001928513694315327, + "loss": 0.1914, + "step": 1365 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019278625753149254, + "loss": 0.1843, + "step": 1370 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019272086153824427, + "loss": 0.1959, + "step": 1375 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019265518165201758, + "loss": 0.1867, + "step": 1380 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019258921807391131, + "loss": 0.1956, + "step": 1385 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019252297100589286, + "loss": 0.1991, + "step": 1390 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019245644065079775, + "loss": 0.1934, + "step": 1395 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019238962721232876, + "loss": 0.1953, + "step": 1400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019232253089505546, + "loss": 0.1865, + "step": 1405 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019225515190441354, + "loss": 0.1864, + "step": 1410 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019218749044670413, + "loss": 0.194, + "step": 1415 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019211954672909327, + "loss": 0.1853, + "step": 1420 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019205132095961126, + "loss": 0.1967, + "step": 1425 + }, + { + "epoch": 0.46, + "learning_rate": 0.0001919828133471519, + "loss": 0.1852, + "step": 1430 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019191402410147204, + "loss": 0.1913, + "step": 1435 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019184495343319068, + "loss": 0.1911, + "step": 1440 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019177560155378865, + "loss": 0.189, + "step": 1445 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019170596867560766, + "loss": 0.1854, + "step": 1450 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019163605501184986, + "loss": 0.1974, + "step": 1455 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019156586077657712, + "loss": 0.1933, + "step": 1460 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019149538618471032, + "loss": 0.1877, + "step": 1465 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001914246314520287, + "loss": 0.1904, + "step": 1470 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001913535967951694, + "loss": 0.1841, + "step": 1475 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001912822824316264, + "loss": 0.1876, + "step": 1480 + }, + { + "epoch": 0.48, + "learning_rate": 0.00019121068857975027, + "loss": 0.1858, + "step": 1485 + }, + { + "epoch": 0.48, + "learning_rate": 0.00019113881545874722, + "loss": 0.1898, + "step": 1490 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001910666632886786, + "loss": 0.1923, + "step": 1495 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019099423229046014, + "loss": 0.1803, + "step": 1500 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019092152268586118, + "loss": 0.1945, + "step": 1505 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019084853469750424, + "loss": 0.1802, + "step": 1510 + }, + { + "epoch": 0.49, + "learning_rate": 0.0001907752685488641, + "loss": 0.197, + "step": 1515 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019070172446426724, + "loss": 0.1791, + "step": 1520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019062790266889117, + "loss": 0.1874, + "step": 1525 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019055380338876363, + "loss": 0.1806, + "step": 1530 + }, + { + "epoch": 0.5, + "learning_rate": 0.000190479426850762, + "loss": 0.1834, + "step": 1535 + }, + { + "epoch": 0.5, + "learning_rate": 0.0001904047732826126, + "loss": 0.1921, + "step": 1540 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019032984291288986, + "loss": 0.1826, + "step": 1545 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019025463597101584, + "loss": 0.1966, + "step": 1550 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019017915268725925, + "loss": 0.1867, + "step": 1555 + }, + { + "epoch": 0.51, + "learning_rate": 0.00019010339329273514, + "loss": 0.196, + "step": 1560 + }, + { + "epoch": 0.51, + "learning_rate": 0.00019002735801940373, + "loss": 0.1971, + "step": 1565 + }, + { + "epoch": 0.51, + "learning_rate": 0.00018995104710007004, + "loss": 0.187, + "step": 1570 + }, + { + "epoch": 0.51, + "learning_rate": 0.00018987446076838312, + "loss": 0.1897, + "step": 1575 + }, + { + "epoch": 0.51, + "learning_rate": 0.00018979759925883509, + "loss": 0.1887, + "step": 1580 + }, + { + "epoch": 0.51, + "learning_rate": 0.0001897204628067608, + "loss": 0.1849, + "step": 1585 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018964305164833678, + "loss": 0.1795, + "step": 1590 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018956536602058078, + "loss": 0.1984, + "step": 1595 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018948740616135085, + "loss": 0.1876, + "step": 1600 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018940917230934464, + "loss": 0.1879, + "step": 1605 + }, + { + "epoch": 0.52, + "learning_rate": 0.0001893306647040988, + "loss": 0.1906, + "step": 1610 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018925188358598813, + "loss": 0.1917, + "step": 1615 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018917282919622488, + "loss": 0.1912, + "step": 1620 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018909350177685795, + "loss": 0.1813, + "step": 1625 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018901390157077225, + "loss": 0.183, + "step": 1630 + }, + { + "epoch": 0.53, + "learning_rate": 0.0001889340288216879, + "loss": 0.1782, + "step": 1635 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018885388377415957, + "loss": 0.1866, + "step": 1640 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018877346667357548, + "loss": 0.1915, + "step": 1645 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018869277776615691, + "loss": 0.1902, + "step": 1650 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018861181729895745, + "loss": 0.1829, + "step": 1655 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018853058551986202, + "loss": 0.1855, + "step": 1660 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018844908267758633, + "loss": 0.1845, + "step": 1665 + }, + { + "epoch": 0.54, + "learning_rate": 0.000188367309021676, + "loss": 0.1794, + "step": 1670 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018828526480250575, + "loss": 0.1796, + "step": 1675 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018820295027127885, + "loss": 0.1806, + "step": 1680 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018812036568002617, + "loss": 0.1846, + "step": 1685 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018803751128160538, + "loss": 0.189, + "step": 1690 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018795438732970032, + "loss": 0.1856, + "step": 1695 + }, + { + "epoch": 0.55, + "learning_rate": 0.0001878709940788201, + "loss": 0.1797, + "step": 1700 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018778733178429844, + "loss": 0.1876, + "step": 1705 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018770340070229272, + "loss": 0.1775, + "step": 1710 + }, + { + "epoch": 0.56, + "learning_rate": 0.0001876192010897834, + "loss": 0.1862, + "step": 1715 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018753473320457306, + "loss": 0.1835, + "step": 1720 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018744999730528568, + "loss": 0.1909, + "step": 1725 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018736499365136584, + "loss": 0.189, + "step": 1730 + }, + { + "epoch": 0.56, + "learning_rate": 0.000187279722503078, + "loss": 0.1861, + "step": 1735 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018719418412150557, + "loss": 0.1809, + "step": 1740 + }, + { + "epoch": 0.57, + "learning_rate": 0.00018710837876855017, + "loss": 0.1829, + "step": 1745 + }, + { + "epoch": 0.57, + "learning_rate": 0.0001870223067069309, + "loss": 0.1886, + "step": 1750 + }, + { + "epoch": 0.57, + "learning_rate": 0.00018693596820018336, + "loss": 0.1787, + "step": 1755 + }, + { + "epoch": 0.57, + "learning_rate": 0.0001868493635126591, + "loss": 0.1816, + "step": 1760 + }, + { + "epoch": 0.57, + "learning_rate": 0.0001867624929095245, + "loss": 0.1759, + "step": 1765 + }, + { + "epoch": 0.57, + "learning_rate": 0.00018667535665676025, + "loss": 0.1702, + "step": 1770 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018658795502116035, + "loss": 0.1802, + "step": 1775 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018650028827033136, + "loss": 0.1836, + "step": 1780 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018641235667269155, + "loss": 0.1787, + "step": 1785 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018632416049747016, + "loss": 0.1811, + "step": 1790 + }, + { + "epoch": 0.58, + "learning_rate": 0.0001862357000147064, + "loss": 0.1835, + "step": 1795 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018614697549524889, + "loss": 0.1857, + "step": 1800 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018605798721075455, + "loss": 0.1867, + "step": 1805 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018596873543368797, + "loss": 0.1766, + "step": 1810 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018587922043732046, + "loss": 0.1895, + "step": 1815 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018578944249572925, + "loss": 0.1888, + "step": 1820 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018569940188379672, + "loss": 0.1871, + "step": 1825 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018560909887720944, + "loss": 0.1804, + "step": 1830 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018551853375245737, + "loss": 0.1877, + "step": 1835 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018542770678683308, + "loss": 0.182, + "step": 1840 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001853366182584308, + "loss": 0.1809, + "step": 1845 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018524526844614564, + "loss": 0.1913, + "step": 1850 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018515365762967266, + "loss": 0.185, + "step": 1855 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018506178608950613, + "loss": 0.1753, + "step": 1860 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018496965410693856, + "loss": 0.1786, + "step": 1865 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001848772619640599, + "loss": 0.1841, + "step": 1870 + }, + { + "epoch": 0.61, + "learning_rate": 0.00018478460994375667, + "loss": 0.184, + "step": 1875 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001846916983297111, + "loss": 0.179, + "step": 1880 + }, + { + "epoch": 0.61, + "learning_rate": 0.00018459852740640014, + "loss": 0.1833, + "step": 1885 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001845050974590948, + "loss": 0.186, + "step": 1890 + }, + { + "epoch": 0.61, + "learning_rate": 0.00018441140877385913, + "loss": 0.1778, + "step": 1895 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018431746163754943, + "loss": 0.1786, + "step": 1900 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018422325633781327, + "loss": 0.1826, + "step": 1905 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018412879316308866, + "loss": 0.1738, + "step": 1910 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018403407240260327, + "loss": 0.1852, + "step": 1915 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018393909434637329, + "loss": 0.1821, + "step": 1920 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001838438592852028, + "loss": 0.1773, + "step": 1925 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018374836751068282, + "loss": 0.1772, + "step": 1930 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018365261931519022, + "loss": 0.1766, + "step": 1935 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018355661499188717, + "loss": 0.1858, + "step": 1940 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018346035483471994, + "loss": 0.1786, + "step": 1945 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001833638391384181, + "loss": 0.1745, + "step": 1950 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018326706819849365, + "loss": 0.1713, + "step": 1955 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018317004231124017, + "loss": 0.1738, + "step": 1960 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018307276177373174, + "loss": 0.1807, + "step": 1965 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018297522688382218, + "loss": 0.1806, + "step": 1970 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018287743794014407, + "loss": 0.1735, + "step": 1975 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018277939524210783, + "loss": 0.1812, + "step": 1980 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001826810990899009, + "loss": 0.1734, + "step": 1985 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001825825497844867, + "loss": 0.1703, + "step": 1990 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018248374762760365, + "loss": 0.1823, + "step": 1995 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018238469292176457, + "loss": 0.1837, + "step": 2000 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018228538597025537, + "loss": 0.1807, + "step": 2005 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018218582707713434, + "loss": 0.1904, + "step": 2010 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001820860165472312, + "loss": 0.1844, + "step": 2015 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018198595468614604, + "loss": 0.1792, + "step": 2020 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018188564180024854, + "loss": 0.1845, + "step": 2025 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018178507819667698, + "loss": 0.1853, + "step": 2030 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018168426418333725, + "loss": 0.1875, + "step": 2035 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018158320006890198, + "loss": 0.1759, + "step": 2040 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018148188616280955, + "loss": 0.1816, + "step": 2045 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001813803227752631, + "loss": 0.1787, + "step": 2050 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001812785102172297, + "loss": 0.1758, + "step": 2055 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018117644880043932, + "loss": 0.1802, + "step": 2060 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018107413883738387, + "loss": 0.186, + "step": 2065 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018097158064131626, + "loss": 0.1743, + "step": 2070 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018086877452624944, + "loss": 0.1877, + "step": 2075 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018076572080695543, + "loss": 0.1742, + "step": 2080 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018066241979896444, + "loss": 0.1727, + "step": 2085 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018055887181856375, + "loss": 0.1806, + "step": 2090 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018045507718279677, + "loss": 0.175, + "step": 2095 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018035103620946225, + "loss": 0.1727, + "step": 2100 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001802467492171131, + "loss": 0.1779, + "step": 2105 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018014221652505556, + "loss": 0.1823, + "step": 2110 + }, + { + "epoch": 0.69, + "learning_rate": 0.000180037438453348, + "loss": 0.1814, + "step": 2115 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001799324153228003, + "loss": 0.1707, + "step": 2120 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001798271474549725, + "loss": 0.1778, + "step": 2125 + }, + { + "epoch": 0.69, + "learning_rate": 0.000179721635172174, + "loss": 0.1888, + "step": 2130 + }, + { + "epoch": 0.69, + "learning_rate": 0.00017961587879746266, + "loss": 0.1785, + "step": 2135 + }, + { + "epoch": 0.69, + "learning_rate": 0.00017950987865464365, + "loss": 0.1661, + "step": 2140 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017940363506826836, + "loss": 0.1792, + "step": 2145 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001792971483636338, + "loss": 0.1732, + "step": 2150 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017919041886678118, + "loss": 0.1771, + "step": 2155 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001790834469044952, + "loss": 0.173, + "step": 2160 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017897623280430284, + "loss": 0.1768, + "step": 2165 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017886877689447258, + "loss": 0.175, + "step": 2170 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001787610795040132, + "loss": 0.1691, + "step": 2175 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001786531409626728, + "loss": 0.1774, + "step": 2180 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001785449616009379, + "loss": 0.1709, + "step": 2185 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001784365417500324, + "loss": 0.1809, + "step": 2190 + }, + { + "epoch": 0.71, + "learning_rate": 0.00017832788174191645, + "loss": 0.1764, + "step": 2195 + }, + { + "epoch": 0.71, + "learning_rate": 0.00017821898190928558, + "loss": 0.1739, + "step": 2200 + }, + { + "epoch": 0.71, + "learning_rate": 0.00017810984258556957, + "loss": 0.1772, + "step": 2205 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017800046410493145, + "loss": 0.1779, + "step": 2210 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001778908468022666, + "loss": 0.1828, + "step": 2215 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017778099101320156, + "loss": 0.1722, + "step": 2220 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017767089707409304, + "loss": 0.1737, + "step": 2225 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017756056532202696, + "loss": 0.1688, + "step": 2230 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017744999609481738, + "loss": 0.1784, + "step": 2235 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017733918973100545, + "loss": 0.1747, + "step": 2240 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017722814656985841, + "loss": 0.1667, + "step": 2245 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001771168669513685, + "loss": 0.1648, + "step": 2250 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017700535121625187, + "loss": 0.1777, + "step": 2255 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017689359970594779, + "loss": 0.175, + "step": 2260 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001767816127626173, + "loss": 0.1715, + "step": 2265 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001766693907291423, + "loss": 0.1742, + "step": 2270 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017655693394912452, + "loss": 0.1868, + "step": 2275 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017644424276688441, + "loss": 0.1757, + "step": 2280 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017633131752746017, + "loss": 0.1803, + "step": 2285 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017621815857660656, + "loss": 0.1812, + "step": 2290 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017610476626079395, + "loss": 0.1691, + "step": 2295 + }, + { + "epoch": 0.75, + "learning_rate": 0.00017599114092720726, + "loss": 0.1662, + "step": 2300 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001758772829237448, + "loss": 0.1709, + "step": 2305 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001757631925990174, + "loss": 0.1822, + "step": 2310 + }, + { + "epoch": 0.75, + "learning_rate": 0.00017564887030234696, + "loss": 0.1732, + "step": 2315 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001755343163837659, + "loss": 0.1854, + "step": 2320 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001754195311940157, + "loss": 0.1768, + "step": 2325 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017530451508454594, + "loss": 0.1785, + "step": 2330 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017518926840751325, + "loss": 0.1724, + "step": 2335 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017507379151578017, + "loss": 0.1812, + "step": 2340 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017495808476291423, + "loss": 0.1754, + "step": 2345 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017484214850318664, + "loss": 0.1799, + "step": 2350 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001747259830915713, + "loss": 0.1769, + "step": 2355 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017460958888374385, + "loss": 0.1764, + "step": 2360 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001744929662360803, + "loss": 0.1716, + "step": 2365 + }, + { + "epoch": 0.77, + "learning_rate": 0.00017437611550565624, + "loss": 0.1699, + "step": 2370 + }, + { + "epoch": 0.77, + "learning_rate": 0.00017425903705024553, + "loss": 0.1661, + "step": 2375 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001741417312283193, + "loss": 0.1699, + "step": 2380 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001740241983990448, + "loss": 0.1682, + "step": 2385 + }, + { + "epoch": 0.77, + "learning_rate": 0.00017390643892228438, + "loss": 0.173, + "step": 2390 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001737884531585943, + "loss": 0.1852, + "step": 2395 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017367024146922372, + "loss": 0.1681, + "step": 2400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017355180421611347, + "loss": 0.1798, + "step": 2405 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017343314176189502, + "loss": 0.1705, + "step": 2410 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017331425446988946, + "loss": 0.177, + "step": 2415 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017319514270410614, + "loss": 0.1754, + "step": 2420 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017307580682924178, + "loss": 0.1679, + "step": 2425 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017295624721067932, + "loss": 0.1763, + "step": 2430 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017283646421448673, + "loss": 0.1769, + "step": 2435 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017271645820741583, + "loss": 0.1753, + "step": 2440 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001725962295569014, + "loss": 0.1715, + "step": 2445 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017247577863105983, + "loss": 0.1784, + "step": 2450 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017235510579868802, + "loss": 0.17, + "step": 2455 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017223421142926245, + "loss": 0.1686, + "step": 2460 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017211309589293774, + "loss": 0.166, + "step": 2465 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017199175956054584, + "loss": 0.1652, + "step": 2470 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017187020280359455, + "loss": 0.1732, + "step": 2475 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001717484259942668, + "loss": 0.1754, + "step": 2480 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017162642950541902, + "loss": 0.1748, + "step": 2485 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017150421371058049, + "loss": 0.1733, + "step": 2490 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017138177898395177, + "loss": 0.1737, + "step": 2495 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017125912570040383, + "loss": 0.175, + "step": 2500 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001711362542354769, + "loss": 0.173, + "step": 2505 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017101316496537905, + "loss": 0.1796, + "step": 2510 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017088985826698542, + "loss": 0.1722, + "step": 2515 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017076633451783675, + "loss": 0.1738, + "step": 2520 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017064259409613834, + "loss": 0.1743, + "step": 2525 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017051863738075902, + "loss": 0.1653, + "step": 2530 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001703944647512297, + "loss": 0.1761, + "step": 2535 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017027007658774256, + "loss": 0.1649, + "step": 2540 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017014547327114956, + "loss": 0.1738, + "step": 2545 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001700206551829614, + "loss": 0.1585, + "step": 2550 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016989562270534655, + "loss": 0.183, + "step": 2555 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001697703762211297, + "loss": 0.1693, + "step": 2560 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001696449161137909, + "loss": 0.173, + "step": 2565 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016951924276746425, + "loss": 0.1691, + "step": 2570 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016939335656693667, + "loss": 0.1698, + "step": 2575 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016926725789764691, + "loss": 0.1706, + "step": 2580 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016914094714568417, + "loss": 0.1691, + "step": 2585 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016901442469778713, + "loss": 0.1665, + "step": 2590 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016888769094134247, + "loss": 0.1831, + "step": 2595 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016876074626438395, + "loss": 0.1637, + "step": 2600 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016863359105559117, + "loss": 0.1757, + "step": 2605 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016850622570428826, + "loss": 0.1721, + "step": 2610 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016837865060044272, + "loss": 0.1647, + "step": 2615 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016825086613466442, + "loss": 0.1734, + "step": 2620 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016812287269820416, + "loss": 0.1847, + "step": 2625 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016799467068295257, + "loss": 0.1682, + "step": 2630 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001678662604814389, + "loss": 0.1741, + "step": 2635 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016773764248682988, + "loss": 0.1692, + "step": 2640 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016760881709292842, + "loss": 0.1805, + "step": 2645 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001674797846941724, + "loss": 0.1685, + "step": 2650 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016735054568563364, + "loss": 0.1673, + "step": 2655 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001672211004630164, + "loss": 0.1686, + "step": 2660 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016709144942265644, + "loss": 0.1749, + "step": 2665 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001669615929615197, + "loss": 0.1644, + "step": 2670 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016683153147720097, + "loss": 0.1673, + "step": 2675 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016670126536792287, + "loss": 0.1722, + "step": 2680 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001665707950325345, + "loss": 0.1588, + "step": 2685 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001664401208705103, + "loss": 0.164, + "step": 2690 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016630924328194877, + "loss": 0.1708, + "step": 2695 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016617816266757123, + "loss": 0.1711, + "step": 2700 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016604687942872067, + "loss": 0.1701, + "step": 2705 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016591539396736046, + "loss": 0.1708, + "step": 2710 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016578370668607307, + "loss": 0.1669, + "step": 2715 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016565181798805907, + "loss": 0.1601, + "step": 2720 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016551972827713555, + "loss": 0.1733, + "step": 2725 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016538743795773515, + "loss": 0.1641, + "step": 2730 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016525494743490474, + "loss": 0.1723, + "step": 2735 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016512225711430415, + "loss": 0.1655, + "step": 2740 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016498936740220492, + "loss": 0.172, + "step": 2745 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016485627870548917, + "loss": 0.1686, + "step": 2750 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016472299143164816, + "loss": 0.1709, + "step": 2755 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016458950598878126, + "loss": 0.1648, + "step": 2760 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016445582278559458, + "loss": 0.1641, + "step": 2765 + }, + { + "epoch": 0.9, + "learning_rate": 0.0001643219422313996, + "loss": 0.1662, + "step": 2770 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016418786473611227, + "loss": 0.1763, + "step": 2775 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016405359071025138, + "loss": 0.1773, + "step": 2780 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016391912056493743, + "loss": 0.1678, + "step": 2785 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016378445471189156, + "loss": 0.1632, + "step": 2790 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016364959356343398, + "loss": 0.1671, + "step": 2795 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016351453753248296, + "loss": 0.1739, + "step": 2800 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016337928703255336, + "loss": 0.1727, + "step": 2805 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016324384247775554, + "loss": 0.1756, + "step": 2810 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016310820428279397, + "loss": 0.1671, + "step": 2815 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016297237286296606, + "loss": 0.169, + "step": 2820 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016283634863416082, + "loss": 0.1693, + "step": 2825 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016270013201285757, + "loss": 0.164, + "step": 2830 + }, + { + "epoch": 0.92, + "learning_rate": 0.0001625637234161247, + "loss": 0.1663, + "step": 2835 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016242712326161845, + "loss": 0.1742, + "step": 2840 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016229033196758154, + "loss": 0.169, + "step": 2845 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016215334995284188, + "loss": 0.1703, + "step": 2850 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001620161776368114, + "loss": 0.1669, + "step": 2855 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001618788154394846, + "loss": 0.1634, + "step": 2860 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001617412637814375, + "loss": 0.163, + "step": 2865 + }, + { + "epoch": 0.93, + "learning_rate": 0.00016160352308382608, + "loss": 0.1651, + "step": 2870 + }, + { + "epoch": 0.93, + "learning_rate": 0.00016146559376838514, + "loss": 0.1657, + "step": 2875 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001613274762574271, + "loss": 0.1638, + "step": 2880 + }, + { + "epoch": 0.93, + "learning_rate": 0.00016118917097384048, + "loss": 0.1691, + "step": 2885 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016105067834108874, + "loss": 0.1627, + "step": 2890 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016091199878320902, + "loss": 0.1608, + "step": 2895 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016077313272481076, + "loss": 0.1617, + "step": 2900 + }, + { + "epoch": 0.94, + "learning_rate": 0.0001606340805910744, + "loss": 0.1696, + "step": 2905 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016049484280775012, + "loss": 0.1604, + "step": 2910 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016035541980115653, + "loss": 0.161, + "step": 2915 + }, + { + "epoch": 0.95, + "learning_rate": 0.00016021581199817943, + "loss": 0.1703, + "step": 2920 + }, + { + "epoch": 0.95, + "learning_rate": 0.0001600760198262703, + "loss": 0.1671, + "step": 2925 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015993604371344517, + "loss": 0.1603, + "step": 2930 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015979588408828333, + "loss": 0.1655, + "step": 2935 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015965554137992584, + "loss": 0.1675, + "step": 2940 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015951501601807437, + "loss": 0.1641, + "step": 2945 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015937430843298986, + "loss": 0.165, + "step": 2950 + }, + { + "epoch": 0.96, + "learning_rate": 0.0001592334190554912, + "loss": 0.1758, + "step": 2955 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015909234831695379, + "loss": 0.1703, + "step": 2960 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015895109664930846, + "loss": 0.1644, + "step": 2965 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015880966448503987, + "loss": 0.1638, + "step": 2970 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015866805225718542, + "loss": 0.1659, + "step": 2975 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015852626039933383, + "loss": 0.1676, + "step": 2980 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015838428934562373, + "loss": 0.1794, + "step": 2985 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015824213953074254, + "loss": 0.1562, + "step": 2990 + }, + { + "epoch": 0.97, + "learning_rate": 0.0001580998113899249, + "loss": 0.171, + "step": 2995 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015795730535895147, + "loss": 0.1681, + "step": 3000 + }, + { + "epoch": 0.97, + "learning_rate": 0.0001578146218741476, + "loss": 0.1617, + "step": 3005 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015767176137238196, + "loss": 0.1708, + "step": 3010 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015752872429106523, + "loss": 0.1636, + "step": 3015 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015738551106814872, + "loss": 0.1678, + "step": 3020 + }, + { + "epoch": 0.98, + "learning_rate": 0.000157242122142123, + "loss": 0.1612, + "step": 3025 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015709855795201675, + "loss": 0.1658, + "step": 3030 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015695481893739514, + "loss": 0.1621, + "step": 3035 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015681090553835864, + "loss": 0.1718, + "step": 3040 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015666681819554175, + "loss": 0.1712, + "step": 3045 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015652255735011135, + "loss": 0.1693, + "step": 3050 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015637812344376583, + "loss": 0.1714, + "step": 3055 + }, + { + "epoch": 0.99, + "learning_rate": 0.0001562335169187332, + "loss": 0.1661, + "step": 3060 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015608873821777012, + "loss": 0.1676, + "step": 3065 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015594378778416044, + "loss": 0.1654, + "step": 3070 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015579866606171372, + "loss": 0.1644, + "step": 3075 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015565337349476404, + "loss": 0.1744, + "step": 3080 + }, + { + "epoch": 1.0, + "learning_rate": 0.0001555079105281686, + "loss": 0.1606, + "step": 3085 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015536227760730632, + "loss": 0.1659, + "step": 3090 + }, + { + "epoch": 1.0, + "learning_rate": 0.0001552164751780764, + "loss": 0.1602, + "step": 3095 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015507050368689714, + "loss": 0.1636, + "step": 3100 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015492436358070438, + "loss": 0.1555, + "step": 3105 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015477805530695035, + "loss": 0.1661, + "step": 3110 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015463157931360207, + "loss": 0.1628, + "step": 3115 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015448493604914014, + "loss": 0.1696, + "step": 3120 + }, + { + "epoch": 1.01, + "learning_rate": 0.0001543381259625573, + "loss": 0.1685, + "step": 3125 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015419114950335704, + "loss": 0.1635, + "step": 3130 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015404400712155228, + "loss": 0.1631, + "step": 3135 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015389669926766395, + "loss": 0.1664, + "step": 3140 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015374922639271963, + "loss": 0.1617, + "step": 3145 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015360158894825216, + "loss": 0.174, + "step": 3150 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015345378738629826, + "loss": 0.1591, + "step": 3155 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015330582215939713, + "loss": 0.1729, + "step": 3160 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015315769372058907, + "loss": 0.1732, + "step": 3165 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015300940252341416, + "loss": 0.1608, + "step": 3170 + }, + { + "epoch": 1.03, + "learning_rate": 0.0001528609490219108, + "loss": 0.168, + "step": 3175 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015271233367061425, + "loss": 0.1731, + "step": 3180 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015256355692455542, + "loss": 0.1656, + "step": 3185 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015241461923925934, + "loss": 0.1591, + "step": 3190 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015226552107074385, + "loss": 0.1635, + "step": 3195 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015211626287551803, + "loss": 0.1622, + "step": 3200 + }, + { + "epoch": 1.04, + "learning_rate": 0.0001519668451105811, + "loss": 0.162, + "step": 3205 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015181726823342072, + "loss": 0.1614, + "step": 3210 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015166753270201176, + "loss": 0.1657, + "step": 3215 + }, + { + "epoch": 1.04, + "learning_rate": 0.0001515176389748149, + "loss": 0.1623, + "step": 3220 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015136758751077504, + "loss": 0.1629, + "step": 3225 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015121737876932034, + "loss": 0.1556, + "step": 3230 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015106701321036017, + "loss": 0.1651, + "step": 3235 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015091649129428424, + "loss": 0.1575, + "step": 3240 + }, + { + "epoch": 1.05, + "learning_rate": 0.000150765813481961, + "loss": 0.1594, + "step": 3245 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015061498023473608, + "loss": 0.1624, + "step": 3250 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015046399201443114, + "loss": 0.1616, + "step": 3255 + }, + { + "epoch": 1.06, + "learning_rate": 0.00015031284928334238, + "loss": 0.1693, + "step": 3260 + }, + { + "epoch": 1.06, + "learning_rate": 0.00015016155250423892, + "loss": 0.1592, + "step": 3265 + }, + { + "epoch": 1.06, + "learning_rate": 0.00015001010214036174, + "loss": 0.1542, + "step": 3270 + }, + { + "epoch": 1.06, + "learning_rate": 0.0001498584986554219, + "loss": 0.1631, + "step": 3275 + }, + { + "epoch": 1.06, + "learning_rate": 0.0001497067425135993, + "loss": 0.1612, + "step": 3280 + }, + { + "epoch": 1.06, + "learning_rate": 0.00014955483417954137, + "loss": 0.1551, + "step": 3285 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014940277411836146, + "loss": 0.155, + "step": 3290 + }, + { + "epoch": 1.07, + "learning_rate": 0.0001492505627956374, + "loss": 0.1593, + "step": 3295 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014909820067741028, + "loss": 0.1622, + "step": 3300 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014894568823018277, + "loss": 0.1637, + "step": 3305 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014879302592091798, + "loss": 0.1643, + "step": 3310 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014864021421703768, + "loss": 0.1642, + "step": 3315 + }, + { + "epoch": 1.08, + "learning_rate": 0.00014848725358642114, + "loss": 0.1607, + "step": 3320 + }, + { + "epoch": 1.08, + "learning_rate": 0.00014833414449740373, + "loss": 0.1591, + "step": 3325 + }, + { + "epoch": 1.08, + "learning_rate": 0.0001481808874187752, + "loss": 0.1682, + "step": 3330 + }, + { + "epoch": 1.08, + "learning_rate": 0.00014802748281977842, + "loss": 0.1572, + "step": 3335 + }, + { + "epoch": 1.08, + "learning_rate": 0.0001478739311701081, + "loss": 0.1595, + "step": 3340 + }, + { + "epoch": 1.08, + "learning_rate": 0.000147720232939909, + "loss": 0.1683, + "step": 3345 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014756638859977483, + "loss": 0.1539, + "step": 3350 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014741239862074655, + "loss": 0.1594, + "step": 3355 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014725826347431113, + "loss": 0.1658, + "step": 3360 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014710398363239994, + "loss": 0.1621, + "step": 3365 + }, + { + "epoch": 1.09, + "learning_rate": 0.0001469495595673874, + "loss": 0.1651, + "step": 3370 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014679499175208953, + "loss": 0.1655, + "step": 3375 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014664028065976246, + "loss": 0.1669, + "step": 3380 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014648542676410105, + "loss": 0.1593, + "step": 3385 + }, + { + "epoch": 1.1, + "learning_rate": 0.0001463304305392373, + "loss": 0.157, + "step": 3390 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014617529245973917, + "loss": 0.1593, + "step": 3395 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014602001300060877, + "loss": 0.1516, + "step": 3400 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014586459263728115, + "loss": 0.156, + "step": 3405 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014570903184562282, + "loss": 0.1637, + "step": 3410 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014555333110193022, + "loss": 0.1552, + "step": 3415 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014539749088292826, + "loss": 0.1624, + "step": 3420 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014524151166576897, + "loss": 0.1568, + "step": 3425 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014508539392802994, + "loss": 0.1583, + "step": 3430 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014492913814771283, + "loss": 0.1625, + "step": 3435 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014477274480324202, + "loss": 0.1563, + "step": 3440 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014461621437346307, + "loss": 0.1634, + "step": 3445 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014445954733764127, + "loss": 0.1452, + "step": 3450 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014430274417546012, + "loss": 0.1539, + "step": 3455 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014414580536702, + "loss": 0.1636, + "step": 3460 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014398873139283657, + "loss": 0.1509, + "step": 3465 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014383152273383934, + "loss": 0.1636, + "step": 3470 + }, + { + "epoch": 1.13, + "learning_rate": 0.00014367417987137015, + "loss": 0.1619, + "step": 3475 + }, + { + "epoch": 1.13, + "learning_rate": 0.00014351670328718183, + "loss": 0.1648, + "step": 3480 + }, + { + "epoch": 1.13, + "learning_rate": 0.00014335909346343658, + "loss": 0.1508, + "step": 3485 + }, + { + "epoch": 1.13, + "learning_rate": 0.0001432013508827046, + "loss": 0.1546, + "step": 3490 + }, + { + "epoch": 1.13, + "learning_rate": 0.0001430434760279625, + "loss": 0.1549, + "step": 3495 + }, + { + "epoch": 1.13, + "learning_rate": 0.000142885469382592, + "loss": 0.1412, + "step": 3500 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001427273314303782, + "loss": 0.1574, + "step": 3505 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001425690626555083, + "loss": 0.16, + "step": 3510 + }, + { + "epoch": 1.14, + "learning_rate": 0.00014241066354257008, + "loss": 0.1633, + "step": 3515 + }, + { + "epoch": 1.14, + "learning_rate": 0.00014225213457655034, + "loss": 0.1499, + "step": 3520 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001420934762428335, + "loss": 0.159, + "step": 3525 + }, + { + "epoch": 1.14, + "learning_rate": 0.00014193468902720007, + "loss": 0.1561, + "step": 3530 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014177577341582516, + "loss": 0.1619, + "step": 3535 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014161672989527698, + "loss": 0.16, + "step": 3540 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014145755895251546, + "loss": 0.148, + "step": 3545 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014129826107489054, + "loss": 0.1502, + "step": 3550 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014113883675014098, + "loss": 0.1427, + "step": 3555 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001409792864663925, + "loss": 0.1613, + "step": 3560 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001408196107121566, + "loss": 0.1597, + "step": 3565 + }, + { + "epoch": 1.16, + "learning_rate": 0.00014065980997632896, + "loss": 0.1501, + "step": 3570 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001404998847481879, + "loss": 0.1537, + "step": 3575 + }, + { + "epoch": 1.16, + "learning_rate": 0.00014033983551739286, + "loss": 0.1608, + "step": 3580 + }, + { + "epoch": 1.16, + "learning_rate": 0.00014017966277398304, + "loss": 0.1488, + "step": 3585 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001400193670083757, + "loss": 0.1593, + "step": 3590 + }, + { + "epoch": 1.16, + "learning_rate": 0.00013985894871136492, + "loss": 0.1525, + "step": 3595 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013969840837411982, + "loss": 0.1529, + "step": 3600 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013953774648818324, + "loss": 0.1489, + "step": 3605 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013937696354547022, + "loss": 0.144, + "step": 3610 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013921606003826627, + "loss": 0.1584, + "step": 3615 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013905503645922628, + "loss": 0.1657, + "step": 3620 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013889389330137264, + "loss": 0.1557, + "step": 3625 + }, + { + "epoch": 1.18, + "learning_rate": 0.00013873263105809388, + "loss": 0.1551, + "step": 3630 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001385712502231432, + "loss": 0.157, + "step": 3635 + }, + { + "epoch": 1.18, + "learning_rate": 0.00013840975129063684, + "loss": 0.1548, + "step": 3640 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001382481347550526, + "loss": 0.1565, + "step": 3645 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001380864011112285, + "loss": 0.1497, + "step": 3650 + }, + { + "epoch": 1.18, + "learning_rate": 0.000137924550854361, + "loss": 0.1569, + "step": 3655 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001377625844800037, + "loss": 0.1582, + "step": 3660 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013760050248406557, + "loss": 0.1571, + "step": 3665 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001374383053628098, + "loss": 0.1524, + "step": 3670 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013727599361285187, + "loss": 0.1567, + "step": 3675 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013711356773115838, + "loss": 0.1548, + "step": 3680 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013695102821504533, + "loss": 0.1557, + "step": 3685 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001367883755621766, + "loss": 0.148, + "step": 3690 + }, + { + "epoch": 1.2, + "learning_rate": 0.00013662561027056252, + "loss": 0.1604, + "step": 3695 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001364627328385583, + "loss": 0.15, + "step": 3700 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001362997437648625, + "loss": 0.1476, + "step": 3705 + }, + { + "epoch": 1.2, + "learning_rate": 0.00013613664354851547, + "loss": 0.1586, + "step": 3710 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001359734326888979, + "loss": 0.1536, + "step": 3715 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013581011168572923, + "loss": 0.1564, + "step": 3720 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013564668103906616, + "loss": 0.1534, + "step": 3725 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013548314124930104, + "loss": 0.1521, + "step": 3730 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013531949281716044, + "loss": 0.1489, + "step": 3735 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013515573624370357, + "loss": 0.1528, + "step": 3740 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001349918720303207, + "loss": 0.1405, + "step": 3745 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013482790067873178, + "loss": 0.1457, + "step": 3750 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013466382269098468, + "loss": 0.1554, + "step": 3755 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013449963856945383, + "loss": 0.1478, + "step": 3760 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013433534881683858, + "loss": 0.1633, + "step": 3765 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013417095393616178, + "loss": 0.15, + "step": 3770 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013400645443076804, + "loss": 0.1552, + "step": 3775 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013384185080432244, + "loss": 0.1485, + "step": 3780 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001336771435608088, + "loss": 0.1531, + "step": 3785 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013351233320452812, + "loss": 0.158, + "step": 3790 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013334742024009725, + "loss": 0.1554, + "step": 3795 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013318240517244713, + "loss": 0.1537, + "step": 3800 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001330172885068213, + "loss": 0.1584, + "step": 3805 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013285207074877447, + "loss": 0.1612, + "step": 3810 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013268675240417073, + "loss": 0.1449, + "step": 3815 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013252133397918226, + "loss": 0.1456, + "step": 3820 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001323558159802876, + "loss": 0.1541, + "step": 3825 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013219019891427025, + "loss": 0.1492, + "step": 3830 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013202448328821694, + "loss": 0.1528, + "step": 3835 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013185866960951622, + "loss": 0.1516, + "step": 3840 + }, + { + "epoch": 1.25, + "learning_rate": 0.0001316927583858569, + "loss": 0.1484, + "step": 3845 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013152675012522629, + "loss": 0.1509, + "step": 3850 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013136064533590897, + "loss": 0.1516, + "step": 3855 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013119444452648504, + "loss": 0.15, + "step": 3860 + }, + { + "epoch": 1.25, + "learning_rate": 0.0001310281482058286, + "loss": 0.1571, + "step": 3865 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013086175688310612, + "loss": 0.1526, + "step": 3870 + }, + { + "epoch": 1.26, + "learning_rate": 0.000130695271067775, + "loss": 0.1445, + "step": 3875 + }, + { + "epoch": 1.26, + "learning_rate": 0.00013052869126958198, + "loss": 0.1598, + "step": 3880 + }, + { + "epoch": 1.26, + "learning_rate": 0.00013036201799856149, + "loss": 0.1499, + "step": 3885 + }, + { + "epoch": 1.26, + "learning_rate": 0.0001301952517650342, + "loss": 0.1507, + "step": 3890 + }, + { + "epoch": 1.26, + "learning_rate": 0.0001300283930796054, + "loss": 0.1489, + "step": 3895 + }, + { + "epoch": 1.26, + "learning_rate": 0.00012986144245316346, + "loss": 0.1477, + "step": 3900 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012969440039687822, + "loss": 0.1528, + "step": 3905 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012952726742219953, + "loss": 0.1514, + "step": 3910 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012936004404085557, + "loss": 0.1482, + "step": 3915 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012919273076485133, + "loss": 0.1558, + "step": 3920 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012902532810646697, + "loss": 0.1539, + "step": 3925 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012885783657825647, + "loss": 0.1479, + "step": 3930 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012869025669304585, + "loss": 0.1449, + "step": 3935 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012852258896393158, + "loss": 0.1489, + "step": 3940 + }, + { + "epoch": 1.28, + "learning_rate": 0.0001283548339042792, + "loss": 0.1479, + "step": 3945 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012818699202772152, + "loss": 0.149, + "step": 3950 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012801906384815733, + "loss": 0.1472, + "step": 3955 + }, + { + "epoch": 1.28, + "learning_rate": 0.0001278510498797495, + "loss": 0.1526, + "step": 3960 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012768295063692376, + "loss": 0.1469, + "step": 3965 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012751476663436668, + "loss": 0.1576, + "step": 3970 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012734649838702455, + "loss": 0.1583, + "step": 3975 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012717814641010153, + "loss": 0.1434, + "step": 3980 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012700971121905812, + "loss": 0.1463, + "step": 3985 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012684119332960963, + "loss": 0.1543, + "step": 3990 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012667259325772464, + "loss": 0.1501, + "step": 3995 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012650391151962324, + "loss": 0.1475, + "step": 4000 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012633514863177566, + "loss": 0.147, + "step": 4005 + }, + { + "epoch": 1.3, + "learning_rate": 0.0001261663051109005, + "loss": 0.1454, + "step": 4010 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012599738147396332, + "loss": 0.1459, + "step": 4015 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012582837823817497, + "loss": 0.1479, + "step": 4020 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012565929592099002, + "loss": 0.1541, + "step": 4025 + }, + { + "epoch": 1.31, + "learning_rate": 0.00012549013504010512, + "loss": 0.145, + "step": 4030 + }, + { + "epoch": 1.31, + "learning_rate": 0.0001253208961134575, + "loss": 0.1467, + "step": 4035 + }, + { + "epoch": 1.31, + "learning_rate": 0.0001251515796592234, + "loss": 0.1497, + "step": 4040 + }, + { + "epoch": 1.31, + "learning_rate": 0.00012498218619581632, + "loss": 0.1439, + "step": 4045 + }, + { + "epoch": 1.31, + "learning_rate": 0.00012481271624188568, + "loss": 0.1419, + "step": 4050 + }, + { + "epoch": 1.31, + "learning_rate": 0.00012464317031631494, + "loss": 0.1518, + "step": 4055 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012447354893822034, + "loss": 0.1462, + "step": 4060 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012430385262694906, + "loss": 0.1473, + "step": 4065 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012413408190207768, + "loss": 0.1495, + "step": 4070 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012396423728341067, + "loss": 0.1566, + "step": 4075 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012379431929097873, + "loss": 0.1507, + "step": 4080 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012362432844503725, + "loss": 0.1544, + "step": 4085 + }, + { + "epoch": 1.33, + "learning_rate": 0.0001234542652660646, + "loss": 0.1528, + "step": 4090 + }, + { + "epoch": 1.33, + "learning_rate": 0.00012328413027476066, + "loss": 0.153, + "step": 4095 + }, + { + "epoch": 1.33, + "learning_rate": 0.00012311392399204526, + "loss": 0.1438, + "step": 4100 + }, + { + "epoch": 1.33, + "learning_rate": 0.0001229436469390564, + "loss": 0.1497, + "step": 4105 + }, + { + "epoch": 1.33, + "learning_rate": 0.00012277329963714882, + "loss": 0.1471, + "step": 4110 + }, + { + "epoch": 1.33, + "learning_rate": 0.0001226028826078923, + "loss": 0.1442, + "step": 4115 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012243239637307023, + "loss": 0.1455, + "step": 4120 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012226184145467775, + "loss": 0.1509, + "step": 4125 + }, + { + "epoch": 1.34, + "learning_rate": 0.0001220912183749204, + "loss": 0.1504, + "step": 4130 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012192052765621234, + "loss": 0.1524, + "step": 4135 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012174976982117488, + "loss": 0.1526, + "step": 4140 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012157894539263482, + "loss": 0.1501, + "step": 4145 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012140805489362281, + "loss": 0.1464, + "step": 4150 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012123709884737189, + "loss": 0.1481, + "step": 4155 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012106607777731567, + "loss": 0.1483, + "step": 4160 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012089499220708696, + "loss": 0.1456, + "step": 4165 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012072384266051595, + "loss": 0.1479, + "step": 4170 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012055262966162883, + "loss": 0.1488, + "step": 4175 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012038135373464602, + "loss": 0.1468, + "step": 4180 + }, + { + "epoch": 1.36, + "learning_rate": 0.00012021001540398058, + "loss": 0.1412, + "step": 4185 + }, + { + "epoch": 1.36, + "learning_rate": 0.00012003861519423662, + "loss": 0.1503, + "step": 4190 + }, + { + "epoch": 1.36, + "learning_rate": 0.00011986715363020784, + "loss": 0.1497, + "step": 4195 + }, + { + "epoch": 1.36, + "learning_rate": 0.00011969563123687566, + "loss": 0.1457, + "step": 4200 + }, + { + "epoch": 1.36, + "learning_rate": 0.00011952404853940782, + "loss": 0.1469, + "step": 4205 + }, + { + "epoch": 1.36, + "learning_rate": 0.00011935240606315666, + "loss": 0.1485, + "step": 4210 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011918070433365754, + "loss": 0.1413, + "step": 4215 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011900894387662731, + "loss": 0.1565, + "step": 4220 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011883712521796253, + "loss": 0.1516, + "step": 4225 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011866524888373811, + "loss": 0.146, + "step": 4230 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011849331540020543, + "loss": 0.1507, + "step": 4235 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011832132529379088, + "loss": 0.1493, + "step": 4240 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011814927909109419, + "loss": 0.1419, + "step": 4245 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011797717731888691, + "loss": 0.1414, + "step": 4250 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011780502050411069, + "loss": 0.1385, + "step": 4255 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011763280917387572, + "loss": 0.1473, + "step": 4260 + }, + { + "epoch": 1.38, + "learning_rate": 0.0001174605438554591, + "loss": 0.1496, + "step": 4265 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011728822507630326, + "loss": 0.1369, + "step": 4270 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011711585336401429, + "loss": 0.1437, + "step": 4275 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011694342924636031, + "loss": 0.1564, + "step": 4280 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011677095325126996, + "loss": 0.1443, + "step": 4285 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011659842590683074, + "loss": 0.1577, + "step": 4290 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011642584774128728, + "loss": 0.1416, + "step": 4295 + }, + { + "epoch": 1.39, + "learning_rate": 0.0001162532192830399, + "loss": 0.145, + "step": 4300 + }, + { + "epoch": 1.4, + "learning_rate": 0.00011608054106064281, + "loss": 0.1458, + "step": 4305 + }, + { + "epoch": 1.4, + "learning_rate": 0.0001159078136028027, + "loss": 0.1445, + "step": 4310 + }, + { + "epoch": 1.4, + "learning_rate": 0.00011573503743837693, + "loss": 0.139, + "step": 4315 + }, + { + "epoch": 1.4, + "learning_rate": 0.00011556221309637203, + "loss": 0.1441, + "step": 4320 + }, + { + "epoch": 1.4, + "learning_rate": 0.000115389341105942, + "loss": 0.1567, + "step": 4325 + }, + { + "epoch": 1.4, + "learning_rate": 0.0001152164219963868, + "loss": 0.1499, + "step": 4330 + }, + { + "epoch": 1.4, + "learning_rate": 0.00011504345629715057, + "loss": 0.15, + "step": 4335 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011487044453782016, + "loss": 0.1449, + "step": 4340 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011469738724812346, + "loss": 0.1423, + "step": 4345 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011452428495792773, + "loss": 0.1468, + "step": 4350 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011435113819723797, + "loss": 0.1519, + "step": 4355 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011417794749619544, + "loss": 0.1498, + "step": 4360 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011400471338507585, + "loss": 0.1406, + "step": 4365 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011383143639428791, + "loss": 0.1485, + "step": 4370 + }, + { + "epoch": 1.42, + "learning_rate": 0.0001136581170543715, + "loss": 0.1502, + "step": 4375 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011348475589599625, + "loss": 0.139, + "step": 4380 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011331135344995985, + "loss": 0.1485, + "step": 4385 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011313791024718634, + "loss": 0.133, + "step": 4390 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011296442681872457, + "loss": 0.1473, + "step": 4395 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011279090369574658, + "loss": 0.1401, + "step": 4400 + }, + { + "epoch": 1.43, + "learning_rate": 0.0001126173414095459, + "loss": 0.1483, + "step": 4405 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011244374049153603, + "loss": 0.1436, + "step": 4410 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011227010147324868, + "loss": 0.1445, + "step": 4415 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011209642488633227, + "loss": 0.1451, + "step": 4420 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011192271126255024, + "loss": 0.1438, + "step": 4425 + }, + { + "epoch": 1.44, + "learning_rate": 0.00011174896113377945, + "loss": 0.1474, + "step": 4430 + }, + { + "epoch": 1.44, + "learning_rate": 0.0001115751750320084, + "loss": 0.1444, + "step": 4435 + }, + { + "epoch": 1.44, + "learning_rate": 0.00011140135348933593, + "loss": 0.147, + "step": 4440 + }, + { + "epoch": 1.44, + "learning_rate": 0.00011122749703796929, + "loss": 0.1447, + "step": 4445 + }, + { + "epoch": 1.44, + "learning_rate": 0.0001110536062102226, + "loss": 0.1423, + "step": 4450 + }, + { + "epoch": 1.44, + "learning_rate": 0.00011087968153851525, + "loss": 0.1405, + "step": 4455 + }, + { + "epoch": 1.45, + "learning_rate": 0.00011070572355537026, + "loss": 0.1468, + "step": 4460 + }, + { + "epoch": 1.45, + "learning_rate": 0.00011053173279341265, + "loss": 0.1442, + "step": 4465 + }, + { + "epoch": 1.45, + "learning_rate": 0.0001103577097853678, + "loss": 0.1447, + "step": 4470 + }, + { + "epoch": 1.45, + "learning_rate": 0.00011018365506405978, + "loss": 0.1514, + "step": 4475 + }, + { + "epoch": 1.45, + "learning_rate": 0.00011000956916240985, + "loss": 0.148, + "step": 4480 + }, + { + "epoch": 1.45, + "learning_rate": 0.00010983545261343464, + "loss": 0.1499, + "step": 4485 + }, + { + "epoch": 1.45, + "learning_rate": 0.00010966130595024466, + "loss": 0.1398, + "step": 4490 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010948712970604266, + "loss": 0.1379, + "step": 4495 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010931292441412191, + "loss": 0.1438, + "step": 4500 + }, + { + "epoch": 1.46, + "learning_rate": 0.0001091386906078646, + "loss": 0.1453, + "step": 4505 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010896442882074029, + "loss": 0.1439, + "step": 4510 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010879013958630418, + "loss": 0.1436, + "step": 4515 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010861582343819548, + "loss": 0.144, + "step": 4520 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010844148091013581, + "loss": 0.1439, + "step": 4525 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010826711253592768, + "loss": 0.1441, + "step": 4530 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010809271884945256, + "loss": 0.1421, + "step": 4535 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010791830038466952, + "loss": 0.1476, + "step": 4540 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010774385767561347, + "loss": 0.1484, + "step": 4545 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010756939125639358, + "loss": 0.1437, + "step": 4550 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010739490166119155, + "loss": 0.1403, + "step": 4555 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010722038942426014, + "loss": 0.1437, + "step": 4560 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010704585507992131, + "loss": 0.1429, + "step": 4565 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010687129916256482, + "loss": 0.1401, + "step": 4570 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010669672220664644, + "loss": 0.1448, + "step": 4575 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010652212474668635, + "loss": 0.1449, + "step": 4580 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010634750731726748, + "loss": 0.1431, + "step": 4585 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010617287045303395, + "loss": 0.1467, + "step": 4590 + }, + { + "epoch": 1.49, + "learning_rate": 0.0001059982146886894, + "loss": 0.1403, + "step": 4595 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010582354055899528, + "loss": 0.1467, + "step": 4600 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010564884859876934, + "loss": 0.1399, + "step": 4605 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010547413934288386, + "loss": 0.1408, + "step": 4610 + }, + { + "epoch": 1.5, + "learning_rate": 0.0001052994133262641, + "loss": 0.1411, + "step": 4615 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010512467108388661, + "loss": 0.1398, + "step": 4620 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010494991315077772, + "loss": 0.1444, + "step": 4625 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010477514006201166, + "loss": 0.1438, + "step": 4630 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010460035235270918, + "loss": 0.1499, + "step": 4635 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010442555055803572, + "loss": 0.1451, + "step": 4640 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010425073521319986, + "loss": 0.15, + "step": 4645 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010407590685345174, + "loss": 0.1471, + "step": 4650 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010390106601408125, + "loss": 0.1454, + "step": 4655 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010372621323041656, + "loss": 0.1476, + "step": 4660 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010355134903782235, + "loss": 0.142, + "step": 4665 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010337647397169832, + "loss": 0.1391, + "step": 4670 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010320158856747737, + "loss": 0.1398, + "step": 4675 + }, + { + "epoch": 1.52, + "learning_rate": 0.0001030266933606241, + "loss": 0.15, + "step": 4680 + }, + { + "epoch": 1.52, + "learning_rate": 0.0001028517888866331, + "loss": 0.1387, + "step": 4685 + }, + { + "epoch": 1.52, + "learning_rate": 0.00010267687568102737, + "loss": 0.1484, + "step": 4690 + }, + { + "epoch": 1.52, + "learning_rate": 0.00010250195427935663, + "loss": 0.1483, + "step": 4695 + }, + { + "epoch": 1.52, + "learning_rate": 0.00010232702521719564, + "loss": 0.1452, + "step": 4700 + }, + { + "epoch": 1.52, + "learning_rate": 0.00010215208903014272, + "loss": 0.1481, + "step": 4705 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010197714625381795, + "loss": 0.1406, + "step": 4710 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010180219742386157, + "loss": 0.139, + "step": 4715 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010162724307593234, + "loss": 0.1354, + "step": 4720 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010145228374570599, + "loss": 0.1379, + "step": 4725 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010127731996887339, + "loss": 0.1481, + "step": 4730 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010110235228113918, + "loss": 0.1464, + "step": 4735 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010092738121821985, + "loss": 0.1426, + "step": 4740 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010075240731584223, + "loss": 0.1415, + "step": 4745 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010057743110974194, + "loss": 0.1393, + "step": 4750 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010040245313566153, + "loss": 0.1408, + "step": 4755 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010022747392934905, + "loss": 0.1358, + "step": 4760 + }, + { + "epoch": 1.54, + "learning_rate": 0.0001000524940265563, + "loss": 0.1401, + "step": 4765 + }, + { + "epoch": 1.55, + "learning_rate": 9.98775139630372e-05, + "loss": 0.1379, + "step": 4770 + }, + { + "epoch": 1.55, + "learning_rate": 9.970253427454616e-05, + "loss": 0.1425, + "step": 4775 + }, + { + "epoch": 1.55, + "learning_rate": 9.95275554968365e-05, + "loss": 0.1475, + "step": 4780 + }, + { + "epoch": 1.55, + "learning_rate": 9.935257816565861e-05, + "loss": 0.1364, + "step": 4785 + }, + { + "epoch": 1.55, + "learning_rate": 9.917760281675866e-05, + "loss": 0.1443, + "step": 4790 + }, + { + "epoch": 1.55, + "learning_rate": 9.900262998587653e-05, + "loss": 0.1388, + "step": 4795 + }, + { + "epoch": 1.56, + "learning_rate": 9.882766020874456e-05, + "loss": 0.1375, + "step": 4800 + }, + { + "epoch": 1.56, + "learning_rate": 9.865269402108565e-05, + "loss": 0.1413, + "step": 4805 + }, + { + "epoch": 1.56, + "learning_rate": 9.847773195861175e-05, + "loss": 0.1499, + "step": 4810 + }, + { + "epoch": 1.56, + "learning_rate": 9.830277455702213e-05, + "loss": 0.1421, + "step": 4815 + }, + { + "epoch": 1.56, + "learning_rate": 9.812782235200189e-05, + "loss": 0.146, + "step": 4820 + }, + { + "epoch": 1.56, + "learning_rate": 9.795287587922006e-05, + "loss": 0.1343, + "step": 4825 + }, + { + "epoch": 1.57, + "learning_rate": 9.777793567432828e-05, + "loss": 0.1451, + "step": 4830 + }, + { + "epoch": 1.57, + "learning_rate": 9.760300227295894e-05, + "loss": 0.1438, + "step": 4835 + }, + { + "epoch": 1.57, + "learning_rate": 9.742807621072353e-05, + "loss": 0.1436, + "step": 4840 + }, + { + "epoch": 1.57, + "learning_rate": 9.725315802321121e-05, + "loss": 0.1384, + "step": 4845 + }, + { + "epoch": 1.57, + "learning_rate": 9.707824824598684e-05, + "loss": 0.1368, + "step": 4850 + }, + { + "epoch": 1.57, + "learning_rate": 9.690334741458974e-05, + "loss": 0.1309, + "step": 4855 + }, + { + "epoch": 1.57, + "learning_rate": 9.672845606453168e-05, + "loss": 0.1384, + "step": 4860 + }, + { + "epoch": 1.58, + "learning_rate": 9.655357473129548e-05, + "loss": 0.1387, + "step": 4865 + }, + { + "epoch": 1.58, + "learning_rate": 9.637870395033324e-05, + "loss": 0.1426, + "step": 4870 + }, + { + "epoch": 1.58, + "learning_rate": 9.620384425706484e-05, + "loss": 0.1407, + "step": 4875 + }, + { + "epoch": 1.58, + "learning_rate": 9.602899618687606e-05, + "loss": 0.1413, + "step": 4880 + }, + { + "epoch": 1.58, + "learning_rate": 9.585416027511727e-05, + "loss": 0.1434, + "step": 4885 + }, + { + "epoch": 1.58, + "learning_rate": 9.567933705710143e-05, + "loss": 0.146, + "step": 4890 + }, + { + "epoch": 1.59, + "learning_rate": 9.550452706810281e-05, + "loss": 0.1384, + "step": 4895 + }, + { + "epoch": 1.59, + "learning_rate": 9.532973084335511e-05, + "loss": 0.1447, + "step": 4900 + }, + { + "epoch": 1.59, + "learning_rate": 9.515494891804979e-05, + "loss": 0.1418, + "step": 4905 + }, + { + "epoch": 1.59, + "learning_rate": 9.498018182733466e-05, + "loss": 0.144, + "step": 4910 + }, + { + "epoch": 1.59, + "learning_rate": 9.480543010631207e-05, + "loss": 0.1449, + "step": 4915 + }, + { + "epoch": 1.59, + "learning_rate": 9.463069429003728e-05, + "loss": 0.1391, + "step": 4920 + }, + { + "epoch": 1.6, + "learning_rate": 9.445597491351685e-05, + "loss": 0.1484, + "step": 4925 + }, + { + "epoch": 1.6, + "learning_rate": 9.42812725117071e-05, + "loss": 0.1402, + "step": 4930 + }, + { + "epoch": 1.6, + "learning_rate": 9.410658761951221e-05, + "loss": 0.1478, + "step": 4935 + }, + { + "epoch": 1.6, + "learning_rate": 9.393192077178295e-05, + "loss": 0.1431, + "step": 4940 + }, + { + "epoch": 1.6, + "learning_rate": 9.375727250331462e-05, + "loss": 0.1405, + "step": 4945 + }, + { + "epoch": 1.6, + "learning_rate": 9.358264334884585e-05, + "loss": 0.1359, + "step": 4950 + }, + { + "epoch": 1.61, + "learning_rate": 9.340803384305657e-05, + "loss": 0.1464, + "step": 4955 + }, + { + "epoch": 1.61, + "learning_rate": 9.323344452056668e-05, + "loss": 0.1408, + "step": 4960 + }, + { + "epoch": 1.61, + "learning_rate": 9.305887591593419e-05, + "loss": 0.1435, + "step": 4965 + }, + { + "epoch": 1.61, + "learning_rate": 9.288432856365374e-05, + "loss": 0.1449, + "step": 4970 + }, + { + "epoch": 1.61, + "learning_rate": 9.270980299815486e-05, + "loss": 0.1403, + "step": 4975 + }, + { + "epoch": 1.61, + "learning_rate": 9.253529975380039e-05, + "loss": 0.1434, + "step": 4980 + }, + { + "epoch": 1.62, + "learning_rate": 9.236081936488485e-05, + "loss": 0.1405, + "step": 4985 + }, + { + "epoch": 1.62, + "learning_rate": 9.21863623656327e-05, + "loss": 0.1394, + "step": 4990 + }, + { + "epoch": 1.62, + "learning_rate": 9.20119292901969e-05, + "loss": 0.1352, + "step": 4995 + }, + { + "epoch": 1.62, + "learning_rate": 9.183752067265703e-05, + "loss": 0.1452, + "step": 5000 + }, + { + "epoch": 1.62, + "learning_rate": 9.166313704701795e-05, + "loss": 0.1423, + "step": 5005 + }, + { + "epoch": 1.62, + "learning_rate": 9.14887789472078e-05, + "loss": 0.1378, + "step": 5010 + }, + { + "epoch": 1.63, + "learning_rate": 9.131444690707673e-05, + "loss": 0.1379, + "step": 5015 + }, + { + "epoch": 1.63, + "learning_rate": 9.114014146039503e-05, + "loss": 0.137, + "step": 5020 + }, + { + "epoch": 1.63, + "learning_rate": 9.096586314085162e-05, + "loss": 0.1459, + "step": 5025 + }, + { + "epoch": 1.63, + "learning_rate": 9.079161248205222e-05, + "loss": 0.1417, + "step": 5030 + }, + { + "epoch": 1.63, + "learning_rate": 9.061739001751803e-05, + "loss": 0.139, + "step": 5035 + }, + { + "epoch": 1.63, + "learning_rate": 9.044319628068387e-05, + "loss": 0.1322, + "step": 5040 + }, + { + "epoch": 1.63, + "learning_rate": 9.02690318048965e-05, + "loss": 0.1324, + "step": 5045 + }, + { + "epoch": 1.64, + "learning_rate": 9.009489712341327e-05, + "loss": 0.1403, + "step": 5050 + }, + { + "epoch": 1.64, + "learning_rate": 8.992079276940011e-05, + "loss": 0.1394, + "step": 5055 + }, + { + "epoch": 1.64, + "learning_rate": 8.974671927593028e-05, + "loss": 0.1389, + "step": 5060 + }, + { + "epoch": 1.64, + "learning_rate": 8.957267717598236e-05, + "loss": 0.141, + "step": 5065 + }, + { + "epoch": 1.64, + "learning_rate": 8.939866700243901e-05, + "loss": 0.1334, + "step": 5070 + }, + { + "epoch": 1.64, + "learning_rate": 8.922468928808497e-05, + "loss": 0.1311, + "step": 5075 + }, + { + "epoch": 1.65, + "learning_rate": 8.905074456560574e-05, + "loss": 0.1439, + "step": 5080 + }, + { + "epoch": 1.65, + "learning_rate": 8.887683336758563e-05, + "loss": 0.1424, + "step": 5085 + }, + { + "epoch": 1.65, + "learning_rate": 8.870295622650651e-05, + "loss": 0.1476, + "step": 5090 + }, + { + "epoch": 1.65, + "learning_rate": 8.852911367474577e-05, + "loss": 0.1436, + "step": 5095 + }, + { + "epoch": 1.65, + "learning_rate": 8.835530624457503e-05, + "loss": 0.1442, + "step": 5100 + }, + { + "epoch": 1.65, + "learning_rate": 8.81815344681584e-05, + "loss": 0.1467, + "step": 5105 + }, + { + "epoch": 1.66, + "learning_rate": 8.800779887755066e-05, + "loss": 0.1409, + "step": 5110 + }, + { + "epoch": 1.66, + "learning_rate": 8.783410000469601e-05, + "loss": 0.1487, + "step": 5115 + }, + { + "epoch": 1.66, + "learning_rate": 8.7660438381426e-05, + "loss": 0.1463, + "step": 5120 + }, + { + "epoch": 1.66, + "learning_rate": 8.748681453945833e-05, + "loss": 0.1398, + "step": 5125 + }, + { + "epoch": 1.66, + "learning_rate": 8.73132290103949e-05, + "loss": 0.1358, + "step": 5130 + }, + { + "epoch": 1.66, + "learning_rate": 8.713968232572039e-05, + "loss": 0.143, + "step": 5135 + }, + { + "epoch": 1.67, + "learning_rate": 8.69661750168004e-05, + "loss": 0.137, + "step": 5140 + }, + { + "epoch": 1.67, + "learning_rate": 8.679270761488019e-05, + "loss": 0.1393, + "step": 5145 + }, + { + "epoch": 1.67, + "learning_rate": 8.661928065108259e-05, + "loss": 0.1472, + "step": 5150 + }, + { + "epoch": 1.67, + "learning_rate": 8.644589465640684e-05, + "loss": 0.1389, + "step": 5155 + }, + { + "epoch": 1.67, + "learning_rate": 8.627255016172652e-05, + "loss": 0.1487, + "step": 5160 + }, + { + "epoch": 1.67, + "learning_rate": 8.609924769778835e-05, + "loss": 0.1403, + "step": 5165 + }, + { + "epoch": 1.68, + "learning_rate": 8.59259877952103e-05, + "loss": 0.1351, + "step": 5170 + }, + { + "epoch": 1.68, + "learning_rate": 8.575277098447987e-05, + "loss": 0.1406, + "step": 5175 + }, + { + "epoch": 1.68, + "learning_rate": 8.557959779595286e-05, + "loss": 0.1378, + "step": 5180 + }, + { + "epoch": 1.68, + "learning_rate": 8.540646875985134e-05, + "loss": 0.1352, + "step": 5185 + }, + { + "epoch": 1.68, + "learning_rate": 8.523338440626232e-05, + "loss": 0.1406, + "step": 5190 + }, + { + "epoch": 1.68, + "learning_rate": 8.50603452651358e-05, + "loss": 0.1432, + "step": 5195 + }, + { + "epoch": 1.69, + "learning_rate": 8.488735186628364e-05, + "loss": 0.1433, + "step": 5200 + }, + { + "epoch": 1.69, + "learning_rate": 8.471440473937732e-05, + "loss": 0.1368, + "step": 5205 + }, + { + "epoch": 1.69, + "learning_rate": 8.454150441394697e-05, + "loss": 0.136, + "step": 5210 + }, + { + "epoch": 1.69, + "learning_rate": 8.436865141937912e-05, + "loss": 0.1502, + "step": 5215 + }, + { + "epoch": 1.69, + "learning_rate": 8.419584628491562e-05, + "loss": 0.1395, + "step": 5220 + }, + { + "epoch": 1.69, + "learning_rate": 8.402308953965165e-05, + "loss": 0.1294, + "step": 5225 + }, + { + "epoch": 1.69, + "learning_rate": 8.385038171253424e-05, + "loss": 0.1438, + "step": 5230 + }, + { + "epoch": 1.7, + "learning_rate": 8.367772333236073e-05, + "loss": 0.1371, + "step": 5235 + }, + { + "epoch": 1.7, + "learning_rate": 8.350511492777698e-05, + "loss": 0.138, + "step": 5240 + }, + { + "epoch": 1.7, + "learning_rate": 8.333255702727588e-05, + "loss": 0.1389, + "step": 5245 + }, + { + "epoch": 1.7, + "learning_rate": 8.316005015919559e-05, + "loss": 0.1384, + "step": 5250 + }, + { + "epoch": 1.7, + "learning_rate": 8.298759485171819e-05, + "loss": 0.1384, + "step": 5255 + }, + { + "epoch": 1.7, + "learning_rate": 8.281519163286772e-05, + "loss": 0.1307, + "step": 5260 + }, + { + "epoch": 1.71, + "learning_rate": 8.264284103050888e-05, + "loss": 0.1413, + "step": 5265 + }, + { + "epoch": 1.71, + "learning_rate": 8.247054357234511e-05, + "loss": 0.1322, + "step": 5270 + }, + { + "epoch": 1.71, + "learning_rate": 8.229829978591729e-05, + "loss": 0.1433, + "step": 5275 + }, + { + "epoch": 1.71, + "learning_rate": 8.212611019860188e-05, + "loss": 0.1384, + "step": 5280 + }, + { + "epoch": 1.71, + "learning_rate": 8.195397533760943e-05, + "loss": 0.1396, + "step": 5285 + }, + { + "epoch": 1.71, + "learning_rate": 8.178189572998284e-05, + "loss": 0.1403, + "step": 5290 + }, + { + "epoch": 1.72, + "learning_rate": 8.160987190259605e-05, + "loss": 0.1429, + "step": 5295 + }, + { + "epoch": 1.72, + "learning_rate": 8.14379043821519e-05, + "loss": 0.1384, + "step": 5300 + }, + { + "epoch": 1.72, + "learning_rate": 8.12659936951811e-05, + "loss": 0.1422, + "step": 5305 + }, + { + "epoch": 1.72, + "learning_rate": 8.109414036804026e-05, + "loss": 0.1355, + "step": 5310 + }, + { + "epoch": 1.72, + "learning_rate": 8.092234492691027e-05, + "loss": 0.1357, + "step": 5315 + }, + { + "epoch": 1.72, + "learning_rate": 8.075060789779494e-05, + "loss": 0.1388, + "step": 5320 + }, + { + "epoch": 1.73, + "learning_rate": 8.057892980651909e-05, + "loss": 0.1349, + "step": 5325 + }, + { + "epoch": 1.73, + "learning_rate": 8.040731117872721e-05, + "loss": 0.1328, + "step": 5330 + }, + { + "epoch": 1.73, + "learning_rate": 8.023575253988165e-05, + "loss": 0.1301, + "step": 5335 + }, + { + "epoch": 1.73, + "learning_rate": 8.00642544152611e-05, + "loss": 0.1376, + "step": 5340 + }, + { + "epoch": 1.73, + "learning_rate": 7.989281732995895e-05, + "loss": 0.1389, + "step": 5345 + }, + { + "epoch": 1.73, + "learning_rate": 7.972144180888177e-05, + "loss": 0.1337, + "step": 5350 + }, + { + "epoch": 1.74, + "learning_rate": 7.955012837674753e-05, + "loss": 0.141, + "step": 5355 + }, + { + "epoch": 1.74, + "learning_rate": 7.93788775580842e-05, + "loss": 0.1456, + "step": 5360 + }, + { + "epoch": 1.74, + "learning_rate": 7.920768987722791e-05, + "loss": 0.1459, + "step": 5365 + }, + { + "epoch": 1.74, + "learning_rate": 7.903656585832162e-05, + "loss": 0.1332, + "step": 5370 + }, + { + "epoch": 1.74, + "learning_rate": 7.886550602531331e-05, + "loss": 0.1445, + "step": 5375 + }, + { + "epoch": 1.74, + "learning_rate": 7.869451090195437e-05, + "loss": 0.1349, + "step": 5380 + }, + { + "epoch": 1.75, + "learning_rate": 7.852358101179817e-05, + "loss": 0.1327, + "step": 5385 + }, + { + "epoch": 1.75, + "learning_rate": 7.835271687819827e-05, + "loss": 0.1305, + "step": 5390 + }, + { + "epoch": 1.75, + "learning_rate": 7.818191902430696e-05, + "loss": 0.1446, + "step": 5395 + }, + { + "epoch": 1.75, + "learning_rate": 7.801118797307349e-05, + "loss": 0.139, + "step": 5400 + }, + { + "epoch": 1.75, + "learning_rate": 7.784052424724276e-05, + "loss": 0.1432, + "step": 5405 + }, + { + "epoch": 1.75, + "learning_rate": 7.766992836935327e-05, + "loss": 0.1439, + "step": 5410 + }, + { + "epoch": 1.75, + "learning_rate": 7.749940086173607e-05, + "loss": 0.1415, + "step": 5415 + }, + { + "epoch": 1.76, + "learning_rate": 7.732894224651261e-05, + "loss": 0.1404, + "step": 5420 + }, + { + "epoch": 1.76, + "learning_rate": 7.715855304559362e-05, + "loss": 0.1415, + "step": 5425 + }, + { + "epoch": 1.76, + "learning_rate": 7.698823378067714e-05, + "loss": 0.142, + "step": 5430 + }, + { + "epoch": 1.76, + "learning_rate": 7.681798497324716e-05, + "loss": 0.1428, + "step": 5435 + }, + { + "epoch": 1.76, + "learning_rate": 7.664780714457198e-05, + "loss": 0.1405, + "step": 5440 + }, + { + "epoch": 1.76, + "learning_rate": 7.647770081570246e-05, + "loss": 0.1355, + "step": 5445 + }, + { + "epoch": 1.77, + "learning_rate": 7.630766650747066e-05, + "loss": 0.1371, + "step": 5450 + }, + { + "epoch": 1.77, + "learning_rate": 7.613770474048802e-05, + "loss": 0.1388, + "step": 5455 + }, + { + "epoch": 1.77, + "learning_rate": 7.596781603514398e-05, + "loss": 0.1306, + "step": 5460 + }, + { + "epoch": 1.77, + "learning_rate": 7.579800091160417e-05, + "loss": 0.1351, + "step": 5465 + }, + { + "epoch": 1.77, + "learning_rate": 7.562825988980904e-05, + "loss": 0.1322, + "step": 5470 + }, + { + "epoch": 1.77, + "learning_rate": 7.545859348947204e-05, + "loss": 0.1389, + "step": 5475 + }, + { + "epoch": 1.78, + "learning_rate": 7.528900223007824e-05, + "loss": 0.1432, + "step": 5480 + }, + { + "epoch": 1.78, + "learning_rate": 7.511948663088261e-05, + "loss": 0.1359, + "step": 5485 + }, + { + "epoch": 1.78, + "learning_rate": 7.495004721090843e-05, + "loss": 0.1411, + "step": 5490 + }, + { + "epoch": 1.78, + "learning_rate": 7.478068448894577e-05, + "loss": 0.1373, + "step": 5495 + }, + { + "epoch": 1.78, + "learning_rate": 7.461139898354986e-05, + "loss": 0.1406, + "step": 5500 + }, + { + "epoch": 1.78, + "learning_rate": 7.444219121303957e-05, + "loss": 0.1388, + "step": 5505 + }, + { + "epoch": 1.79, + "learning_rate": 7.42730616954956e-05, + "loss": 0.1344, + "step": 5510 + }, + { + "epoch": 1.79, + "learning_rate": 7.410401094875921e-05, + "loss": 0.1416, + "step": 5515 + }, + { + "epoch": 1.79, + "learning_rate": 7.393503949043039e-05, + "loss": 0.1417, + "step": 5520 + }, + { + "epoch": 1.79, + "learning_rate": 7.376614783786646e-05, + "loss": 0.1425, + "step": 5525 + }, + { + "epoch": 1.79, + "learning_rate": 7.359733650818024e-05, + "loss": 0.1326, + "step": 5530 + }, + { + "epoch": 1.79, + "learning_rate": 7.342860601823875e-05, + "loss": 0.1449, + "step": 5535 + }, + { + "epoch": 1.8, + "learning_rate": 7.325995688466146e-05, + "loss": 0.1339, + "step": 5540 + }, + { + "epoch": 1.8, + "learning_rate": 7.309138962381872e-05, + "loss": 0.1341, + "step": 5545 + }, + { + "epoch": 1.8, + "learning_rate": 7.29229047518302e-05, + "loss": 0.1324, + "step": 5550 + }, + { + "epoch": 1.8, + "learning_rate": 7.275450278456335e-05, + "loss": 0.129, + "step": 5555 + }, + { + "epoch": 1.8, + "learning_rate": 7.258618423763171e-05, + "loss": 0.1388, + "step": 5560 + }, + { + "epoch": 1.8, + "learning_rate": 7.24179496263935e-05, + "loss": 0.1406, + "step": 5565 + }, + { + "epoch": 1.8, + "learning_rate": 7.224979946594992e-05, + "loss": 0.138, + "step": 5570 + }, + { + "epoch": 1.81, + "learning_rate": 7.20817342711435e-05, + "loss": 0.1412, + "step": 5575 + }, + { + "epoch": 1.81, + "learning_rate": 7.191375455655676e-05, + "loss": 0.1377, + "step": 5580 + }, + { + "epoch": 1.81, + "learning_rate": 7.174586083651038e-05, + "loss": 0.1387, + "step": 5585 + }, + { + "epoch": 1.81, + "learning_rate": 7.157805362506184e-05, + "loss": 0.1352, + "step": 5590 + }, + { + "epoch": 1.81, + "learning_rate": 7.14103334360037e-05, + "loss": 0.1457, + "step": 5595 + }, + { + "epoch": 1.81, + "learning_rate": 7.124270078286203e-05, + "loss": 0.1359, + "step": 5600 + }, + { + "epoch": 1.82, + "learning_rate": 7.107515617889498e-05, + "loss": 0.138, + "step": 5605 + }, + { + "epoch": 1.82, + "learning_rate": 7.090770013709105e-05, + "loss": 0.1402, + "step": 5610 + }, + { + "epoch": 1.82, + "learning_rate": 7.074033317016755e-05, + "loss": 0.133, + "step": 5615 + }, + { + "epoch": 1.82, + "learning_rate": 7.057305579056919e-05, + "loss": 0.1419, + "step": 5620 + }, + { + "epoch": 1.82, + "learning_rate": 7.040586851046618e-05, + "loss": 0.1282, + "step": 5625 + }, + { + "epoch": 1.82, + "learning_rate": 7.023877184175307e-05, + "loss": 0.1393, + "step": 5630 + }, + { + "epoch": 1.83, + "learning_rate": 7.007176629604681e-05, + "loss": 0.1273, + "step": 5635 + }, + { + "epoch": 1.83, + "learning_rate": 6.990485238468545e-05, + "loss": 0.1417, + "step": 5640 + }, + { + "epoch": 1.83, + "learning_rate": 6.973803061872646e-05, + "loss": 0.1376, + "step": 5645 + }, + { + "epoch": 1.83, + "learning_rate": 6.957130150894516e-05, + "loss": 0.1421, + "step": 5650 + }, + { + "epoch": 1.83, + "learning_rate": 6.940466556583316e-05, + "loss": 0.1311, + "step": 5655 + }, + { + "epoch": 1.83, + "learning_rate": 6.923812329959683e-05, + "loss": 0.1376, + "step": 5660 + }, + { + "epoch": 1.84, + "learning_rate": 6.907167522015577e-05, + "loss": 0.1375, + "step": 5665 + }, + { + "epoch": 1.84, + "learning_rate": 6.890532183714105e-05, + "loss": 0.1304, + "step": 5670 + }, + { + "epoch": 1.84, + "learning_rate": 6.8739063659894e-05, + "loss": 0.1358, + "step": 5675 + }, + { + "epoch": 1.84, + "learning_rate": 6.857290119746425e-05, + "loss": 0.1418, + "step": 5680 + }, + { + "epoch": 1.84, + "learning_rate": 6.840683495860857e-05, + "loss": 0.1367, + "step": 5685 + }, + { + "epoch": 1.84, + "learning_rate": 6.82408654517889e-05, + "loss": 0.1373, + "step": 5690 + }, + { + "epoch": 1.85, + "learning_rate": 6.80749931851712e-05, + "loss": 0.1408, + "step": 5695 + }, + { + "epoch": 1.85, + "learning_rate": 6.790921866662355e-05, + "loss": 0.1326, + "step": 5700 + }, + { + "epoch": 1.85, + "learning_rate": 6.774354240371487e-05, + "loss": 0.137, + "step": 5705 + }, + { + "epoch": 1.85, + "learning_rate": 6.757796490371314e-05, + "loss": 0.1509, + "step": 5710 + }, + { + "epoch": 1.85, + "learning_rate": 6.741248667358397e-05, + "loss": 0.1317, + "step": 5715 + }, + { + "epoch": 1.85, + "learning_rate": 6.724710821998912e-05, + "loss": 0.1395, + "step": 5720 + }, + { + "epoch": 1.86, + "learning_rate": 6.708183004928468e-05, + "loss": 0.1359, + "step": 5725 + }, + { + "epoch": 1.86, + "learning_rate": 6.691665266751985e-05, + "loss": 0.1448, + "step": 5730 + }, + { + "epoch": 1.86, + "learning_rate": 6.675157658043513e-05, + "loss": 0.1365, + "step": 5735 + }, + { + "epoch": 1.86, + "learning_rate": 6.658660229346097e-05, + "loss": 0.1341, + "step": 5740 + }, + { + "epoch": 1.86, + "learning_rate": 6.642173031171603e-05, + "loss": 0.137, + "step": 5745 + }, + { + "epoch": 1.86, + "learning_rate": 6.62569611400058e-05, + "loss": 0.1376, + "step": 5750 + }, + { + "epoch": 1.86, + "learning_rate": 6.609229528282094e-05, + "loss": 0.1297, + "step": 5755 + }, + { + "epoch": 1.87, + "learning_rate": 6.592773324433588e-05, + "loss": 0.1344, + "step": 5760 + }, + { + "epoch": 1.87, + "learning_rate": 6.5763275528407e-05, + "loss": 0.1359, + "step": 5765 + }, + { + "epoch": 1.87, + "learning_rate": 6.559892263857144e-05, + "loss": 0.1302, + "step": 5770 + }, + { + "epoch": 1.87, + "learning_rate": 6.54346750780453e-05, + "loss": 0.132, + "step": 5775 + }, + { + "epoch": 1.87, + "learning_rate": 6.527053334972218e-05, + "loss": 0.1349, + "step": 5780 + }, + { + "epoch": 1.87, + "learning_rate": 6.510649795617168e-05, + "loss": 0.1406, + "step": 5785 + }, + { + "epoch": 1.88, + "learning_rate": 6.494256939963775e-05, + "loss": 0.1345, + "step": 5790 + }, + { + "epoch": 1.88, + "learning_rate": 6.477874818203736e-05, + "loss": 0.1392, + "step": 5795 + }, + { + "epoch": 1.88, + "learning_rate": 6.461503480495864e-05, + "loss": 0.1359, + "step": 5800 + }, + { + "epoch": 1.88, + "learning_rate": 6.445142976965972e-05, + "loss": 0.1284, + "step": 5805 + }, + { + "epoch": 1.88, + "learning_rate": 6.428793357706689e-05, + "loss": 0.1343, + "step": 5810 + }, + { + "epoch": 1.88, + "learning_rate": 6.412454672777328e-05, + "loss": 0.1387, + "step": 5815 + }, + { + "epoch": 1.89, + "learning_rate": 6.396126972203708e-05, + "loss": 0.1334, + "step": 5820 + }, + { + "epoch": 1.89, + "learning_rate": 6.379810305978038e-05, + "loss": 0.1329, + "step": 5825 + }, + { + "epoch": 1.89, + "learning_rate": 6.363504724058717e-05, + "loss": 0.1378, + "step": 5830 + }, + { + "epoch": 1.89, + "learning_rate": 6.347210276370224e-05, + "loss": 0.1358, + "step": 5835 + }, + { + "epoch": 1.89, + "learning_rate": 6.330927012802947e-05, + "loss": 0.1412, + "step": 5840 + }, + { + "epoch": 1.89, + "learning_rate": 6.314654983213018e-05, + "loss": 0.1308, + "step": 5845 + }, + { + "epoch": 1.9, + "learning_rate": 6.298394237422186e-05, + "loss": 0.131, + "step": 5850 + }, + { + "epoch": 1.9, + "learning_rate": 6.282144825217639e-05, + "loss": 0.1341, + "step": 5855 + }, + { + "epoch": 1.9, + "learning_rate": 6.265906796351875e-05, + "loss": 0.1435, + "step": 5860 + }, + { + "epoch": 1.9, + "learning_rate": 6.24968020054253e-05, + "loss": 0.1417, + "step": 5865 + }, + { + "epoch": 1.9, + "learning_rate": 6.233465087472244e-05, + "loss": 0.1376, + "step": 5870 + }, + { + "epoch": 1.9, + "learning_rate": 6.217261506788485e-05, + "loss": 0.1268, + "step": 5875 + }, + { + "epoch": 1.91, + "learning_rate": 6.201069508103426e-05, + "loss": 0.1389, + "step": 5880 + }, + { + "epoch": 1.91, + "learning_rate": 6.184889140993765e-05, + "loss": 0.14, + "step": 5885 + }, + { + "epoch": 1.91, + "learning_rate": 6.168720455000601e-05, + "loss": 0.1394, + "step": 5890 + }, + { + "epoch": 1.91, + "learning_rate": 6.152563499629249e-05, + "loss": 0.1413, + "step": 5895 + }, + { + "epoch": 1.91, + "learning_rate": 6.136418324349125e-05, + "loss": 0.1354, + "step": 5900 + }, + { + "epoch": 1.91, + "learning_rate": 6.120284978593569e-05, + "loss": 0.1335, + "step": 5905 + }, + { + "epoch": 1.92, + "learning_rate": 6.104163511759696e-05, + "loss": 0.1381, + "step": 5910 + }, + { + "epoch": 1.92, + "learning_rate": 6.088053973208262e-05, + "loss": 0.1313, + "step": 5915 + }, + { + "epoch": 1.92, + "learning_rate": 6.071956412263491e-05, + "loss": 0.1314, + "step": 5920 + }, + { + "epoch": 1.92, + "learning_rate": 6.0558708782129416e-05, + "loss": 0.1432, + "step": 5925 + }, + { + "epoch": 1.92, + "learning_rate": 6.039797420307338e-05, + "loss": 0.1398, + "step": 5930 + }, + { + "epoch": 1.92, + "learning_rate": 6.023736087760443e-05, + "loss": 0.1334, + "step": 5935 + }, + { + "epoch": 1.92, + "learning_rate": 6.00768692974888e-05, + "loss": 0.1393, + "step": 5940 + }, + { + "epoch": 1.93, + "learning_rate": 5.991649995412011e-05, + "loss": 0.1319, + "step": 5945 + }, + { + "epoch": 1.93, + "learning_rate": 5.975625333851755e-05, + "loss": 0.1304, + "step": 5950 + }, + { + "epoch": 1.93, + "learning_rate": 5.9596129941324705e-05, + "loss": 0.1332, + "step": 5955 + }, + { + "epoch": 1.93, + "learning_rate": 5.9436130252807785e-05, + "loss": 0.1344, + "step": 5960 + }, + { + "epoch": 1.93, + "learning_rate": 5.927625476285426e-05, + "loss": 0.1329, + "step": 5965 + }, + { + "epoch": 1.93, + "learning_rate": 5.91165039609713e-05, + "loss": 0.1359, + "step": 5970 + }, + { + "epoch": 1.94, + "learning_rate": 5.895687833628437e-05, + "loss": 0.1311, + "step": 5975 + }, + { + "epoch": 1.94, + "learning_rate": 5.879737837753565e-05, + "loss": 0.1324, + "step": 5980 + }, + { + "epoch": 1.94, + "learning_rate": 5.863800457308246e-05, + "loss": 0.1286, + "step": 5985 + }, + { + "epoch": 1.94, + "learning_rate": 5.847875741089604e-05, + "loss": 0.134, + "step": 5990 + }, + { + "epoch": 1.94, + "learning_rate": 5.8319637378559635e-05, + "loss": 0.1312, + "step": 5995 + }, + { + "epoch": 1.94, + "learning_rate": 5.816064496326752e-05, + "loss": 0.1277, + "step": 6000 + } + ], + "logging_steps": 5, + "max_steps": 9255, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 1000, + "total_flos": 1.2219978814213587e+19, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/save_lora_math_r8_2e4/checkpoint-6000/training_args.bin b/save_lora_math_r8_2e4/checkpoint-6000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..175cae96ce99657f1e2b29179eee355d72511307 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-6000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0cedf67f015d3f83c601cebdd463c220d745d8e241fab282366821363eb224d +size 5880 diff --git a/save_lora_math_r8_2e4/checkpoint-6000/zero_to_fp32.py b/save_lora_math_r8_2e4/checkpoint-6000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..c98caae31534368be22b67fc4ae906836c992a8d --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-6000/zero_to_fp32.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in param_shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, args.output_file, tag=args.tag) diff --git a/save_lora_math_r8_2e4/checkpoint-7000/README.md b/save_lora_math_r8_2e4/checkpoint-7000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d380bbbf0bf0fa1f211b53410559607e5007a209 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-7000/README.md @@ -0,0 +1,202 @@ +--- +library_name: peft +base_model: meta-llama/Llama-2-7b-hf +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.9.0 \ No newline at end of file diff --git a/save_lora_math_r8_2e4/checkpoint-7000/adapter_config.json b/save_lora_math_r8_2e4/checkpoint-7000/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9c81a592f0ff08374a4f61f6003029070a8429ae --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-7000/adapter_config.json @@ -0,0 +1,35 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "meta-llama/Llama-2-7b-hf", + "bias": "none", + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "mora_type": 1, + "peft_type": "LORA", + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "down_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "use_dora": false, + "use_mora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/save_lora_math_r8_2e4/checkpoint-7000/adapter_model.safetensors b/save_lora_math_r8_2e4/checkpoint-7000/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..529fba7d250d0adcfa3ccb6b0f551c54b06f7f6a --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-7000/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ec9cf53a4ba874f28129e0612c503afbd6936c63615f7be5ac2262faf887b1a +size 40036488 diff --git a/save_lora_math_r8_2e4/checkpoint-7000/global_step7000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/save_lora_math_r8_2e4/checkpoint-7000/global_step7000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3892af16654c91bff7e61de124ffdfcb74ef6a06 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-7000/global_step7000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f97eaa09adfdff2ab4c07a2ef233e2e3d852cef1543a2d773b219bef95c40f82 +size 119958768 diff --git a/save_lora_math_r8_2e4/checkpoint-7000/global_step7000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/save_lora_math_r8_2e4/checkpoint-7000/global_step7000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6917ddb0f249f6e042a3e64c4e3fa1d3e91297fc --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-7000/global_step7000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aed69d4153c1b963955c5c5b83f1dc6c2cdeb803029f568feb8822fca7866f05 +size 119958896 diff --git a/save_lora_math_r8_2e4/checkpoint-7000/global_step7000/mp_rank_00_model_states.pt b/save_lora_math_r8_2e4/checkpoint-7000/global_step7000/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..edf028ff2702f9c30453fa3980e64114b7b41d8c --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-7000/global_step7000/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a759cad7d736251af04399c5d438a04503c6ac7572830a224713a5f24eca7d6f +size 40303404 diff --git a/save_lora_math_r8_2e4/checkpoint-7000/latest b/save_lora_math_r8_2e4/checkpoint-7000/latest new file mode 100644 index 0000000000000000000000000000000000000000..02f8b21f7d75fb06ab0c850d040bab5a1b2af782 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-7000/latest @@ -0,0 +1 @@ +global_step7000 \ No newline at end of file diff --git a/save_lora_math_r8_2e4/checkpoint-7000/rng_state_0.pth b/save_lora_math_r8_2e4/checkpoint-7000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..f04d3d088e9a3b973b670673559e512a030dc384 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-7000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7140a72282af0a742a5f90639214b066dea4ed7d774aae343a1dd935bbdd911a +size 14512 diff --git a/save_lora_math_r8_2e4/checkpoint-7000/rng_state_1.pth b/save_lora_math_r8_2e4/checkpoint-7000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..6a5905a39f457ed0b2c1965b00d0c23691459824 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-7000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67c545754bee81a4d2cf7777b5afdbfab09337a9c2e722fa7b52c4906e1528cc +size 14512 diff --git a/save_lora_math_r8_2e4/checkpoint-7000/scheduler.pt b/save_lora_math_r8_2e4/checkpoint-7000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..49273b3ba585e3922479277d47b60a5058a9be53 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-7000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a74ef86c34d1c2145e98b2bdc1c6ed0ae94e210706a18f9f841f398a568440e2 +size 1064 diff --git a/save_lora_math_r8_2e4/checkpoint-7000/trainer_state.json b/save_lora_math_r8_2e4/checkpoint-7000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..037af11e8452b261949f8f7ad8143873ba9efd49 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-7000/trainer_state.json @@ -0,0 +1,8421 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.268354430379747, + "eval_steps": 500, + "global_step": 7000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 3.5971223021582732e-06, + "loss": 0.7628, + "step": 5 + }, + { + "epoch": 0.0, + "learning_rate": 7.1942446043165465e-06, + "loss": 0.769, + "step": 10 + }, + { + "epoch": 0.0, + "learning_rate": 1.0791366906474821e-05, + "loss": 0.7619, + "step": 15 + }, + { + "epoch": 0.01, + "learning_rate": 1.4388489208633093e-05, + "loss": 0.7564, + "step": 20 + }, + { + "epoch": 0.01, + "learning_rate": 1.7985611510791367e-05, + "loss": 0.7549, + "step": 25 + }, + { + "epoch": 0.01, + "learning_rate": 2.1582733812949642e-05, + "loss": 0.7001, + "step": 30 + }, + { + "epoch": 0.01, + "learning_rate": 2.5179856115107914e-05, + "loss": 0.6289, + "step": 35 + }, + { + "epoch": 0.01, + "learning_rate": 2.8776978417266186e-05, + "loss": 0.5632, + "step": 40 + }, + { + "epoch": 0.01, + "learning_rate": 3.237410071942446e-05, + "loss": 0.4923, + "step": 45 + }, + { + "epoch": 0.02, + "learning_rate": 3.597122302158273e-05, + "loss": 0.4582, + "step": 50 + }, + { + "epoch": 0.02, + "learning_rate": 3.956834532374101e-05, + "loss": 0.4357, + "step": 55 + }, + { + "epoch": 0.02, + "learning_rate": 4.3165467625899284e-05, + "loss": 0.4079, + "step": 60 + }, + { + "epoch": 0.02, + "learning_rate": 4.676258992805755e-05, + "loss": 0.4047, + "step": 65 + }, + { + "epoch": 0.02, + "learning_rate": 5.035971223021583e-05, + "loss": 0.3916, + "step": 70 + }, + { + "epoch": 0.02, + "learning_rate": 5.39568345323741e-05, + "loss": 0.3926, + "step": 75 + }, + { + "epoch": 0.03, + "learning_rate": 5.755395683453237e-05, + "loss": 0.3725, + "step": 80 + }, + { + "epoch": 0.03, + "learning_rate": 6.115107913669065e-05, + "loss": 0.3769, + "step": 85 + }, + { + "epoch": 0.03, + "learning_rate": 6.474820143884892e-05, + "loss": 0.352, + "step": 90 + }, + { + "epoch": 0.03, + "learning_rate": 6.83453237410072e-05, + "loss": 0.3534, + "step": 95 + }, + { + "epoch": 0.03, + "learning_rate": 7.194244604316547e-05, + "loss": 0.3432, + "step": 100 + }, + { + "epoch": 0.03, + "learning_rate": 7.553956834532374e-05, + "loss": 0.3286, + "step": 105 + }, + { + "epoch": 0.04, + "learning_rate": 7.913669064748202e-05, + "loss": 0.3298, + "step": 110 + }, + { + "epoch": 0.04, + "learning_rate": 8.273381294964029e-05, + "loss": 0.3223, + "step": 115 + }, + { + "epoch": 0.04, + "learning_rate": 8.633093525179857e-05, + "loss": 0.3181, + "step": 120 + }, + { + "epoch": 0.04, + "learning_rate": 8.992805755395684e-05, + "loss": 0.3225, + "step": 125 + }, + { + "epoch": 0.04, + "learning_rate": 9.35251798561151e-05, + "loss": 0.3145, + "step": 130 + }, + { + "epoch": 0.04, + "learning_rate": 9.712230215827339e-05, + "loss": 0.3137, + "step": 135 + }, + { + "epoch": 0.05, + "learning_rate": 0.00010071942446043166, + "loss": 0.3102, + "step": 140 + }, + { + "epoch": 0.05, + "learning_rate": 0.00010431654676258992, + "loss": 0.2951, + "step": 145 + }, + { + "epoch": 0.05, + "learning_rate": 0.0001079136690647482, + "loss": 0.3021, + "step": 150 + }, + { + "epoch": 0.05, + "learning_rate": 0.00011151079136690648, + "loss": 0.2923, + "step": 155 + }, + { + "epoch": 0.05, + "learning_rate": 0.00011510791366906474, + "loss": 0.2953, + "step": 160 + }, + { + "epoch": 0.05, + "learning_rate": 0.00011870503597122303, + "loss": 0.2893, + "step": 165 + }, + { + "epoch": 0.06, + "learning_rate": 0.0001223021582733813, + "loss": 0.2953, + "step": 170 + }, + { + "epoch": 0.06, + "learning_rate": 0.00012589928057553956, + "loss": 0.2931, + "step": 175 + }, + { + "epoch": 0.06, + "learning_rate": 0.00012949640287769783, + "loss": 0.2879, + "step": 180 + }, + { + "epoch": 0.06, + "learning_rate": 0.00013309352517985613, + "loss": 0.2642, + "step": 185 + }, + { + "epoch": 0.06, + "learning_rate": 0.0001366906474820144, + "loss": 0.2772, + "step": 190 + }, + { + "epoch": 0.06, + "learning_rate": 0.00014028776978417266, + "loss": 0.2858, + "step": 195 + }, + { + "epoch": 0.06, + "learning_rate": 0.00014388489208633093, + "loss": 0.2724, + "step": 200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0001474820143884892, + "loss": 0.2726, + "step": 205 + }, + { + "epoch": 0.07, + "learning_rate": 0.00015107913669064747, + "loss": 0.2678, + "step": 210 + }, + { + "epoch": 0.07, + "learning_rate": 0.00015467625899280577, + "loss": 0.2789, + "step": 215 + }, + { + "epoch": 0.07, + "learning_rate": 0.00015827338129496403, + "loss": 0.2742, + "step": 220 + }, + { + "epoch": 0.07, + "learning_rate": 0.0001618705035971223, + "loss": 0.2732, + "step": 225 + }, + { + "epoch": 0.07, + "learning_rate": 0.00016546762589928057, + "loss": 0.2741, + "step": 230 + }, + { + "epoch": 0.08, + "learning_rate": 0.00016906474820143884, + "loss": 0.2665, + "step": 235 + }, + { + "epoch": 0.08, + "learning_rate": 0.00017266187050359714, + "loss": 0.2771, + "step": 240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0001762589928057554, + "loss": 0.2751, + "step": 245 + }, + { + "epoch": 0.08, + "learning_rate": 0.00017985611510791367, + "loss": 0.2597, + "step": 250 + }, + { + "epoch": 0.08, + "learning_rate": 0.00018345323741007194, + "loss": 0.275, + "step": 255 + }, + { + "epoch": 0.08, + "learning_rate": 0.0001870503597122302, + "loss": 0.2791, + "step": 260 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019064748201438848, + "loss": 0.2493, + "step": 265 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019424460431654677, + "loss": 0.27, + "step": 270 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019784172661870504, + "loss": 0.2676, + "step": 275 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019999997550557367, + "loss": 0.2613, + "step": 280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019999969994341502, + "loss": 0.2747, + "step": 285 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019999911820191125, + "loss": 0.2699, + "step": 290 + }, + { + "epoch": 0.1, + "learning_rate": 0.0001999982302828436, + "loss": 0.2692, + "step": 295 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999703618893072, + "loss": 0.2522, + "step": 300 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999553592382863, + "loss": 0.2541, + "step": 305 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999372949213085, + "loss": 0.2614, + "step": 310 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999161689936837, + "loss": 0.2481, + "step": 315 + }, + { + "epoch": 0.1, + "learning_rate": 0.0001999891981520095, + "loss": 0.2497, + "step": 320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019998647325745995, + "loss": 0.2545, + "step": 325 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019998344222406283, + "loss": 0.2501, + "step": 330 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019998010506109854, + "loss": 0.2577, + "step": 335 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019997646177878487, + "loss": 0.2585, + "step": 340 + }, + { + "epoch": 0.11, + "learning_rate": 0.0001999725123882768, + "loss": 0.2494, + "step": 345 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019996825690166657, + "loss": 0.2538, + "step": 350 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019996369533198364, + "loss": 0.2469, + "step": 355 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019995882769319466, + "loss": 0.2502, + "step": 360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0001999536540002034, + "loss": 0.2361, + "step": 365 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019994817426885057, + "loss": 0.2446, + "step": 370 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019994238851591417, + "loss": 0.256, + "step": 375 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019993629675910896, + "loss": 0.2352, + "step": 380 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019992989901708672, + "loss": 0.2538, + "step": 385 + }, + { + "epoch": 0.13, + "learning_rate": 0.0001999231953094361, + "loss": 0.2523, + "step": 390 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019991618565668243, + "loss": 0.2512, + "step": 395 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019990887008028798, + "loss": 0.2402, + "step": 400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019990124860265156, + "loss": 0.2423, + "step": 405 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019989332124710862, + "loss": 0.2335, + "step": 410 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019988508803793124, + "loss": 0.2284, + "step": 415 + }, + { + "epoch": 0.14, + "learning_rate": 0.00019987654900032774, + "loss": 0.2382, + "step": 420 + }, + { + "epoch": 0.14, + "learning_rate": 0.0001998677041604431, + "loss": 0.2526, + "step": 425 + }, + { + "epoch": 0.14, + "learning_rate": 0.0001998585535453584, + "loss": 0.2417, + "step": 430 + }, + { + "epoch": 0.14, + "learning_rate": 0.00019984909718309105, + "loss": 0.2381, + "step": 435 + }, + { + "epoch": 0.14, + "learning_rate": 0.0001998393351025945, + "loss": 0.2418, + "step": 440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00019982926733375843, + "loss": 0.2409, + "step": 445 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001998188939074083, + "loss": 0.2467, + "step": 450 + }, + { + "epoch": 0.15, + "learning_rate": 0.00019980821485530544, + "loss": 0.2305, + "step": 455 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001997972302101471, + "loss": 0.232, + "step": 460 + }, + { + "epoch": 0.15, + "learning_rate": 0.00019978594000556602, + "loss": 0.225, + "step": 465 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001997743442761306, + "loss": 0.2259, + "step": 470 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001997624430573447, + "loss": 0.2414, + "step": 475 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019975023638564747, + "loss": 0.238, + "step": 480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019973772429841336, + "loss": 0.224, + "step": 485 + }, + { + "epoch": 0.16, + "learning_rate": 0.0001997249068339519, + "loss": 0.2369, + "step": 490 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019971178403150762, + "loss": 0.2393, + "step": 495 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019969835593125997, + "loss": 0.2249, + "step": 500 + }, + { + "epoch": 0.16, + "learning_rate": 0.0001996846225743231, + "loss": 0.2369, + "step": 505 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019967058400274592, + "loss": 0.2267, + "step": 510 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019965624025951168, + "loss": 0.2294, + "step": 515 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019964159138853815, + "loss": 0.2255, + "step": 520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019962663743467726, + "loss": 0.22, + "step": 525 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019961137844371507, + "loss": 0.2398, + "step": 530 + }, + { + "epoch": 0.17, + "learning_rate": 0.0001995958144623716, + "loss": 0.2431, + "step": 535 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019957994553830067, + "loss": 0.2245, + "step": 540 + }, + { + "epoch": 0.18, + "learning_rate": 0.0001995637717200898, + "loss": 0.2295, + "step": 545 + }, + { + "epoch": 0.18, + "learning_rate": 0.00019954729305726008, + "loss": 0.2288, + "step": 550 + }, + { + "epoch": 0.18, + "learning_rate": 0.00019953050960026586, + "loss": 0.2333, + "step": 555 + }, + { + "epoch": 0.18, + "learning_rate": 0.0001995134214004948, + "loss": 0.2316, + "step": 560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0001994960285102676, + "loss": 0.2285, + "step": 565 + }, + { + "epoch": 0.18, + "learning_rate": 0.00019947833098283782, + "loss": 0.2273, + "step": 570 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019946032887239182, + "loss": 0.2344, + "step": 575 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019944202223404854, + "loss": 0.2251, + "step": 580 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019942341112385922, + "loss": 0.226, + "step": 585 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019940449559880752, + "loss": 0.2353, + "step": 590 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019938527571680892, + "loss": 0.226, + "step": 595 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019936575153671097, + "loss": 0.221, + "step": 600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019934592311829285, + "loss": 0.2219, + "step": 605 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019932579052226528, + "loss": 0.234, + "step": 610 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019930535381027027, + "loss": 0.2181, + "step": 615 + }, + { + "epoch": 0.2, + "learning_rate": 0.000199284613044881, + "loss": 0.2191, + "step": 620 + }, + { + "epoch": 0.2, + "learning_rate": 0.0001992635682896016, + "loss": 0.235, + "step": 625 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019924221960886698, + "loss": 0.2256, + "step": 630 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019922056706804257, + "loss": 0.2265, + "step": 635 + }, + { + "epoch": 0.21, + "learning_rate": 0.0001991986107334242, + "loss": 0.2158, + "step": 640 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019917635067223777, + "loss": 0.2238, + "step": 645 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019915378695263924, + "loss": 0.2182, + "step": 650 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019913091964371423, + "loss": 0.2185, + "step": 655 + }, + { + "epoch": 0.21, + "learning_rate": 0.000199107748815478, + "loss": 0.2063, + "step": 660 + }, + { + "epoch": 0.22, + "learning_rate": 0.000199084274538875, + "loss": 0.2162, + "step": 665 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019906049688577887, + "loss": 0.2185, + "step": 670 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019903641592899205, + "loss": 0.2186, + "step": 675 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019901203174224564, + "loss": 0.2292, + "step": 680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019898734440019933, + "loss": 0.2162, + "step": 685 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019896235397844078, + "loss": 0.219, + "step": 690 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019893706055348578, + "loss": 0.218, + "step": 695 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019891146420277777, + "loss": 0.2211, + "step": 700 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019888556500468775, + "loss": 0.2227, + "step": 705 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019885936303851395, + "loss": 0.2214, + "step": 710 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019883285838448163, + "loss": 0.2204, + "step": 715 + }, + { + "epoch": 0.23, + "learning_rate": 0.0001988060511237428, + "loss": 0.2228, + "step": 720 + }, + { + "epoch": 0.23, + "learning_rate": 0.000198778941338376, + "loss": 0.2259, + "step": 725 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019875152911138602, + "loss": 0.2062, + "step": 730 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019872381452670372, + "loss": 0.2073, + "step": 735 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019869579766918568, + "loss": 0.2172, + "step": 740 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019866747862461396, + "loss": 0.21, + "step": 745 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019863885747969598, + "loss": 0.2195, + "step": 750 + }, + { + "epoch": 0.24, + "learning_rate": 0.0001986099343220639, + "loss": 0.2129, + "step": 755 + }, + { + "epoch": 0.25, + "learning_rate": 0.0001985807092402748, + "loss": 0.215, + "step": 760 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019855118232381015, + "loss": 0.2085, + "step": 765 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019852135366307545, + "loss": 0.2196, + "step": 770 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019849122334940022, + "loss": 0.2129, + "step": 775 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019846079147503755, + "loss": 0.2221, + "step": 780 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019843005813316377, + "loss": 0.2133, + "step": 785 + }, + { + "epoch": 0.26, + "learning_rate": 0.0001983990234178784, + "loss": 0.2066, + "step": 790 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019836768742420352, + "loss": 0.2188, + "step": 795 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019833605024808382, + "loss": 0.2128, + "step": 800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019830411198638607, + "loss": 0.2109, + "step": 805 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019827187273689894, + "loss": 0.2141, + "step": 810 + }, + { + "epoch": 0.26, + "learning_rate": 0.0001982393325983326, + "loss": 0.2098, + "step": 815 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019820649167031856, + "loss": 0.2188, + "step": 820 + }, + { + "epoch": 0.27, + "learning_rate": 0.0001981733500534093, + "loss": 0.2107, + "step": 825 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019813990784907785, + "loss": 0.2162, + "step": 830 + }, + { + "epoch": 0.27, + "learning_rate": 0.0001981061651597177, + "loss": 0.2189, + "step": 835 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019807212208864227, + "loss": 0.2117, + "step": 840 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019803777874008468, + "loss": 0.209, + "step": 845 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019800313521919753, + "loss": 0.206, + "step": 850 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019796819163205246, + "loss": 0.2104, + "step": 855 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019793294808563984, + "loss": 0.2016, + "step": 860 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019789740468786841, + "loss": 0.212, + "step": 865 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019786156154756506, + "loss": 0.2051, + "step": 870 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019782541877447443, + "loss": 0.21, + "step": 875 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019778897647925855, + "loss": 0.2104, + "step": 880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0001977522347734965, + "loss": 0.2179, + "step": 885 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019771519376968416, + "loss": 0.2139, + "step": 890 + }, + { + "epoch": 0.29, + "learning_rate": 0.0001976778535812338, + "loss": 0.2034, + "step": 895 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019764021432247368, + "loss": 0.2067, + "step": 900 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019760227610864776, + "loss": 0.2189, + "step": 905 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019756403905591538, + "loss": 0.2119, + "step": 910 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019752550328135086, + "loss": 0.2004, + "step": 915 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019748666890294315, + "loss": 0.2073, + "step": 920 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019744753603959545, + "loss": 0.1995, + "step": 925 + }, + { + "epoch": 0.3, + "learning_rate": 0.0001974081048111248, + "loss": 0.2034, + "step": 930 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019736837533826192, + "loss": 0.2015, + "step": 935 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019732834774265057, + "loss": 0.2166, + "step": 940 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019728802214684734, + "loss": 0.1987, + "step": 945 + }, + { + "epoch": 0.31, + "learning_rate": 0.0001972473986743213, + "loss": 0.2051, + "step": 950 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019720647744945344, + "loss": 0.2055, + "step": 955 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019716525859753652, + "loss": 0.2005, + "step": 960 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019712374224477452, + "loss": 0.1995, + "step": 965 + }, + { + "epoch": 0.31, + "learning_rate": 0.0001970819285182823, + "loss": 0.2096, + "step": 970 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019703981754608525, + "loss": 0.2024, + "step": 975 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019699740945711884, + "loss": 0.2006, + "step": 980 + }, + { + "epoch": 0.32, + "learning_rate": 0.0001969547043812283, + "loss": 0.2077, + "step": 985 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019691170244916814, + "loss": 0.2094, + "step": 990 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019686840379260176, + "loss": 0.2051, + "step": 995 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019682480854410115, + "loss": 0.2158, + "step": 1000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0001967809168371463, + "loss": 0.2035, + "step": 1005 + }, + { + "epoch": 0.33, + "learning_rate": 0.000196736728806125, + "loss": 0.2104, + "step": 1010 + }, + { + "epoch": 0.33, + "learning_rate": 0.00019669224458633222, + "loss": 0.1969, + "step": 1015 + }, + { + "epoch": 0.33, + "learning_rate": 0.0001966474643139699, + "loss": 0.2058, + "step": 1020 + }, + { + "epoch": 0.33, + "learning_rate": 0.00019660238812614647, + "loss": 0.2045, + "step": 1025 + }, + { + "epoch": 0.33, + "learning_rate": 0.0001965570161608762, + "loss": 0.1916, + "step": 1030 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019651134855707913, + "loss": 0.2008, + "step": 1035 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019646538545458047, + "loss": 0.2095, + "step": 1040 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019641912699411016, + "loss": 0.2009, + "step": 1045 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019637257331730248, + "loss": 0.2059, + "step": 1050 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019632572456669558, + "loss": 0.2084, + "step": 1055 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019627858088573112, + "loss": 0.1997, + "step": 1060 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001962311424187537, + "loss": 0.2068, + "step": 1065 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001961834093110106, + "loss": 0.1978, + "step": 1070 + }, + { + "epoch": 0.35, + "learning_rate": 0.00019613538170865118, + "loss": 0.2028, + "step": 1075 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001960870597587264, + "loss": 0.1996, + "step": 1080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001960384436091886, + "loss": 0.2002, + "step": 1085 + }, + { + "epoch": 0.35, + "learning_rate": 0.00019598953340889082, + "loss": 0.198, + "step": 1090 + }, + { + "epoch": 0.35, + "learning_rate": 0.00019594032930758643, + "loss": 0.1974, + "step": 1095 + }, + { + "epoch": 0.36, + "learning_rate": 0.0001958908314559287, + "loss": 0.1983, + "step": 1100 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019584104000547025, + "loss": 0.1987, + "step": 1105 + }, + { + "epoch": 0.36, + "learning_rate": 0.0001957909551086627, + "loss": 0.2003, + "step": 1110 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019574057691885613, + "loss": 0.1982, + "step": 1115 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019568990559029858, + "loss": 0.2031, + "step": 1120 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019563894127813564, + "loss": 0.1969, + "step": 1125 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019558768413841006, + "loss": 0.1916, + "step": 1130 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019553613432806102, + "loss": 0.2124, + "step": 1135 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019548429200492388, + "loss": 0.2042, + "step": 1140 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019543215732772956, + "loss": 0.201, + "step": 1145 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019537973045610424, + "loss": 0.2001, + "step": 1150 + }, + { + "epoch": 0.37, + "learning_rate": 0.0001953270115505686, + "loss": 0.1978, + "step": 1155 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019527400077253755, + "loss": 0.192, + "step": 1160 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019522069828431957, + "loss": 0.1946, + "step": 1165 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019516710424911642, + "loss": 0.1848, + "step": 1170 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019511321883102244, + "loss": 0.2005, + "step": 1175 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019505904219502414, + "loss": 0.1966, + "step": 1180 + }, + { + "epoch": 0.38, + "learning_rate": 0.0001950045745069997, + "loss": 0.1863, + "step": 1185 + }, + { + "epoch": 0.39, + "learning_rate": 0.0001949498159337184, + "loss": 0.1983, + "step": 1190 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019489476664284023, + "loss": 0.2113, + "step": 1195 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019483942680291523, + "loss": 0.1956, + "step": 1200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019478379658338305, + "loss": 0.2055, + "step": 1205 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019472787615457246, + "loss": 0.1879, + "step": 1210 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019467166568770078, + "loss": 0.1924, + "step": 1215 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019461516535487338, + "loss": 0.1971, + "step": 1220 + }, + { + "epoch": 0.4, + "learning_rate": 0.0001945583753290831, + "loss": 0.1952, + "step": 1225 + }, + { + "epoch": 0.4, + "learning_rate": 0.0001945012957842098, + "loss": 0.1838, + "step": 1230 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019444392689501978, + "loss": 0.1931, + "step": 1235 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019438626883716526, + "loss": 0.2093, + "step": 1240 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019432832178718385, + "loss": 0.1992, + "step": 1245 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019427008592249795, + "loss": 0.1963, + "step": 1250 + }, + { + "epoch": 0.41, + "learning_rate": 0.0001942115614214143, + "loss": 0.2003, + "step": 1255 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019415274846312338, + "loss": 0.1914, + "step": 1260 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019409364722769882, + "loss": 0.1917, + "step": 1265 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019403425789609697, + "loss": 0.2033, + "step": 1270 + }, + { + "epoch": 0.41, + "learning_rate": 0.0001939745806501563, + "loss": 0.1975, + "step": 1275 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019391461567259662, + "loss": 0.1922, + "step": 1280 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019385436314701897, + "loss": 0.1999, + "step": 1285 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019379382325790468, + "loss": 0.1997, + "step": 1290 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019373299619061487, + "loss": 0.1872, + "step": 1295 + }, + { + "epoch": 0.42, + "learning_rate": 0.0001936718821313901, + "loss": 0.1936, + "step": 1300 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019361048126734952, + "loss": 0.1798, + "step": 1305 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019354879378649046, + "loss": 0.1959, + "step": 1310 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019348681987768783, + "loss": 0.1924, + "step": 1315 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019342455973069345, + "loss": 0.1967, + "step": 1320 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019336201353613568, + "loss": 0.1925, + "step": 1325 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019329918148551856, + "loss": 0.1878, + "step": 1330 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019323606377122146, + "loss": 0.1979, + "step": 1335 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019317266058649834, + "loss": 0.189, + "step": 1340 + }, + { + "epoch": 0.44, + "learning_rate": 0.0001931089721254773, + "loss": 0.1995, + "step": 1345 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019304499858315976, + "loss": 0.1874, + "step": 1350 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019298074015542015, + "loss": 0.1991, + "step": 1355 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019291619703900502, + "loss": 0.188, + "step": 1360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0001928513694315327, + "loss": 0.1914, + "step": 1365 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019278625753149254, + "loss": 0.1843, + "step": 1370 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019272086153824427, + "loss": 0.1959, + "step": 1375 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019265518165201758, + "loss": 0.1867, + "step": 1380 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019258921807391131, + "loss": 0.1956, + "step": 1385 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019252297100589286, + "loss": 0.1991, + "step": 1390 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019245644065079775, + "loss": 0.1934, + "step": 1395 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019238962721232876, + "loss": 0.1953, + "step": 1400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019232253089505546, + "loss": 0.1865, + "step": 1405 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019225515190441354, + "loss": 0.1864, + "step": 1410 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019218749044670413, + "loss": 0.194, + "step": 1415 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019211954672909327, + "loss": 0.1853, + "step": 1420 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019205132095961126, + "loss": 0.1967, + "step": 1425 + }, + { + "epoch": 0.46, + "learning_rate": 0.0001919828133471519, + "loss": 0.1852, + "step": 1430 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019191402410147204, + "loss": 0.1913, + "step": 1435 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019184495343319068, + "loss": 0.1911, + "step": 1440 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019177560155378865, + "loss": 0.189, + "step": 1445 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019170596867560766, + "loss": 0.1854, + "step": 1450 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019163605501184986, + "loss": 0.1974, + "step": 1455 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019156586077657712, + "loss": 0.1933, + "step": 1460 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019149538618471032, + "loss": 0.1877, + "step": 1465 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001914246314520287, + "loss": 0.1904, + "step": 1470 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001913535967951694, + "loss": 0.1841, + "step": 1475 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001912822824316264, + "loss": 0.1876, + "step": 1480 + }, + { + "epoch": 0.48, + "learning_rate": 0.00019121068857975027, + "loss": 0.1858, + "step": 1485 + }, + { + "epoch": 0.48, + "learning_rate": 0.00019113881545874722, + "loss": 0.1898, + "step": 1490 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001910666632886786, + "loss": 0.1923, + "step": 1495 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019099423229046014, + "loss": 0.1803, + "step": 1500 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019092152268586118, + "loss": 0.1945, + "step": 1505 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019084853469750424, + "loss": 0.1802, + "step": 1510 + }, + { + "epoch": 0.49, + "learning_rate": 0.0001907752685488641, + "loss": 0.197, + "step": 1515 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019070172446426724, + "loss": 0.1791, + "step": 1520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019062790266889117, + "loss": 0.1874, + "step": 1525 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019055380338876363, + "loss": 0.1806, + "step": 1530 + }, + { + "epoch": 0.5, + "learning_rate": 0.000190479426850762, + "loss": 0.1834, + "step": 1535 + }, + { + "epoch": 0.5, + "learning_rate": 0.0001904047732826126, + "loss": 0.1921, + "step": 1540 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019032984291288986, + "loss": 0.1826, + "step": 1545 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019025463597101584, + "loss": 0.1966, + "step": 1550 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019017915268725925, + "loss": 0.1867, + "step": 1555 + }, + { + "epoch": 0.51, + "learning_rate": 0.00019010339329273514, + "loss": 0.196, + "step": 1560 + }, + { + "epoch": 0.51, + "learning_rate": 0.00019002735801940373, + "loss": 0.1971, + "step": 1565 + }, + { + "epoch": 0.51, + "learning_rate": 0.00018995104710007004, + "loss": 0.187, + "step": 1570 + }, + { + "epoch": 0.51, + "learning_rate": 0.00018987446076838312, + "loss": 0.1897, + "step": 1575 + }, + { + "epoch": 0.51, + "learning_rate": 0.00018979759925883509, + "loss": 0.1887, + "step": 1580 + }, + { + "epoch": 0.51, + "learning_rate": 0.0001897204628067608, + "loss": 0.1849, + "step": 1585 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018964305164833678, + "loss": 0.1795, + "step": 1590 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018956536602058078, + "loss": 0.1984, + "step": 1595 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018948740616135085, + "loss": 0.1876, + "step": 1600 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018940917230934464, + "loss": 0.1879, + "step": 1605 + }, + { + "epoch": 0.52, + "learning_rate": 0.0001893306647040988, + "loss": 0.1906, + "step": 1610 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018925188358598813, + "loss": 0.1917, + "step": 1615 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018917282919622488, + "loss": 0.1912, + "step": 1620 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018909350177685795, + "loss": 0.1813, + "step": 1625 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018901390157077225, + "loss": 0.183, + "step": 1630 + }, + { + "epoch": 0.53, + "learning_rate": 0.0001889340288216879, + "loss": 0.1782, + "step": 1635 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018885388377415957, + "loss": 0.1866, + "step": 1640 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018877346667357548, + "loss": 0.1915, + "step": 1645 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018869277776615691, + "loss": 0.1902, + "step": 1650 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018861181729895745, + "loss": 0.1829, + "step": 1655 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018853058551986202, + "loss": 0.1855, + "step": 1660 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018844908267758633, + "loss": 0.1845, + "step": 1665 + }, + { + "epoch": 0.54, + "learning_rate": 0.000188367309021676, + "loss": 0.1794, + "step": 1670 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018828526480250575, + "loss": 0.1796, + "step": 1675 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018820295027127885, + "loss": 0.1806, + "step": 1680 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018812036568002617, + "loss": 0.1846, + "step": 1685 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018803751128160538, + "loss": 0.189, + "step": 1690 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018795438732970032, + "loss": 0.1856, + "step": 1695 + }, + { + "epoch": 0.55, + "learning_rate": 0.0001878709940788201, + "loss": 0.1797, + "step": 1700 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018778733178429844, + "loss": 0.1876, + "step": 1705 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018770340070229272, + "loss": 0.1775, + "step": 1710 + }, + { + "epoch": 0.56, + "learning_rate": 0.0001876192010897834, + "loss": 0.1862, + "step": 1715 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018753473320457306, + "loss": 0.1835, + "step": 1720 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018744999730528568, + "loss": 0.1909, + "step": 1725 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018736499365136584, + "loss": 0.189, + "step": 1730 + }, + { + "epoch": 0.56, + "learning_rate": 0.000187279722503078, + "loss": 0.1861, + "step": 1735 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018719418412150557, + "loss": 0.1809, + "step": 1740 + }, + { + "epoch": 0.57, + "learning_rate": 0.00018710837876855017, + "loss": 0.1829, + "step": 1745 + }, + { + "epoch": 0.57, + "learning_rate": 0.0001870223067069309, + "loss": 0.1886, + "step": 1750 + }, + { + "epoch": 0.57, + "learning_rate": 0.00018693596820018336, + "loss": 0.1787, + "step": 1755 + }, + { + "epoch": 0.57, + "learning_rate": 0.0001868493635126591, + "loss": 0.1816, + "step": 1760 + }, + { + "epoch": 0.57, + "learning_rate": 0.0001867624929095245, + "loss": 0.1759, + "step": 1765 + }, + { + "epoch": 0.57, + "learning_rate": 0.00018667535665676025, + "loss": 0.1702, + "step": 1770 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018658795502116035, + "loss": 0.1802, + "step": 1775 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018650028827033136, + "loss": 0.1836, + "step": 1780 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018641235667269155, + "loss": 0.1787, + "step": 1785 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018632416049747016, + "loss": 0.1811, + "step": 1790 + }, + { + "epoch": 0.58, + "learning_rate": 0.0001862357000147064, + "loss": 0.1835, + "step": 1795 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018614697549524889, + "loss": 0.1857, + "step": 1800 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018605798721075455, + "loss": 0.1867, + "step": 1805 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018596873543368797, + "loss": 0.1766, + "step": 1810 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018587922043732046, + "loss": 0.1895, + "step": 1815 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018578944249572925, + "loss": 0.1888, + "step": 1820 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018569940188379672, + "loss": 0.1871, + "step": 1825 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018560909887720944, + "loss": 0.1804, + "step": 1830 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018551853375245737, + "loss": 0.1877, + "step": 1835 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018542770678683308, + "loss": 0.182, + "step": 1840 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001853366182584308, + "loss": 0.1809, + "step": 1845 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018524526844614564, + "loss": 0.1913, + "step": 1850 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018515365762967266, + "loss": 0.185, + "step": 1855 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018506178608950613, + "loss": 0.1753, + "step": 1860 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018496965410693856, + "loss": 0.1786, + "step": 1865 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001848772619640599, + "loss": 0.1841, + "step": 1870 + }, + { + "epoch": 0.61, + "learning_rate": 0.00018478460994375667, + "loss": 0.184, + "step": 1875 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001846916983297111, + "loss": 0.179, + "step": 1880 + }, + { + "epoch": 0.61, + "learning_rate": 0.00018459852740640014, + "loss": 0.1833, + "step": 1885 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001845050974590948, + "loss": 0.186, + "step": 1890 + }, + { + "epoch": 0.61, + "learning_rate": 0.00018441140877385913, + "loss": 0.1778, + "step": 1895 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018431746163754943, + "loss": 0.1786, + "step": 1900 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018422325633781327, + "loss": 0.1826, + "step": 1905 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018412879316308866, + "loss": 0.1738, + "step": 1910 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018403407240260327, + "loss": 0.1852, + "step": 1915 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018393909434637329, + "loss": 0.1821, + "step": 1920 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001838438592852028, + "loss": 0.1773, + "step": 1925 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018374836751068282, + "loss": 0.1772, + "step": 1930 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018365261931519022, + "loss": 0.1766, + "step": 1935 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018355661499188717, + "loss": 0.1858, + "step": 1940 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018346035483471994, + "loss": 0.1786, + "step": 1945 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001833638391384181, + "loss": 0.1745, + "step": 1950 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018326706819849365, + "loss": 0.1713, + "step": 1955 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018317004231124017, + "loss": 0.1738, + "step": 1960 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018307276177373174, + "loss": 0.1807, + "step": 1965 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018297522688382218, + "loss": 0.1806, + "step": 1970 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018287743794014407, + "loss": 0.1735, + "step": 1975 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018277939524210783, + "loss": 0.1812, + "step": 1980 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001826810990899009, + "loss": 0.1734, + "step": 1985 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001825825497844867, + "loss": 0.1703, + "step": 1990 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018248374762760365, + "loss": 0.1823, + "step": 1995 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018238469292176457, + "loss": 0.1837, + "step": 2000 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018228538597025537, + "loss": 0.1807, + "step": 2005 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018218582707713434, + "loss": 0.1904, + "step": 2010 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001820860165472312, + "loss": 0.1844, + "step": 2015 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018198595468614604, + "loss": 0.1792, + "step": 2020 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018188564180024854, + "loss": 0.1845, + "step": 2025 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018178507819667698, + "loss": 0.1853, + "step": 2030 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018168426418333725, + "loss": 0.1875, + "step": 2035 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018158320006890198, + "loss": 0.1759, + "step": 2040 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018148188616280955, + "loss": 0.1816, + "step": 2045 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001813803227752631, + "loss": 0.1787, + "step": 2050 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001812785102172297, + "loss": 0.1758, + "step": 2055 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018117644880043932, + "loss": 0.1802, + "step": 2060 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018107413883738387, + "loss": 0.186, + "step": 2065 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018097158064131626, + "loss": 0.1743, + "step": 2070 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018086877452624944, + "loss": 0.1877, + "step": 2075 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018076572080695543, + "loss": 0.1742, + "step": 2080 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018066241979896444, + "loss": 0.1727, + "step": 2085 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018055887181856375, + "loss": 0.1806, + "step": 2090 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018045507718279677, + "loss": 0.175, + "step": 2095 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018035103620946225, + "loss": 0.1727, + "step": 2100 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001802467492171131, + "loss": 0.1779, + "step": 2105 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018014221652505556, + "loss": 0.1823, + "step": 2110 + }, + { + "epoch": 0.69, + "learning_rate": 0.000180037438453348, + "loss": 0.1814, + "step": 2115 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001799324153228003, + "loss": 0.1707, + "step": 2120 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001798271474549725, + "loss": 0.1778, + "step": 2125 + }, + { + "epoch": 0.69, + "learning_rate": 0.000179721635172174, + "loss": 0.1888, + "step": 2130 + }, + { + "epoch": 0.69, + "learning_rate": 0.00017961587879746266, + "loss": 0.1785, + "step": 2135 + }, + { + "epoch": 0.69, + "learning_rate": 0.00017950987865464365, + "loss": 0.1661, + "step": 2140 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017940363506826836, + "loss": 0.1792, + "step": 2145 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001792971483636338, + "loss": 0.1732, + "step": 2150 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017919041886678118, + "loss": 0.1771, + "step": 2155 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001790834469044952, + "loss": 0.173, + "step": 2160 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017897623280430284, + "loss": 0.1768, + "step": 2165 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017886877689447258, + "loss": 0.175, + "step": 2170 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001787610795040132, + "loss": 0.1691, + "step": 2175 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001786531409626728, + "loss": 0.1774, + "step": 2180 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001785449616009379, + "loss": 0.1709, + "step": 2185 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001784365417500324, + "loss": 0.1809, + "step": 2190 + }, + { + "epoch": 0.71, + "learning_rate": 0.00017832788174191645, + "loss": 0.1764, + "step": 2195 + }, + { + "epoch": 0.71, + "learning_rate": 0.00017821898190928558, + "loss": 0.1739, + "step": 2200 + }, + { + "epoch": 0.71, + "learning_rate": 0.00017810984258556957, + "loss": 0.1772, + "step": 2205 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017800046410493145, + "loss": 0.1779, + "step": 2210 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001778908468022666, + "loss": 0.1828, + "step": 2215 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017778099101320156, + "loss": 0.1722, + "step": 2220 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017767089707409304, + "loss": 0.1737, + "step": 2225 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017756056532202696, + "loss": 0.1688, + "step": 2230 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017744999609481738, + "loss": 0.1784, + "step": 2235 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017733918973100545, + "loss": 0.1747, + "step": 2240 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017722814656985841, + "loss": 0.1667, + "step": 2245 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001771168669513685, + "loss": 0.1648, + "step": 2250 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017700535121625187, + "loss": 0.1777, + "step": 2255 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017689359970594779, + "loss": 0.175, + "step": 2260 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001767816127626173, + "loss": 0.1715, + "step": 2265 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001766693907291423, + "loss": 0.1742, + "step": 2270 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017655693394912452, + "loss": 0.1868, + "step": 2275 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017644424276688441, + "loss": 0.1757, + "step": 2280 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017633131752746017, + "loss": 0.1803, + "step": 2285 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017621815857660656, + "loss": 0.1812, + "step": 2290 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017610476626079395, + "loss": 0.1691, + "step": 2295 + }, + { + "epoch": 0.75, + "learning_rate": 0.00017599114092720726, + "loss": 0.1662, + "step": 2300 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001758772829237448, + "loss": 0.1709, + "step": 2305 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001757631925990174, + "loss": 0.1822, + "step": 2310 + }, + { + "epoch": 0.75, + "learning_rate": 0.00017564887030234696, + "loss": 0.1732, + "step": 2315 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001755343163837659, + "loss": 0.1854, + "step": 2320 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001754195311940157, + "loss": 0.1768, + "step": 2325 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017530451508454594, + "loss": 0.1785, + "step": 2330 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017518926840751325, + "loss": 0.1724, + "step": 2335 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017507379151578017, + "loss": 0.1812, + "step": 2340 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017495808476291423, + "loss": 0.1754, + "step": 2345 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017484214850318664, + "loss": 0.1799, + "step": 2350 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001747259830915713, + "loss": 0.1769, + "step": 2355 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017460958888374385, + "loss": 0.1764, + "step": 2360 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001744929662360803, + "loss": 0.1716, + "step": 2365 + }, + { + "epoch": 0.77, + "learning_rate": 0.00017437611550565624, + "loss": 0.1699, + "step": 2370 + }, + { + "epoch": 0.77, + "learning_rate": 0.00017425903705024553, + "loss": 0.1661, + "step": 2375 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001741417312283193, + "loss": 0.1699, + "step": 2380 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001740241983990448, + "loss": 0.1682, + "step": 2385 + }, + { + "epoch": 0.77, + "learning_rate": 0.00017390643892228438, + "loss": 0.173, + "step": 2390 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001737884531585943, + "loss": 0.1852, + "step": 2395 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017367024146922372, + "loss": 0.1681, + "step": 2400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017355180421611347, + "loss": 0.1798, + "step": 2405 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017343314176189502, + "loss": 0.1705, + "step": 2410 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017331425446988946, + "loss": 0.177, + "step": 2415 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017319514270410614, + "loss": 0.1754, + "step": 2420 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017307580682924178, + "loss": 0.1679, + "step": 2425 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017295624721067932, + "loss": 0.1763, + "step": 2430 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017283646421448673, + "loss": 0.1769, + "step": 2435 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017271645820741583, + "loss": 0.1753, + "step": 2440 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001725962295569014, + "loss": 0.1715, + "step": 2445 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017247577863105983, + "loss": 0.1784, + "step": 2450 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017235510579868802, + "loss": 0.17, + "step": 2455 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017223421142926245, + "loss": 0.1686, + "step": 2460 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017211309589293774, + "loss": 0.166, + "step": 2465 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017199175956054584, + "loss": 0.1652, + "step": 2470 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017187020280359455, + "loss": 0.1732, + "step": 2475 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001717484259942668, + "loss": 0.1754, + "step": 2480 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017162642950541902, + "loss": 0.1748, + "step": 2485 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017150421371058049, + "loss": 0.1733, + "step": 2490 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017138177898395177, + "loss": 0.1737, + "step": 2495 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017125912570040383, + "loss": 0.175, + "step": 2500 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001711362542354769, + "loss": 0.173, + "step": 2505 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017101316496537905, + "loss": 0.1796, + "step": 2510 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017088985826698542, + "loss": 0.1722, + "step": 2515 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017076633451783675, + "loss": 0.1738, + "step": 2520 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017064259409613834, + "loss": 0.1743, + "step": 2525 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017051863738075902, + "loss": 0.1653, + "step": 2530 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001703944647512297, + "loss": 0.1761, + "step": 2535 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017027007658774256, + "loss": 0.1649, + "step": 2540 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017014547327114956, + "loss": 0.1738, + "step": 2545 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001700206551829614, + "loss": 0.1585, + "step": 2550 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016989562270534655, + "loss": 0.183, + "step": 2555 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001697703762211297, + "loss": 0.1693, + "step": 2560 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001696449161137909, + "loss": 0.173, + "step": 2565 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016951924276746425, + "loss": 0.1691, + "step": 2570 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016939335656693667, + "loss": 0.1698, + "step": 2575 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016926725789764691, + "loss": 0.1706, + "step": 2580 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016914094714568417, + "loss": 0.1691, + "step": 2585 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016901442469778713, + "loss": 0.1665, + "step": 2590 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016888769094134247, + "loss": 0.1831, + "step": 2595 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016876074626438395, + "loss": 0.1637, + "step": 2600 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016863359105559117, + "loss": 0.1757, + "step": 2605 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016850622570428826, + "loss": 0.1721, + "step": 2610 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016837865060044272, + "loss": 0.1647, + "step": 2615 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016825086613466442, + "loss": 0.1734, + "step": 2620 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016812287269820416, + "loss": 0.1847, + "step": 2625 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016799467068295257, + "loss": 0.1682, + "step": 2630 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001678662604814389, + "loss": 0.1741, + "step": 2635 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016773764248682988, + "loss": 0.1692, + "step": 2640 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016760881709292842, + "loss": 0.1805, + "step": 2645 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001674797846941724, + "loss": 0.1685, + "step": 2650 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016735054568563364, + "loss": 0.1673, + "step": 2655 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001672211004630164, + "loss": 0.1686, + "step": 2660 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016709144942265644, + "loss": 0.1749, + "step": 2665 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001669615929615197, + "loss": 0.1644, + "step": 2670 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016683153147720097, + "loss": 0.1673, + "step": 2675 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016670126536792287, + "loss": 0.1722, + "step": 2680 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001665707950325345, + "loss": 0.1588, + "step": 2685 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001664401208705103, + "loss": 0.164, + "step": 2690 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016630924328194877, + "loss": 0.1708, + "step": 2695 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016617816266757123, + "loss": 0.1711, + "step": 2700 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016604687942872067, + "loss": 0.1701, + "step": 2705 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016591539396736046, + "loss": 0.1708, + "step": 2710 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016578370668607307, + "loss": 0.1669, + "step": 2715 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016565181798805907, + "loss": 0.1601, + "step": 2720 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016551972827713555, + "loss": 0.1733, + "step": 2725 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016538743795773515, + "loss": 0.1641, + "step": 2730 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016525494743490474, + "loss": 0.1723, + "step": 2735 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016512225711430415, + "loss": 0.1655, + "step": 2740 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016498936740220492, + "loss": 0.172, + "step": 2745 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016485627870548917, + "loss": 0.1686, + "step": 2750 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016472299143164816, + "loss": 0.1709, + "step": 2755 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016458950598878126, + "loss": 0.1648, + "step": 2760 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016445582278559458, + "loss": 0.1641, + "step": 2765 + }, + { + "epoch": 0.9, + "learning_rate": 0.0001643219422313996, + "loss": 0.1662, + "step": 2770 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016418786473611227, + "loss": 0.1763, + "step": 2775 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016405359071025138, + "loss": 0.1773, + "step": 2780 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016391912056493743, + "loss": 0.1678, + "step": 2785 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016378445471189156, + "loss": 0.1632, + "step": 2790 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016364959356343398, + "loss": 0.1671, + "step": 2795 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016351453753248296, + "loss": 0.1739, + "step": 2800 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016337928703255336, + "loss": 0.1727, + "step": 2805 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016324384247775554, + "loss": 0.1756, + "step": 2810 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016310820428279397, + "loss": 0.1671, + "step": 2815 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016297237286296606, + "loss": 0.169, + "step": 2820 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016283634863416082, + "loss": 0.1693, + "step": 2825 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016270013201285757, + "loss": 0.164, + "step": 2830 + }, + { + "epoch": 0.92, + "learning_rate": 0.0001625637234161247, + "loss": 0.1663, + "step": 2835 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016242712326161845, + "loss": 0.1742, + "step": 2840 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016229033196758154, + "loss": 0.169, + "step": 2845 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016215334995284188, + "loss": 0.1703, + "step": 2850 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001620161776368114, + "loss": 0.1669, + "step": 2855 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001618788154394846, + "loss": 0.1634, + "step": 2860 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001617412637814375, + "loss": 0.163, + "step": 2865 + }, + { + "epoch": 0.93, + "learning_rate": 0.00016160352308382608, + "loss": 0.1651, + "step": 2870 + }, + { + "epoch": 0.93, + "learning_rate": 0.00016146559376838514, + "loss": 0.1657, + "step": 2875 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001613274762574271, + "loss": 0.1638, + "step": 2880 + }, + { + "epoch": 0.93, + "learning_rate": 0.00016118917097384048, + "loss": 0.1691, + "step": 2885 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016105067834108874, + "loss": 0.1627, + "step": 2890 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016091199878320902, + "loss": 0.1608, + "step": 2895 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016077313272481076, + "loss": 0.1617, + "step": 2900 + }, + { + "epoch": 0.94, + "learning_rate": 0.0001606340805910744, + "loss": 0.1696, + "step": 2905 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016049484280775012, + "loss": 0.1604, + "step": 2910 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016035541980115653, + "loss": 0.161, + "step": 2915 + }, + { + "epoch": 0.95, + "learning_rate": 0.00016021581199817943, + "loss": 0.1703, + "step": 2920 + }, + { + "epoch": 0.95, + "learning_rate": 0.0001600760198262703, + "loss": 0.1671, + "step": 2925 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015993604371344517, + "loss": 0.1603, + "step": 2930 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015979588408828333, + "loss": 0.1655, + "step": 2935 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015965554137992584, + "loss": 0.1675, + "step": 2940 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015951501601807437, + "loss": 0.1641, + "step": 2945 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015937430843298986, + "loss": 0.165, + "step": 2950 + }, + { + "epoch": 0.96, + "learning_rate": 0.0001592334190554912, + "loss": 0.1758, + "step": 2955 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015909234831695379, + "loss": 0.1703, + "step": 2960 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015895109664930846, + "loss": 0.1644, + "step": 2965 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015880966448503987, + "loss": 0.1638, + "step": 2970 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015866805225718542, + "loss": 0.1659, + "step": 2975 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015852626039933383, + "loss": 0.1676, + "step": 2980 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015838428934562373, + "loss": 0.1794, + "step": 2985 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015824213953074254, + "loss": 0.1562, + "step": 2990 + }, + { + "epoch": 0.97, + "learning_rate": 0.0001580998113899249, + "loss": 0.171, + "step": 2995 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015795730535895147, + "loss": 0.1681, + "step": 3000 + }, + { + "epoch": 0.97, + "learning_rate": 0.0001578146218741476, + "loss": 0.1617, + "step": 3005 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015767176137238196, + "loss": 0.1708, + "step": 3010 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015752872429106523, + "loss": 0.1636, + "step": 3015 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015738551106814872, + "loss": 0.1678, + "step": 3020 + }, + { + "epoch": 0.98, + "learning_rate": 0.000157242122142123, + "loss": 0.1612, + "step": 3025 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015709855795201675, + "loss": 0.1658, + "step": 3030 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015695481893739514, + "loss": 0.1621, + "step": 3035 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015681090553835864, + "loss": 0.1718, + "step": 3040 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015666681819554175, + "loss": 0.1712, + "step": 3045 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015652255735011135, + "loss": 0.1693, + "step": 3050 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015637812344376583, + "loss": 0.1714, + "step": 3055 + }, + { + "epoch": 0.99, + "learning_rate": 0.0001562335169187332, + "loss": 0.1661, + "step": 3060 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015608873821777012, + "loss": 0.1676, + "step": 3065 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015594378778416044, + "loss": 0.1654, + "step": 3070 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015579866606171372, + "loss": 0.1644, + "step": 3075 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015565337349476404, + "loss": 0.1744, + "step": 3080 + }, + { + "epoch": 1.0, + "learning_rate": 0.0001555079105281686, + "loss": 0.1606, + "step": 3085 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015536227760730632, + "loss": 0.1659, + "step": 3090 + }, + { + "epoch": 1.0, + "learning_rate": 0.0001552164751780764, + "loss": 0.1602, + "step": 3095 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015507050368689714, + "loss": 0.1636, + "step": 3100 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015492436358070438, + "loss": 0.1555, + "step": 3105 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015477805530695035, + "loss": 0.1661, + "step": 3110 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015463157931360207, + "loss": 0.1628, + "step": 3115 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015448493604914014, + "loss": 0.1696, + "step": 3120 + }, + { + "epoch": 1.01, + "learning_rate": 0.0001543381259625573, + "loss": 0.1685, + "step": 3125 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015419114950335704, + "loss": 0.1635, + "step": 3130 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015404400712155228, + "loss": 0.1631, + "step": 3135 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015389669926766395, + "loss": 0.1664, + "step": 3140 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015374922639271963, + "loss": 0.1617, + "step": 3145 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015360158894825216, + "loss": 0.174, + "step": 3150 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015345378738629826, + "loss": 0.1591, + "step": 3155 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015330582215939713, + "loss": 0.1729, + "step": 3160 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015315769372058907, + "loss": 0.1732, + "step": 3165 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015300940252341416, + "loss": 0.1608, + "step": 3170 + }, + { + "epoch": 1.03, + "learning_rate": 0.0001528609490219108, + "loss": 0.168, + "step": 3175 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015271233367061425, + "loss": 0.1731, + "step": 3180 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015256355692455542, + "loss": 0.1656, + "step": 3185 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015241461923925934, + "loss": 0.1591, + "step": 3190 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015226552107074385, + "loss": 0.1635, + "step": 3195 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015211626287551803, + "loss": 0.1622, + "step": 3200 + }, + { + "epoch": 1.04, + "learning_rate": 0.0001519668451105811, + "loss": 0.162, + "step": 3205 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015181726823342072, + "loss": 0.1614, + "step": 3210 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015166753270201176, + "loss": 0.1657, + "step": 3215 + }, + { + "epoch": 1.04, + "learning_rate": 0.0001515176389748149, + "loss": 0.1623, + "step": 3220 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015136758751077504, + "loss": 0.1629, + "step": 3225 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015121737876932034, + "loss": 0.1556, + "step": 3230 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015106701321036017, + "loss": 0.1651, + "step": 3235 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015091649129428424, + "loss": 0.1575, + "step": 3240 + }, + { + "epoch": 1.05, + "learning_rate": 0.000150765813481961, + "loss": 0.1594, + "step": 3245 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015061498023473608, + "loss": 0.1624, + "step": 3250 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015046399201443114, + "loss": 0.1616, + "step": 3255 + }, + { + "epoch": 1.06, + "learning_rate": 0.00015031284928334238, + "loss": 0.1693, + "step": 3260 + }, + { + "epoch": 1.06, + "learning_rate": 0.00015016155250423892, + "loss": 0.1592, + "step": 3265 + }, + { + "epoch": 1.06, + "learning_rate": 0.00015001010214036174, + "loss": 0.1542, + "step": 3270 + }, + { + "epoch": 1.06, + "learning_rate": 0.0001498584986554219, + "loss": 0.1631, + "step": 3275 + }, + { + "epoch": 1.06, + "learning_rate": 0.0001497067425135993, + "loss": 0.1612, + "step": 3280 + }, + { + "epoch": 1.06, + "learning_rate": 0.00014955483417954137, + "loss": 0.1551, + "step": 3285 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014940277411836146, + "loss": 0.155, + "step": 3290 + }, + { + "epoch": 1.07, + "learning_rate": 0.0001492505627956374, + "loss": 0.1593, + "step": 3295 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014909820067741028, + "loss": 0.1622, + "step": 3300 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014894568823018277, + "loss": 0.1637, + "step": 3305 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014879302592091798, + "loss": 0.1643, + "step": 3310 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014864021421703768, + "loss": 0.1642, + "step": 3315 + }, + { + "epoch": 1.08, + "learning_rate": 0.00014848725358642114, + "loss": 0.1607, + "step": 3320 + }, + { + "epoch": 1.08, + "learning_rate": 0.00014833414449740373, + "loss": 0.1591, + "step": 3325 + }, + { + "epoch": 1.08, + "learning_rate": 0.0001481808874187752, + "loss": 0.1682, + "step": 3330 + }, + { + "epoch": 1.08, + "learning_rate": 0.00014802748281977842, + "loss": 0.1572, + "step": 3335 + }, + { + "epoch": 1.08, + "learning_rate": 0.0001478739311701081, + "loss": 0.1595, + "step": 3340 + }, + { + "epoch": 1.08, + "learning_rate": 0.000147720232939909, + "loss": 0.1683, + "step": 3345 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014756638859977483, + "loss": 0.1539, + "step": 3350 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014741239862074655, + "loss": 0.1594, + "step": 3355 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014725826347431113, + "loss": 0.1658, + "step": 3360 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014710398363239994, + "loss": 0.1621, + "step": 3365 + }, + { + "epoch": 1.09, + "learning_rate": 0.0001469495595673874, + "loss": 0.1651, + "step": 3370 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014679499175208953, + "loss": 0.1655, + "step": 3375 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014664028065976246, + "loss": 0.1669, + "step": 3380 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014648542676410105, + "loss": 0.1593, + "step": 3385 + }, + { + "epoch": 1.1, + "learning_rate": 0.0001463304305392373, + "loss": 0.157, + "step": 3390 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014617529245973917, + "loss": 0.1593, + "step": 3395 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014602001300060877, + "loss": 0.1516, + "step": 3400 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014586459263728115, + "loss": 0.156, + "step": 3405 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014570903184562282, + "loss": 0.1637, + "step": 3410 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014555333110193022, + "loss": 0.1552, + "step": 3415 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014539749088292826, + "loss": 0.1624, + "step": 3420 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014524151166576897, + "loss": 0.1568, + "step": 3425 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014508539392802994, + "loss": 0.1583, + "step": 3430 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014492913814771283, + "loss": 0.1625, + "step": 3435 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014477274480324202, + "loss": 0.1563, + "step": 3440 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014461621437346307, + "loss": 0.1634, + "step": 3445 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014445954733764127, + "loss": 0.1452, + "step": 3450 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014430274417546012, + "loss": 0.1539, + "step": 3455 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014414580536702, + "loss": 0.1636, + "step": 3460 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014398873139283657, + "loss": 0.1509, + "step": 3465 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014383152273383934, + "loss": 0.1636, + "step": 3470 + }, + { + "epoch": 1.13, + "learning_rate": 0.00014367417987137015, + "loss": 0.1619, + "step": 3475 + }, + { + "epoch": 1.13, + "learning_rate": 0.00014351670328718183, + "loss": 0.1648, + "step": 3480 + }, + { + "epoch": 1.13, + "learning_rate": 0.00014335909346343658, + "loss": 0.1508, + "step": 3485 + }, + { + "epoch": 1.13, + "learning_rate": 0.0001432013508827046, + "loss": 0.1546, + "step": 3490 + }, + { + "epoch": 1.13, + "learning_rate": 0.0001430434760279625, + "loss": 0.1549, + "step": 3495 + }, + { + "epoch": 1.13, + "learning_rate": 0.000142885469382592, + "loss": 0.1412, + "step": 3500 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001427273314303782, + "loss": 0.1574, + "step": 3505 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001425690626555083, + "loss": 0.16, + "step": 3510 + }, + { + "epoch": 1.14, + "learning_rate": 0.00014241066354257008, + "loss": 0.1633, + "step": 3515 + }, + { + "epoch": 1.14, + "learning_rate": 0.00014225213457655034, + "loss": 0.1499, + "step": 3520 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001420934762428335, + "loss": 0.159, + "step": 3525 + }, + { + "epoch": 1.14, + "learning_rate": 0.00014193468902720007, + "loss": 0.1561, + "step": 3530 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014177577341582516, + "loss": 0.1619, + "step": 3535 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014161672989527698, + "loss": 0.16, + "step": 3540 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014145755895251546, + "loss": 0.148, + "step": 3545 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014129826107489054, + "loss": 0.1502, + "step": 3550 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014113883675014098, + "loss": 0.1427, + "step": 3555 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001409792864663925, + "loss": 0.1613, + "step": 3560 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001408196107121566, + "loss": 0.1597, + "step": 3565 + }, + { + "epoch": 1.16, + "learning_rate": 0.00014065980997632896, + "loss": 0.1501, + "step": 3570 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001404998847481879, + "loss": 0.1537, + "step": 3575 + }, + { + "epoch": 1.16, + "learning_rate": 0.00014033983551739286, + "loss": 0.1608, + "step": 3580 + }, + { + "epoch": 1.16, + "learning_rate": 0.00014017966277398304, + "loss": 0.1488, + "step": 3585 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001400193670083757, + "loss": 0.1593, + "step": 3590 + }, + { + "epoch": 1.16, + "learning_rate": 0.00013985894871136492, + "loss": 0.1525, + "step": 3595 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013969840837411982, + "loss": 0.1529, + "step": 3600 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013953774648818324, + "loss": 0.1489, + "step": 3605 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013937696354547022, + "loss": 0.144, + "step": 3610 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013921606003826627, + "loss": 0.1584, + "step": 3615 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013905503645922628, + "loss": 0.1657, + "step": 3620 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013889389330137264, + "loss": 0.1557, + "step": 3625 + }, + { + "epoch": 1.18, + "learning_rate": 0.00013873263105809388, + "loss": 0.1551, + "step": 3630 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001385712502231432, + "loss": 0.157, + "step": 3635 + }, + { + "epoch": 1.18, + "learning_rate": 0.00013840975129063684, + "loss": 0.1548, + "step": 3640 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001382481347550526, + "loss": 0.1565, + "step": 3645 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001380864011112285, + "loss": 0.1497, + "step": 3650 + }, + { + "epoch": 1.18, + "learning_rate": 0.000137924550854361, + "loss": 0.1569, + "step": 3655 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001377625844800037, + "loss": 0.1582, + "step": 3660 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013760050248406557, + "loss": 0.1571, + "step": 3665 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001374383053628098, + "loss": 0.1524, + "step": 3670 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013727599361285187, + "loss": 0.1567, + "step": 3675 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013711356773115838, + "loss": 0.1548, + "step": 3680 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013695102821504533, + "loss": 0.1557, + "step": 3685 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001367883755621766, + "loss": 0.148, + "step": 3690 + }, + { + "epoch": 1.2, + "learning_rate": 0.00013662561027056252, + "loss": 0.1604, + "step": 3695 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001364627328385583, + "loss": 0.15, + "step": 3700 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001362997437648625, + "loss": 0.1476, + "step": 3705 + }, + { + "epoch": 1.2, + "learning_rate": 0.00013613664354851547, + "loss": 0.1586, + "step": 3710 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001359734326888979, + "loss": 0.1536, + "step": 3715 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013581011168572923, + "loss": 0.1564, + "step": 3720 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013564668103906616, + "loss": 0.1534, + "step": 3725 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013548314124930104, + "loss": 0.1521, + "step": 3730 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013531949281716044, + "loss": 0.1489, + "step": 3735 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013515573624370357, + "loss": 0.1528, + "step": 3740 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001349918720303207, + "loss": 0.1405, + "step": 3745 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013482790067873178, + "loss": 0.1457, + "step": 3750 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013466382269098468, + "loss": 0.1554, + "step": 3755 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013449963856945383, + "loss": 0.1478, + "step": 3760 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013433534881683858, + "loss": 0.1633, + "step": 3765 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013417095393616178, + "loss": 0.15, + "step": 3770 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013400645443076804, + "loss": 0.1552, + "step": 3775 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013384185080432244, + "loss": 0.1485, + "step": 3780 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001336771435608088, + "loss": 0.1531, + "step": 3785 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013351233320452812, + "loss": 0.158, + "step": 3790 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013334742024009725, + "loss": 0.1554, + "step": 3795 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013318240517244713, + "loss": 0.1537, + "step": 3800 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001330172885068213, + "loss": 0.1584, + "step": 3805 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013285207074877447, + "loss": 0.1612, + "step": 3810 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013268675240417073, + "loss": 0.1449, + "step": 3815 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013252133397918226, + "loss": 0.1456, + "step": 3820 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001323558159802876, + "loss": 0.1541, + "step": 3825 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013219019891427025, + "loss": 0.1492, + "step": 3830 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013202448328821694, + "loss": 0.1528, + "step": 3835 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013185866960951622, + "loss": 0.1516, + "step": 3840 + }, + { + "epoch": 1.25, + "learning_rate": 0.0001316927583858569, + "loss": 0.1484, + "step": 3845 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013152675012522629, + "loss": 0.1509, + "step": 3850 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013136064533590897, + "loss": 0.1516, + "step": 3855 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013119444452648504, + "loss": 0.15, + "step": 3860 + }, + { + "epoch": 1.25, + "learning_rate": 0.0001310281482058286, + "loss": 0.1571, + "step": 3865 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013086175688310612, + "loss": 0.1526, + "step": 3870 + }, + { + "epoch": 1.26, + "learning_rate": 0.000130695271067775, + "loss": 0.1445, + "step": 3875 + }, + { + "epoch": 1.26, + "learning_rate": 0.00013052869126958198, + "loss": 0.1598, + "step": 3880 + }, + { + "epoch": 1.26, + "learning_rate": 0.00013036201799856149, + "loss": 0.1499, + "step": 3885 + }, + { + "epoch": 1.26, + "learning_rate": 0.0001301952517650342, + "loss": 0.1507, + "step": 3890 + }, + { + "epoch": 1.26, + "learning_rate": 0.0001300283930796054, + "loss": 0.1489, + "step": 3895 + }, + { + "epoch": 1.26, + "learning_rate": 0.00012986144245316346, + "loss": 0.1477, + "step": 3900 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012969440039687822, + "loss": 0.1528, + "step": 3905 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012952726742219953, + "loss": 0.1514, + "step": 3910 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012936004404085557, + "loss": 0.1482, + "step": 3915 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012919273076485133, + "loss": 0.1558, + "step": 3920 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012902532810646697, + "loss": 0.1539, + "step": 3925 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012885783657825647, + "loss": 0.1479, + "step": 3930 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012869025669304585, + "loss": 0.1449, + "step": 3935 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012852258896393158, + "loss": 0.1489, + "step": 3940 + }, + { + "epoch": 1.28, + "learning_rate": 0.0001283548339042792, + "loss": 0.1479, + "step": 3945 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012818699202772152, + "loss": 0.149, + "step": 3950 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012801906384815733, + "loss": 0.1472, + "step": 3955 + }, + { + "epoch": 1.28, + "learning_rate": 0.0001278510498797495, + "loss": 0.1526, + "step": 3960 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012768295063692376, + "loss": 0.1469, + "step": 3965 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012751476663436668, + "loss": 0.1576, + "step": 3970 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012734649838702455, + "loss": 0.1583, + "step": 3975 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012717814641010153, + "loss": 0.1434, + "step": 3980 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012700971121905812, + "loss": 0.1463, + "step": 3985 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012684119332960963, + "loss": 0.1543, + "step": 3990 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012667259325772464, + "loss": 0.1501, + "step": 3995 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012650391151962324, + "loss": 0.1475, + "step": 4000 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012633514863177566, + "loss": 0.147, + "step": 4005 + }, + { + "epoch": 1.3, + "learning_rate": 0.0001261663051109005, + "loss": 0.1454, + "step": 4010 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012599738147396332, + "loss": 0.1459, + "step": 4015 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012582837823817497, + "loss": 0.1479, + "step": 4020 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012565929592099002, + "loss": 0.1541, + "step": 4025 + }, + { + "epoch": 1.31, + "learning_rate": 0.00012549013504010512, + "loss": 0.145, + "step": 4030 + }, + { + "epoch": 1.31, + "learning_rate": 0.0001253208961134575, + "loss": 0.1467, + "step": 4035 + }, + { + "epoch": 1.31, + "learning_rate": 0.0001251515796592234, + "loss": 0.1497, + "step": 4040 + }, + { + "epoch": 1.31, + "learning_rate": 0.00012498218619581632, + "loss": 0.1439, + "step": 4045 + }, + { + "epoch": 1.31, + "learning_rate": 0.00012481271624188568, + "loss": 0.1419, + "step": 4050 + }, + { + "epoch": 1.31, + "learning_rate": 0.00012464317031631494, + "loss": 0.1518, + "step": 4055 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012447354893822034, + "loss": 0.1462, + "step": 4060 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012430385262694906, + "loss": 0.1473, + "step": 4065 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012413408190207768, + "loss": 0.1495, + "step": 4070 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012396423728341067, + "loss": 0.1566, + "step": 4075 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012379431929097873, + "loss": 0.1507, + "step": 4080 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012362432844503725, + "loss": 0.1544, + "step": 4085 + }, + { + "epoch": 1.33, + "learning_rate": 0.0001234542652660646, + "loss": 0.1528, + "step": 4090 + }, + { + "epoch": 1.33, + "learning_rate": 0.00012328413027476066, + "loss": 0.153, + "step": 4095 + }, + { + "epoch": 1.33, + "learning_rate": 0.00012311392399204526, + "loss": 0.1438, + "step": 4100 + }, + { + "epoch": 1.33, + "learning_rate": 0.0001229436469390564, + "loss": 0.1497, + "step": 4105 + }, + { + "epoch": 1.33, + "learning_rate": 0.00012277329963714882, + "loss": 0.1471, + "step": 4110 + }, + { + "epoch": 1.33, + "learning_rate": 0.0001226028826078923, + "loss": 0.1442, + "step": 4115 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012243239637307023, + "loss": 0.1455, + "step": 4120 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012226184145467775, + "loss": 0.1509, + "step": 4125 + }, + { + "epoch": 1.34, + "learning_rate": 0.0001220912183749204, + "loss": 0.1504, + "step": 4130 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012192052765621234, + "loss": 0.1524, + "step": 4135 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012174976982117488, + "loss": 0.1526, + "step": 4140 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012157894539263482, + "loss": 0.1501, + "step": 4145 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012140805489362281, + "loss": 0.1464, + "step": 4150 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012123709884737189, + "loss": 0.1481, + "step": 4155 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012106607777731567, + "loss": 0.1483, + "step": 4160 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012089499220708696, + "loss": 0.1456, + "step": 4165 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012072384266051595, + "loss": 0.1479, + "step": 4170 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012055262966162883, + "loss": 0.1488, + "step": 4175 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012038135373464602, + "loss": 0.1468, + "step": 4180 + }, + { + "epoch": 1.36, + "learning_rate": 0.00012021001540398058, + "loss": 0.1412, + "step": 4185 + }, + { + "epoch": 1.36, + "learning_rate": 0.00012003861519423662, + "loss": 0.1503, + "step": 4190 + }, + { + "epoch": 1.36, + "learning_rate": 0.00011986715363020784, + "loss": 0.1497, + "step": 4195 + }, + { + "epoch": 1.36, + "learning_rate": 0.00011969563123687566, + "loss": 0.1457, + "step": 4200 + }, + { + "epoch": 1.36, + "learning_rate": 0.00011952404853940782, + "loss": 0.1469, + "step": 4205 + }, + { + "epoch": 1.36, + "learning_rate": 0.00011935240606315666, + "loss": 0.1485, + "step": 4210 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011918070433365754, + "loss": 0.1413, + "step": 4215 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011900894387662731, + "loss": 0.1565, + "step": 4220 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011883712521796253, + "loss": 0.1516, + "step": 4225 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011866524888373811, + "loss": 0.146, + "step": 4230 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011849331540020543, + "loss": 0.1507, + "step": 4235 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011832132529379088, + "loss": 0.1493, + "step": 4240 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011814927909109419, + "loss": 0.1419, + "step": 4245 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011797717731888691, + "loss": 0.1414, + "step": 4250 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011780502050411069, + "loss": 0.1385, + "step": 4255 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011763280917387572, + "loss": 0.1473, + "step": 4260 + }, + { + "epoch": 1.38, + "learning_rate": 0.0001174605438554591, + "loss": 0.1496, + "step": 4265 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011728822507630326, + "loss": 0.1369, + "step": 4270 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011711585336401429, + "loss": 0.1437, + "step": 4275 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011694342924636031, + "loss": 0.1564, + "step": 4280 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011677095325126996, + "loss": 0.1443, + "step": 4285 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011659842590683074, + "loss": 0.1577, + "step": 4290 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011642584774128728, + "loss": 0.1416, + "step": 4295 + }, + { + "epoch": 1.39, + "learning_rate": 0.0001162532192830399, + "loss": 0.145, + "step": 4300 + }, + { + "epoch": 1.4, + "learning_rate": 0.00011608054106064281, + "loss": 0.1458, + "step": 4305 + }, + { + "epoch": 1.4, + "learning_rate": 0.0001159078136028027, + "loss": 0.1445, + "step": 4310 + }, + { + "epoch": 1.4, + "learning_rate": 0.00011573503743837693, + "loss": 0.139, + "step": 4315 + }, + { + "epoch": 1.4, + "learning_rate": 0.00011556221309637203, + "loss": 0.1441, + "step": 4320 + }, + { + "epoch": 1.4, + "learning_rate": 0.000115389341105942, + "loss": 0.1567, + "step": 4325 + }, + { + "epoch": 1.4, + "learning_rate": 0.0001152164219963868, + "loss": 0.1499, + "step": 4330 + }, + { + "epoch": 1.4, + "learning_rate": 0.00011504345629715057, + "loss": 0.15, + "step": 4335 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011487044453782016, + "loss": 0.1449, + "step": 4340 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011469738724812346, + "loss": 0.1423, + "step": 4345 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011452428495792773, + "loss": 0.1468, + "step": 4350 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011435113819723797, + "loss": 0.1519, + "step": 4355 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011417794749619544, + "loss": 0.1498, + "step": 4360 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011400471338507585, + "loss": 0.1406, + "step": 4365 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011383143639428791, + "loss": 0.1485, + "step": 4370 + }, + { + "epoch": 1.42, + "learning_rate": 0.0001136581170543715, + "loss": 0.1502, + "step": 4375 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011348475589599625, + "loss": 0.139, + "step": 4380 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011331135344995985, + "loss": 0.1485, + "step": 4385 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011313791024718634, + "loss": 0.133, + "step": 4390 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011296442681872457, + "loss": 0.1473, + "step": 4395 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011279090369574658, + "loss": 0.1401, + "step": 4400 + }, + { + "epoch": 1.43, + "learning_rate": 0.0001126173414095459, + "loss": 0.1483, + "step": 4405 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011244374049153603, + "loss": 0.1436, + "step": 4410 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011227010147324868, + "loss": 0.1445, + "step": 4415 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011209642488633227, + "loss": 0.1451, + "step": 4420 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011192271126255024, + "loss": 0.1438, + "step": 4425 + }, + { + "epoch": 1.44, + "learning_rate": 0.00011174896113377945, + "loss": 0.1474, + "step": 4430 + }, + { + "epoch": 1.44, + "learning_rate": 0.0001115751750320084, + "loss": 0.1444, + "step": 4435 + }, + { + "epoch": 1.44, + "learning_rate": 0.00011140135348933593, + "loss": 0.147, + "step": 4440 + }, + { + "epoch": 1.44, + "learning_rate": 0.00011122749703796929, + "loss": 0.1447, + "step": 4445 + }, + { + "epoch": 1.44, + "learning_rate": 0.0001110536062102226, + "loss": 0.1423, + "step": 4450 + }, + { + "epoch": 1.44, + "learning_rate": 0.00011087968153851525, + "loss": 0.1405, + "step": 4455 + }, + { + "epoch": 1.45, + "learning_rate": 0.00011070572355537026, + "loss": 0.1468, + "step": 4460 + }, + { + "epoch": 1.45, + "learning_rate": 0.00011053173279341265, + "loss": 0.1442, + "step": 4465 + }, + { + "epoch": 1.45, + "learning_rate": 0.0001103577097853678, + "loss": 0.1447, + "step": 4470 + }, + { + "epoch": 1.45, + "learning_rate": 0.00011018365506405978, + "loss": 0.1514, + "step": 4475 + }, + { + "epoch": 1.45, + "learning_rate": 0.00011000956916240985, + "loss": 0.148, + "step": 4480 + }, + { + "epoch": 1.45, + "learning_rate": 0.00010983545261343464, + "loss": 0.1499, + "step": 4485 + }, + { + "epoch": 1.45, + "learning_rate": 0.00010966130595024466, + "loss": 0.1398, + "step": 4490 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010948712970604266, + "loss": 0.1379, + "step": 4495 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010931292441412191, + "loss": 0.1438, + "step": 4500 + }, + { + "epoch": 1.46, + "learning_rate": 0.0001091386906078646, + "loss": 0.1453, + "step": 4505 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010896442882074029, + "loss": 0.1439, + "step": 4510 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010879013958630418, + "loss": 0.1436, + "step": 4515 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010861582343819548, + "loss": 0.144, + "step": 4520 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010844148091013581, + "loss": 0.1439, + "step": 4525 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010826711253592768, + "loss": 0.1441, + "step": 4530 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010809271884945256, + "loss": 0.1421, + "step": 4535 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010791830038466952, + "loss": 0.1476, + "step": 4540 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010774385767561347, + "loss": 0.1484, + "step": 4545 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010756939125639358, + "loss": 0.1437, + "step": 4550 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010739490166119155, + "loss": 0.1403, + "step": 4555 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010722038942426014, + "loss": 0.1437, + "step": 4560 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010704585507992131, + "loss": 0.1429, + "step": 4565 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010687129916256482, + "loss": 0.1401, + "step": 4570 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010669672220664644, + "loss": 0.1448, + "step": 4575 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010652212474668635, + "loss": 0.1449, + "step": 4580 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010634750731726748, + "loss": 0.1431, + "step": 4585 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010617287045303395, + "loss": 0.1467, + "step": 4590 + }, + { + "epoch": 1.49, + "learning_rate": 0.0001059982146886894, + "loss": 0.1403, + "step": 4595 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010582354055899528, + "loss": 0.1467, + "step": 4600 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010564884859876934, + "loss": 0.1399, + "step": 4605 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010547413934288386, + "loss": 0.1408, + "step": 4610 + }, + { + "epoch": 1.5, + "learning_rate": 0.0001052994133262641, + "loss": 0.1411, + "step": 4615 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010512467108388661, + "loss": 0.1398, + "step": 4620 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010494991315077772, + "loss": 0.1444, + "step": 4625 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010477514006201166, + "loss": 0.1438, + "step": 4630 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010460035235270918, + "loss": 0.1499, + "step": 4635 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010442555055803572, + "loss": 0.1451, + "step": 4640 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010425073521319986, + "loss": 0.15, + "step": 4645 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010407590685345174, + "loss": 0.1471, + "step": 4650 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010390106601408125, + "loss": 0.1454, + "step": 4655 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010372621323041656, + "loss": 0.1476, + "step": 4660 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010355134903782235, + "loss": 0.142, + "step": 4665 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010337647397169832, + "loss": 0.1391, + "step": 4670 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010320158856747737, + "loss": 0.1398, + "step": 4675 + }, + { + "epoch": 1.52, + "learning_rate": 0.0001030266933606241, + "loss": 0.15, + "step": 4680 + }, + { + "epoch": 1.52, + "learning_rate": 0.0001028517888866331, + "loss": 0.1387, + "step": 4685 + }, + { + "epoch": 1.52, + "learning_rate": 0.00010267687568102737, + "loss": 0.1484, + "step": 4690 + }, + { + "epoch": 1.52, + "learning_rate": 0.00010250195427935663, + "loss": 0.1483, + "step": 4695 + }, + { + "epoch": 1.52, + "learning_rate": 0.00010232702521719564, + "loss": 0.1452, + "step": 4700 + }, + { + "epoch": 1.52, + "learning_rate": 0.00010215208903014272, + "loss": 0.1481, + "step": 4705 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010197714625381795, + "loss": 0.1406, + "step": 4710 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010180219742386157, + "loss": 0.139, + "step": 4715 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010162724307593234, + "loss": 0.1354, + "step": 4720 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010145228374570599, + "loss": 0.1379, + "step": 4725 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010127731996887339, + "loss": 0.1481, + "step": 4730 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010110235228113918, + "loss": 0.1464, + "step": 4735 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010092738121821985, + "loss": 0.1426, + "step": 4740 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010075240731584223, + "loss": 0.1415, + "step": 4745 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010057743110974194, + "loss": 0.1393, + "step": 4750 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010040245313566153, + "loss": 0.1408, + "step": 4755 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010022747392934905, + "loss": 0.1358, + "step": 4760 + }, + { + "epoch": 1.54, + "learning_rate": 0.0001000524940265563, + "loss": 0.1401, + "step": 4765 + }, + { + "epoch": 1.55, + "learning_rate": 9.98775139630372e-05, + "loss": 0.1379, + "step": 4770 + }, + { + "epoch": 1.55, + "learning_rate": 9.970253427454616e-05, + "loss": 0.1425, + "step": 4775 + }, + { + "epoch": 1.55, + "learning_rate": 9.95275554968365e-05, + "loss": 0.1475, + "step": 4780 + }, + { + "epoch": 1.55, + "learning_rate": 9.935257816565861e-05, + "loss": 0.1364, + "step": 4785 + }, + { + "epoch": 1.55, + "learning_rate": 9.917760281675866e-05, + "loss": 0.1443, + "step": 4790 + }, + { + "epoch": 1.55, + "learning_rate": 9.900262998587653e-05, + "loss": 0.1388, + "step": 4795 + }, + { + "epoch": 1.56, + "learning_rate": 9.882766020874456e-05, + "loss": 0.1375, + "step": 4800 + }, + { + "epoch": 1.56, + "learning_rate": 9.865269402108565e-05, + "loss": 0.1413, + "step": 4805 + }, + { + "epoch": 1.56, + "learning_rate": 9.847773195861175e-05, + "loss": 0.1499, + "step": 4810 + }, + { + "epoch": 1.56, + "learning_rate": 9.830277455702213e-05, + "loss": 0.1421, + "step": 4815 + }, + { + "epoch": 1.56, + "learning_rate": 9.812782235200189e-05, + "loss": 0.146, + "step": 4820 + }, + { + "epoch": 1.56, + "learning_rate": 9.795287587922006e-05, + "loss": 0.1343, + "step": 4825 + }, + { + "epoch": 1.57, + "learning_rate": 9.777793567432828e-05, + "loss": 0.1451, + "step": 4830 + }, + { + "epoch": 1.57, + "learning_rate": 9.760300227295894e-05, + "loss": 0.1438, + "step": 4835 + }, + { + "epoch": 1.57, + "learning_rate": 9.742807621072353e-05, + "loss": 0.1436, + "step": 4840 + }, + { + "epoch": 1.57, + "learning_rate": 9.725315802321121e-05, + "loss": 0.1384, + "step": 4845 + }, + { + "epoch": 1.57, + "learning_rate": 9.707824824598684e-05, + "loss": 0.1368, + "step": 4850 + }, + { + "epoch": 1.57, + "learning_rate": 9.690334741458974e-05, + "loss": 0.1309, + "step": 4855 + }, + { + "epoch": 1.57, + "learning_rate": 9.672845606453168e-05, + "loss": 0.1384, + "step": 4860 + }, + { + "epoch": 1.58, + "learning_rate": 9.655357473129548e-05, + "loss": 0.1387, + "step": 4865 + }, + { + "epoch": 1.58, + "learning_rate": 9.637870395033324e-05, + "loss": 0.1426, + "step": 4870 + }, + { + "epoch": 1.58, + "learning_rate": 9.620384425706484e-05, + "loss": 0.1407, + "step": 4875 + }, + { + "epoch": 1.58, + "learning_rate": 9.602899618687606e-05, + "loss": 0.1413, + "step": 4880 + }, + { + "epoch": 1.58, + "learning_rate": 9.585416027511727e-05, + "loss": 0.1434, + "step": 4885 + }, + { + "epoch": 1.58, + "learning_rate": 9.567933705710143e-05, + "loss": 0.146, + "step": 4890 + }, + { + "epoch": 1.59, + "learning_rate": 9.550452706810281e-05, + "loss": 0.1384, + "step": 4895 + }, + { + "epoch": 1.59, + "learning_rate": 9.532973084335511e-05, + "loss": 0.1447, + "step": 4900 + }, + { + "epoch": 1.59, + "learning_rate": 9.515494891804979e-05, + "loss": 0.1418, + "step": 4905 + }, + { + "epoch": 1.59, + "learning_rate": 9.498018182733466e-05, + "loss": 0.144, + "step": 4910 + }, + { + "epoch": 1.59, + "learning_rate": 9.480543010631207e-05, + "loss": 0.1449, + "step": 4915 + }, + { + "epoch": 1.59, + "learning_rate": 9.463069429003728e-05, + "loss": 0.1391, + "step": 4920 + }, + { + "epoch": 1.6, + "learning_rate": 9.445597491351685e-05, + "loss": 0.1484, + "step": 4925 + }, + { + "epoch": 1.6, + "learning_rate": 9.42812725117071e-05, + "loss": 0.1402, + "step": 4930 + }, + { + "epoch": 1.6, + "learning_rate": 9.410658761951221e-05, + "loss": 0.1478, + "step": 4935 + }, + { + "epoch": 1.6, + "learning_rate": 9.393192077178295e-05, + "loss": 0.1431, + "step": 4940 + }, + { + "epoch": 1.6, + "learning_rate": 9.375727250331462e-05, + "loss": 0.1405, + "step": 4945 + }, + { + "epoch": 1.6, + "learning_rate": 9.358264334884585e-05, + "loss": 0.1359, + "step": 4950 + }, + { + "epoch": 1.61, + "learning_rate": 9.340803384305657e-05, + "loss": 0.1464, + "step": 4955 + }, + { + "epoch": 1.61, + "learning_rate": 9.323344452056668e-05, + "loss": 0.1408, + "step": 4960 + }, + { + "epoch": 1.61, + "learning_rate": 9.305887591593419e-05, + "loss": 0.1435, + "step": 4965 + }, + { + "epoch": 1.61, + "learning_rate": 9.288432856365374e-05, + "loss": 0.1449, + "step": 4970 + }, + { + "epoch": 1.61, + "learning_rate": 9.270980299815486e-05, + "loss": 0.1403, + "step": 4975 + }, + { + "epoch": 1.61, + "learning_rate": 9.253529975380039e-05, + "loss": 0.1434, + "step": 4980 + }, + { + "epoch": 1.62, + "learning_rate": 9.236081936488485e-05, + "loss": 0.1405, + "step": 4985 + }, + { + "epoch": 1.62, + "learning_rate": 9.21863623656327e-05, + "loss": 0.1394, + "step": 4990 + }, + { + "epoch": 1.62, + "learning_rate": 9.20119292901969e-05, + "loss": 0.1352, + "step": 4995 + }, + { + "epoch": 1.62, + "learning_rate": 9.183752067265703e-05, + "loss": 0.1452, + "step": 5000 + }, + { + "epoch": 1.62, + "learning_rate": 9.166313704701795e-05, + "loss": 0.1423, + "step": 5005 + }, + { + "epoch": 1.62, + "learning_rate": 9.14887789472078e-05, + "loss": 0.1378, + "step": 5010 + }, + { + "epoch": 1.63, + "learning_rate": 9.131444690707673e-05, + "loss": 0.1379, + "step": 5015 + }, + { + "epoch": 1.63, + "learning_rate": 9.114014146039503e-05, + "loss": 0.137, + "step": 5020 + }, + { + "epoch": 1.63, + "learning_rate": 9.096586314085162e-05, + "loss": 0.1459, + "step": 5025 + }, + { + "epoch": 1.63, + "learning_rate": 9.079161248205222e-05, + "loss": 0.1417, + "step": 5030 + }, + { + "epoch": 1.63, + "learning_rate": 9.061739001751803e-05, + "loss": 0.139, + "step": 5035 + }, + { + "epoch": 1.63, + "learning_rate": 9.044319628068387e-05, + "loss": 0.1322, + "step": 5040 + }, + { + "epoch": 1.63, + "learning_rate": 9.02690318048965e-05, + "loss": 0.1324, + "step": 5045 + }, + { + "epoch": 1.64, + "learning_rate": 9.009489712341327e-05, + "loss": 0.1403, + "step": 5050 + }, + { + "epoch": 1.64, + "learning_rate": 8.992079276940011e-05, + "loss": 0.1394, + "step": 5055 + }, + { + "epoch": 1.64, + "learning_rate": 8.974671927593028e-05, + "loss": 0.1389, + "step": 5060 + }, + { + "epoch": 1.64, + "learning_rate": 8.957267717598236e-05, + "loss": 0.141, + "step": 5065 + }, + { + "epoch": 1.64, + "learning_rate": 8.939866700243901e-05, + "loss": 0.1334, + "step": 5070 + }, + { + "epoch": 1.64, + "learning_rate": 8.922468928808497e-05, + "loss": 0.1311, + "step": 5075 + }, + { + "epoch": 1.65, + "learning_rate": 8.905074456560574e-05, + "loss": 0.1439, + "step": 5080 + }, + { + "epoch": 1.65, + "learning_rate": 8.887683336758563e-05, + "loss": 0.1424, + "step": 5085 + }, + { + "epoch": 1.65, + "learning_rate": 8.870295622650651e-05, + "loss": 0.1476, + "step": 5090 + }, + { + "epoch": 1.65, + "learning_rate": 8.852911367474577e-05, + "loss": 0.1436, + "step": 5095 + }, + { + "epoch": 1.65, + "learning_rate": 8.835530624457503e-05, + "loss": 0.1442, + "step": 5100 + }, + { + "epoch": 1.65, + "learning_rate": 8.81815344681584e-05, + "loss": 0.1467, + "step": 5105 + }, + { + "epoch": 1.66, + "learning_rate": 8.800779887755066e-05, + "loss": 0.1409, + "step": 5110 + }, + { + "epoch": 1.66, + "learning_rate": 8.783410000469601e-05, + "loss": 0.1487, + "step": 5115 + }, + { + "epoch": 1.66, + "learning_rate": 8.7660438381426e-05, + "loss": 0.1463, + "step": 5120 + }, + { + "epoch": 1.66, + "learning_rate": 8.748681453945833e-05, + "loss": 0.1398, + "step": 5125 + }, + { + "epoch": 1.66, + "learning_rate": 8.73132290103949e-05, + "loss": 0.1358, + "step": 5130 + }, + { + "epoch": 1.66, + "learning_rate": 8.713968232572039e-05, + "loss": 0.143, + "step": 5135 + }, + { + "epoch": 1.67, + "learning_rate": 8.69661750168004e-05, + "loss": 0.137, + "step": 5140 + }, + { + "epoch": 1.67, + "learning_rate": 8.679270761488019e-05, + "loss": 0.1393, + "step": 5145 + }, + { + "epoch": 1.67, + "learning_rate": 8.661928065108259e-05, + "loss": 0.1472, + "step": 5150 + }, + { + "epoch": 1.67, + "learning_rate": 8.644589465640684e-05, + "loss": 0.1389, + "step": 5155 + }, + { + "epoch": 1.67, + "learning_rate": 8.627255016172652e-05, + "loss": 0.1487, + "step": 5160 + }, + { + "epoch": 1.67, + "learning_rate": 8.609924769778835e-05, + "loss": 0.1403, + "step": 5165 + }, + { + "epoch": 1.68, + "learning_rate": 8.59259877952103e-05, + "loss": 0.1351, + "step": 5170 + }, + { + "epoch": 1.68, + "learning_rate": 8.575277098447987e-05, + "loss": 0.1406, + "step": 5175 + }, + { + "epoch": 1.68, + "learning_rate": 8.557959779595286e-05, + "loss": 0.1378, + "step": 5180 + }, + { + "epoch": 1.68, + "learning_rate": 8.540646875985134e-05, + "loss": 0.1352, + "step": 5185 + }, + { + "epoch": 1.68, + "learning_rate": 8.523338440626232e-05, + "loss": 0.1406, + "step": 5190 + }, + { + "epoch": 1.68, + "learning_rate": 8.50603452651358e-05, + "loss": 0.1432, + "step": 5195 + }, + { + "epoch": 1.69, + "learning_rate": 8.488735186628364e-05, + "loss": 0.1433, + "step": 5200 + }, + { + "epoch": 1.69, + "learning_rate": 8.471440473937732e-05, + "loss": 0.1368, + "step": 5205 + }, + { + "epoch": 1.69, + "learning_rate": 8.454150441394697e-05, + "loss": 0.136, + "step": 5210 + }, + { + "epoch": 1.69, + "learning_rate": 8.436865141937912e-05, + "loss": 0.1502, + "step": 5215 + }, + { + "epoch": 1.69, + "learning_rate": 8.419584628491562e-05, + "loss": 0.1395, + "step": 5220 + }, + { + "epoch": 1.69, + "learning_rate": 8.402308953965165e-05, + "loss": 0.1294, + "step": 5225 + }, + { + "epoch": 1.69, + "learning_rate": 8.385038171253424e-05, + "loss": 0.1438, + "step": 5230 + }, + { + "epoch": 1.7, + "learning_rate": 8.367772333236073e-05, + "loss": 0.1371, + "step": 5235 + }, + { + "epoch": 1.7, + "learning_rate": 8.350511492777698e-05, + "loss": 0.138, + "step": 5240 + }, + { + "epoch": 1.7, + "learning_rate": 8.333255702727588e-05, + "loss": 0.1389, + "step": 5245 + }, + { + "epoch": 1.7, + "learning_rate": 8.316005015919559e-05, + "loss": 0.1384, + "step": 5250 + }, + { + "epoch": 1.7, + "learning_rate": 8.298759485171819e-05, + "loss": 0.1384, + "step": 5255 + }, + { + "epoch": 1.7, + "learning_rate": 8.281519163286772e-05, + "loss": 0.1307, + "step": 5260 + }, + { + "epoch": 1.71, + "learning_rate": 8.264284103050888e-05, + "loss": 0.1413, + "step": 5265 + }, + { + "epoch": 1.71, + "learning_rate": 8.247054357234511e-05, + "loss": 0.1322, + "step": 5270 + }, + { + "epoch": 1.71, + "learning_rate": 8.229829978591729e-05, + "loss": 0.1433, + "step": 5275 + }, + { + "epoch": 1.71, + "learning_rate": 8.212611019860188e-05, + "loss": 0.1384, + "step": 5280 + }, + { + "epoch": 1.71, + "learning_rate": 8.195397533760943e-05, + "loss": 0.1396, + "step": 5285 + }, + { + "epoch": 1.71, + "learning_rate": 8.178189572998284e-05, + "loss": 0.1403, + "step": 5290 + }, + { + "epoch": 1.72, + "learning_rate": 8.160987190259605e-05, + "loss": 0.1429, + "step": 5295 + }, + { + "epoch": 1.72, + "learning_rate": 8.14379043821519e-05, + "loss": 0.1384, + "step": 5300 + }, + { + "epoch": 1.72, + "learning_rate": 8.12659936951811e-05, + "loss": 0.1422, + "step": 5305 + }, + { + "epoch": 1.72, + "learning_rate": 8.109414036804026e-05, + "loss": 0.1355, + "step": 5310 + }, + { + "epoch": 1.72, + "learning_rate": 8.092234492691027e-05, + "loss": 0.1357, + "step": 5315 + }, + { + "epoch": 1.72, + "learning_rate": 8.075060789779494e-05, + "loss": 0.1388, + "step": 5320 + }, + { + "epoch": 1.73, + "learning_rate": 8.057892980651909e-05, + "loss": 0.1349, + "step": 5325 + }, + { + "epoch": 1.73, + "learning_rate": 8.040731117872721e-05, + "loss": 0.1328, + "step": 5330 + }, + { + "epoch": 1.73, + "learning_rate": 8.023575253988165e-05, + "loss": 0.1301, + "step": 5335 + }, + { + "epoch": 1.73, + "learning_rate": 8.00642544152611e-05, + "loss": 0.1376, + "step": 5340 + }, + { + "epoch": 1.73, + "learning_rate": 7.989281732995895e-05, + "loss": 0.1389, + "step": 5345 + }, + { + "epoch": 1.73, + "learning_rate": 7.972144180888177e-05, + "loss": 0.1337, + "step": 5350 + }, + { + "epoch": 1.74, + "learning_rate": 7.955012837674753e-05, + "loss": 0.141, + "step": 5355 + }, + { + "epoch": 1.74, + "learning_rate": 7.93788775580842e-05, + "loss": 0.1456, + "step": 5360 + }, + { + "epoch": 1.74, + "learning_rate": 7.920768987722791e-05, + "loss": 0.1459, + "step": 5365 + }, + { + "epoch": 1.74, + "learning_rate": 7.903656585832162e-05, + "loss": 0.1332, + "step": 5370 + }, + { + "epoch": 1.74, + "learning_rate": 7.886550602531331e-05, + "loss": 0.1445, + "step": 5375 + }, + { + "epoch": 1.74, + "learning_rate": 7.869451090195437e-05, + "loss": 0.1349, + "step": 5380 + }, + { + "epoch": 1.75, + "learning_rate": 7.852358101179817e-05, + "loss": 0.1327, + "step": 5385 + }, + { + "epoch": 1.75, + "learning_rate": 7.835271687819827e-05, + "loss": 0.1305, + "step": 5390 + }, + { + "epoch": 1.75, + "learning_rate": 7.818191902430696e-05, + "loss": 0.1446, + "step": 5395 + }, + { + "epoch": 1.75, + "learning_rate": 7.801118797307349e-05, + "loss": 0.139, + "step": 5400 + }, + { + "epoch": 1.75, + "learning_rate": 7.784052424724276e-05, + "loss": 0.1432, + "step": 5405 + }, + { + "epoch": 1.75, + "learning_rate": 7.766992836935327e-05, + "loss": 0.1439, + "step": 5410 + }, + { + "epoch": 1.75, + "learning_rate": 7.749940086173607e-05, + "loss": 0.1415, + "step": 5415 + }, + { + "epoch": 1.76, + "learning_rate": 7.732894224651261e-05, + "loss": 0.1404, + "step": 5420 + }, + { + "epoch": 1.76, + "learning_rate": 7.715855304559362e-05, + "loss": 0.1415, + "step": 5425 + }, + { + "epoch": 1.76, + "learning_rate": 7.698823378067714e-05, + "loss": 0.142, + "step": 5430 + }, + { + "epoch": 1.76, + "learning_rate": 7.681798497324716e-05, + "loss": 0.1428, + "step": 5435 + }, + { + "epoch": 1.76, + "learning_rate": 7.664780714457198e-05, + "loss": 0.1405, + "step": 5440 + }, + { + "epoch": 1.76, + "learning_rate": 7.647770081570246e-05, + "loss": 0.1355, + "step": 5445 + }, + { + "epoch": 1.77, + "learning_rate": 7.630766650747066e-05, + "loss": 0.1371, + "step": 5450 + }, + { + "epoch": 1.77, + "learning_rate": 7.613770474048802e-05, + "loss": 0.1388, + "step": 5455 + }, + { + "epoch": 1.77, + "learning_rate": 7.596781603514398e-05, + "loss": 0.1306, + "step": 5460 + }, + { + "epoch": 1.77, + "learning_rate": 7.579800091160417e-05, + "loss": 0.1351, + "step": 5465 + }, + { + "epoch": 1.77, + "learning_rate": 7.562825988980904e-05, + "loss": 0.1322, + "step": 5470 + }, + { + "epoch": 1.77, + "learning_rate": 7.545859348947204e-05, + "loss": 0.1389, + "step": 5475 + }, + { + "epoch": 1.78, + "learning_rate": 7.528900223007824e-05, + "loss": 0.1432, + "step": 5480 + }, + { + "epoch": 1.78, + "learning_rate": 7.511948663088261e-05, + "loss": 0.1359, + "step": 5485 + }, + { + "epoch": 1.78, + "learning_rate": 7.495004721090843e-05, + "loss": 0.1411, + "step": 5490 + }, + { + "epoch": 1.78, + "learning_rate": 7.478068448894577e-05, + "loss": 0.1373, + "step": 5495 + }, + { + "epoch": 1.78, + "learning_rate": 7.461139898354986e-05, + "loss": 0.1406, + "step": 5500 + }, + { + "epoch": 1.78, + "learning_rate": 7.444219121303957e-05, + "loss": 0.1388, + "step": 5505 + }, + { + "epoch": 1.79, + "learning_rate": 7.42730616954956e-05, + "loss": 0.1344, + "step": 5510 + }, + { + "epoch": 1.79, + "learning_rate": 7.410401094875921e-05, + "loss": 0.1416, + "step": 5515 + }, + { + "epoch": 1.79, + "learning_rate": 7.393503949043039e-05, + "loss": 0.1417, + "step": 5520 + }, + { + "epoch": 1.79, + "learning_rate": 7.376614783786646e-05, + "loss": 0.1425, + "step": 5525 + }, + { + "epoch": 1.79, + "learning_rate": 7.359733650818024e-05, + "loss": 0.1326, + "step": 5530 + }, + { + "epoch": 1.79, + "learning_rate": 7.342860601823875e-05, + "loss": 0.1449, + "step": 5535 + }, + { + "epoch": 1.8, + "learning_rate": 7.325995688466146e-05, + "loss": 0.1339, + "step": 5540 + }, + { + "epoch": 1.8, + "learning_rate": 7.309138962381872e-05, + "loss": 0.1341, + "step": 5545 + }, + { + "epoch": 1.8, + "learning_rate": 7.29229047518302e-05, + "loss": 0.1324, + "step": 5550 + }, + { + "epoch": 1.8, + "learning_rate": 7.275450278456335e-05, + "loss": 0.129, + "step": 5555 + }, + { + "epoch": 1.8, + "learning_rate": 7.258618423763171e-05, + "loss": 0.1388, + "step": 5560 + }, + { + "epoch": 1.8, + "learning_rate": 7.24179496263935e-05, + "loss": 0.1406, + "step": 5565 + }, + { + "epoch": 1.8, + "learning_rate": 7.224979946594992e-05, + "loss": 0.138, + "step": 5570 + }, + { + "epoch": 1.81, + "learning_rate": 7.20817342711435e-05, + "loss": 0.1412, + "step": 5575 + }, + { + "epoch": 1.81, + "learning_rate": 7.191375455655676e-05, + "loss": 0.1377, + "step": 5580 + }, + { + "epoch": 1.81, + "learning_rate": 7.174586083651038e-05, + "loss": 0.1387, + "step": 5585 + }, + { + "epoch": 1.81, + "learning_rate": 7.157805362506184e-05, + "loss": 0.1352, + "step": 5590 + }, + { + "epoch": 1.81, + "learning_rate": 7.14103334360037e-05, + "loss": 0.1457, + "step": 5595 + }, + { + "epoch": 1.81, + "learning_rate": 7.124270078286203e-05, + "loss": 0.1359, + "step": 5600 + }, + { + "epoch": 1.82, + "learning_rate": 7.107515617889498e-05, + "loss": 0.138, + "step": 5605 + }, + { + "epoch": 1.82, + "learning_rate": 7.090770013709105e-05, + "loss": 0.1402, + "step": 5610 + }, + { + "epoch": 1.82, + "learning_rate": 7.074033317016755e-05, + "loss": 0.133, + "step": 5615 + }, + { + "epoch": 1.82, + "learning_rate": 7.057305579056919e-05, + "loss": 0.1419, + "step": 5620 + }, + { + "epoch": 1.82, + "learning_rate": 7.040586851046618e-05, + "loss": 0.1282, + "step": 5625 + }, + { + "epoch": 1.82, + "learning_rate": 7.023877184175307e-05, + "loss": 0.1393, + "step": 5630 + }, + { + "epoch": 1.83, + "learning_rate": 7.007176629604681e-05, + "loss": 0.1273, + "step": 5635 + }, + { + "epoch": 1.83, + "learning_rate": 6.990485238468545e-05, + "loss": 0.1417, + "step": 5640 + }, + { + "epoch": 1.83, + "learning_rate": 6.973803061872646e-05, + "loss": 0.1376, + "step": 5645 + }, + { + "epoch": 1.83, + "learning_rate": 6.957130150894516e-05, + "loss": 0.1421, + "step": 5650 + }, + { + "epoch": 1.83, + "learning_rate": 6.940466556583316e-05, + "loss": 0.1311, + "step": 5655 + }, + { + "epoch": 1.83, + "learning_rate": 6.923812329959683e-05, + "loss": 0.1376, + "step": 5660 + }, + { + "epoch": 1.84, + "learning_rate": 6.907167522015577e-05, + "loss": 0.1375, + "step": 5665 + }, + { + "epoch": 1.84, + "learning_rate": 6.890532183714105e-05, + "loss": 0.1304, + "step": 5670 + }, + { + "epoch": 1.84, + "learning_rate": 6.8739063659894e-05, + "loss": 0.1358, + "step": 5675 + }, + { + "epoch": 1.84, + "learning_rate": 6.857290119746425e-05, + "loss": 0.1418, + "step": 5680 + }, + { + "epoch": 1.84, + "learning_rate": 6.840683495860857e-05, + "loss": 0.1367, + "step": 5685 + }, + { + "epoch": 1.84, + "learning_rate": 6.82408654517889e-05, + "loss": 0.1373, + "step": 5690 + }, + { + "epoch": 1.85, + "learning_rate": 6.80749931851712e-05, + "loss": 0.1408, + "step": 5695 + }, + { + "epoch": 1.85, + "learning_rate": 6.790921866662355e-05, + "loss": 0.1326, + "step": 5700 + }, + { + "epoch": 1.85, + "learning_rate": 6.774354240371487e-05, + "loss": 0.137, + "step": 5705 + }, + { + "epoch": 1.85, + "learning_rate": 6.757796490371314e-05, + "loss": 0.1509, + "step": 5710 + }, + { + "epoch": 1.85, + "learning_rate": 6.741248667358397e-05, + "loss": 0.1317, + "step": 5715 + }, + { + "epoch": 1.85, + "learning_rate": 6.724710821998912e-05, + "loss": 0.1395, + "step": 5720 + }, + { + "epoch": 1.86, + "learning_rate": 6.708183004928468e-05, + "loss": 0.1359, + "step": 5725 + }, + { + "epoch": 1.86, + "learning_rate": 6.691665266751985e-05, + "loss": 0.1448, + "step": 5730 + }, + { + "epoch": 1.86, + "learning_rate": 6.675157658043513e-05, + "loss": 0.1365, + "step": 5735 + }, + { + "epoch": 1.86, + "learning_rate": 6.658660229346097e-05, + "loss": 0.1341, + "step": 5740 + }, + { + "epoch": 1.86, + "learning_rate": 6.642173031171603e-05, + "loss": 0.137, + "step": 5745 + }, + { + "epoch": 1.86, + "learning_rate": 6.62569611400058e-05, + "loss": 0.1376, + "step": 5750 + }, + { + "epoch": 1.86, + "learning_rate": 6.609229528282094e-05, + "loss": 0.1297, + "step": 5755 + }, + { + "epoch": 1.87, + "learning_rate": 6.592773324433588e-05, + "loss": 0.1344, + "step": 5760 + }, + { + "epoch": 1.87, + "learning_rate": 6.5763275528407e-05, + "loss": 0.1359, + "step": 5765 + }, + { + "epoch": 1.87, + "learning_rate": 6.559892263857144e-05, + "loss": 0.1302, + "step": 5770 + }, + { + "epoch": 1.87, + "learning_rate": 6.54346750780453e-05, + "loss": 0.132, + "step": 5775 + }, + { + "epoch": 1.87, + "learning_rate": 6.527053334972218e-05, + "loss": 0.1349, + "step": 5780 + }, + { + "epoch": 1.87, + "learning_rate": 6.510649795617168e-05, + "loss": 0.1406, + "step": 5785 + }, + { + "epoch": 1.88, + "learning_rate": 6.494256939963775e-05, + "loss": 0.1345, + "step": 5790 + }, + { + "epoch": 1.88, + "learning_rate": 6.477874818203736e-05, + "loss": 0.1392, + "step": 5795 + }, + { + "epoch": 1.88, + "learning_rate": 6.461503480495864e-05, + "loss": 0.1359, + "step": 5800 + }, + { + "epoch": 1.88, + "learning_rate": 6.445142976965972e-05, + "loss": 0.1284, + "step": 5805 + }, + { + "epoch": 1.88, + "learning_rate": 6.428793357706689e-05, + "loss": 0.1343, + "step": 5810 + }, + { + "epoch": 1.88, + "learning_rate": 6.412454672777328e-05, + "loss": 0.1387, + "step": 5815 + }, + { + "epoch": 1.89, + "learning_rate": 6.396126972203708e-05, + "loss": 0.1334, + "step": 5820 + }, + { + "epoch": 1.89, + "learning_rate": 6.379810305978038e-05, + "loss": 0.1329, + "step": 5825 + }, + { + "epoch": 1.89, + "learning_rate": 6.363504724058717e-05, + "loss": 0.1378, + "step": 5830 + }, + { + "epoch": 1.89, + "learning_rate": 6.347210276370224e-05, + "loss": 0.1358, + "step": 5835 + }, + { + "epoch": 1.89, + "learning_rate": 6.330927012802947e-05, + "loss": 0.1412, + "step": 5840 + }, + { + "epoch": 1.89, + "learning_rate": 6.314654983213018e-05, + "loss": 0.1308, + "step": 5845 + }, + { + "epoch": 1.9, + "learning_rate": 6.298394237422186e-05, + "loss": 0.131, + "step": 5850 + }, + { + "epoch": 1.9, + "learning_rate": 6.282144825217639e-05, + "loss": 0.1341, + "step": 5855 + }, + { + "epoch": 1.9, + "learning_rate": 6.265906796351875e-05, + "loss": 0.1435, + "step": 5860 + }, + { + "epoch": 1.9, + "learning_rate": 6.24968020054253e-05, + "loss": 0.1417, + "step": 5865 + }, + { + "epoch": 1.9, + "learning_rate": 6.233465087472244e-05, + "loss": 0.1376, + "step": 5870 + }, + { + "epoch": 1.9, + "learning_rate": 6.217261506788485e-05, + "loss": 0.1268, + "step": 5875 + }, + { + "epoch": 1.91, + "learning_rate": 6.201069508103426e-05, + "loss": 0.1389, + "step": 5880 + }, + { + "epoch": 1.91, + "learning_rate": 6.184889140993765e-05, + "loss": 0.14, + "step": 5885 + }, + { + "epoch": 1.91, + "learning_rate": 6.168720455000601e-05, + "loss": 0.1394, + "step": 5890 + }, + { + "epoch": 1.91, + "learning_rate": 6.152563499629249e-05, + "loss": 0.1413, + "step": 5895 + }, + { + "epoch": 1.91, + "learning_rate": 6.136418324349125e-05, + "loss": 0.1354, + "step": 5900 + }, + { + "epoch": 1.91, + "learning_rate": 6.120284978593569e-05, + "loss": 0.1335, + "step": 5905 + }, + { + "epoch": 1.92, + "learning_rate": 6.104163511759696e-05, + "loss": 0.1381, + "step": 5910 + }, + { + "epoch": 1.92, + "learning_rate": 6.088053973208262e-05, + "loss": 0.1313, + "step": 5915 + }, + { + "epoch": 1.92, + "learning_rate": 6.071956412263491e-05, + "loss": 0.1314, + "step": 5920 + }, + { + "epoch": 1.92, + "learning_rate": 6.0558708782129416e-05, + "loss": 0.1432, + "step": 5925 + }, + { + "epoch": 1.92, + "learning_rate": 6.039797420307338e-05, + "loss": 0.1398, + "step": 5930 + }, + { + "epoch": 1.92, + "learning_rate": 6.023736087760443e-05, + "loss": 0.1334, + "step": 5935 + }, + { + "epoch": 1.92, + "learning_rate": 6.00768692974888e-05, + "loss": 0.1393, + "step": 5940 + }, + { + "epoch": 1.93, + "learning_rate": 5.991649995412011e-05, + "loss": 0.1319, + "step": 5945 + }, + { + "epoch": 1.93, + "learning_rate": 5.975625333851755e-05, + "loss": 0.1304, + "step": 5950 + }, + { + "epoch": 1.93, + "learning_rate": 5.9596129941324705e-05, + "loss": 0.1332, + "step": 5955 + }, + { + "epoch": 1.93, + "learning_rate": 5.9436130252807785e-05, + "loss": 0.1344, + "step": 5960 + }, + { + "epoch": 1.93, + "learning_rate": 5.927625476285426e-05, + "loss": 0.1329, + "step": 5965 + }, + { + "epoch": 1.93, + "learning_rate": 5.91165039609713e-05, + "loss": 0.1359, + "step": 5970 + }, + { + "epoch": 1.94, + "learning_rate": 5.895687833628437e-05, + "loss": 0.1311, + "step": 5975 + }, + { + "epoch": 1.94, + "learning_rate": 5.879737837753565e-05, + "loss": 0.1324, + "step": 5980 + }, + { + "epoch": 1.94, + "learning_rate": 5.863800457308246e-05, + "loss": 0.1286, + "step": 5985 + }, + { + "epoch": 1.94, + "learning_rate": 5.847875741089604e-05, + "loss": 0.134, + "step": 5990 + }, + { + "epoch": 1.94, + "learning_rate": 5.8319637378559635e-05, + "loss": 0.1312, + "step": 5995 + }, + { + "epoch": 1.94, + "learning_rate": 5.816064496326752e-05, + "loss": 0.1277, + "step": 6000 + }, + { + "epoch": 1.95, + "learning_rate": 5.800178065182296e-05, + "loss": 0.1377, + "step": 6005 + }, + { + "epoch": 1.95, + "learning_rate": 5.784304493063725e-05, + "loss": 0.138, + "step": 6010 + }, + { + "epoch": 1.95, + "learning_rate": 5.768443828572772e-05, + "loss": 0.1283, + "step": 6015 + }, + { + "epoch": 1.95, + "learning_rate": 5.7525961202716714e-05, + "loss": 0.1307, + "step": 6020 + }, + { + "epoch": 1.95, + "learning_rate": 5.7367614166829706e-05, + "loss": 0.136, + "step": 6025 + }, + { + "epoch": 1.95, + "learning_rate": 5.720939766289415e-05, + "loss": 0.137, + "step": 6030 + }, + { + "epoch": 1.96, + "learning_rate": 5.7051312175337656e-05, + "loss": 0.1341, + "step": 6035 + }, + { + "epoch": 1.96, + "learning_rate": 5.689335818818684e-05, + "loss": 0.1384, + "step": 6040 + }, + { + "epoch": 1.96, + "learning_rate": 5.673553618506562e-05, + "loss": 0.1378, + "step": 6045 + }, + { + "epoch": 1.96, + "learning_rate": 5.65778466491939e-05, + "loss": 0.1348, + "step": 6050 + }, + { + "epoch": 1.96, + "learning_rate": 5.6420290063385805e-05, + "loss": 0.1344, + "step": 6055 + }, + { + "epoch": 1.96, + "learning_rate": 5.626286691004854e-05, + "loss": 0.1354, + "step": 6060 + }, + { + "epoch": 1.97, + "learning_rate": 5.61055776711808e-05, + "loss": 0.1345, + "step": 6065 + }, + { + "epoch": 1.97, + "learning_rate": 5.594842282837109e-05, + "loss": 0.1497, + "step": 6070 + }, + { + "epoch": 1.97, + "learning_rate": 5.5791402862796604e-05, + "loss": 0.1257, + "step": 6075 + }, + { + "epoch": 1.97, + "learning_rate": 5.5634518255221415e-05, + "loss": 0.1384, + "step": 6080 + }, + { + "epoch": 1.97, + "learning_rate": 5.54777694859953e-05, + "loss": 0.1344, + "step": 6085 + }, + { + "epoch": 1.97, + "learning_rate": 5.532115703505198e-05, + "loss": 0.1321, + "step": 6090 + }, + { + "epoch": 1.98, + "learning_rate": 5.516468138190796e-05, + "loss": 0.1389, + "step": 6095 + }, + { + "epoch": 1.98, + "learning_rate": 5.500834300566069e-05, + "loss": 0.1357, + "step": 6100 + }, + { + "epoch": 1.98, + "learning_rate": 5.4852142384987504e-05, + "loss": 0.1333, + "step": 6105 + }, + { + "epoch": 1.98, + "learning_rate": 5.46960799981439e-05, + "loss": 0.1296, + "step": 6110 + }, + { + "epoch": 1.98, + "learning_rate": 5.454015632296203e-05, + "loss": 0.1352, + "step": 6115 + }, + { + "epoch": 1.98, + "learning_rate": 5.438437183684948e-05, + "loss": 0.132, + "step": 6120 + }, + { + "epoch": 1.98, + "learning_rate": 5.422872701678754e-05, + "loss": 0.1382, + "step": 6125 + }, + { + "epoch": 1.99, + "learning_rate": 5.407322233932999e-05, + "loss": 0.1363, + "step": 6130 + }, + { + "epoch": 1.99, + "learning_rate": 5.391785828060139e-05, + "loss": 0.138, + "step": 6135 + }, + { + "epoch": 1.99, + "learning_rate": 5.376263531629591e-05, + "loss": 0.1357, + "step": 6140 + }, + { + "epoch": 1.99, + "learning_rate": 5.360755392167555e-05, + "loss": 0.1376, + "step": 6145 + }, + { + "epoch": 1.99, + "learning_rate": 5.345261457156897e-05, + "loss": 0.1372, + "step": 6150 + }, + { + "epoch": 1.99, + "learning_rate": 5.329781774036988e-05, + "loss": 0.1362, + "step": 6155 + }, + { + "epoch": 2.0, + "learning_rate": 5.3143163902035664e-05, + "loss": 0.1321, + "step": 6160 + }, + { + "epoch": 2.0, + "learning_rate": 5.2988653530085796e-05, + "loss": 0.1423, + "step": 6165 + }, + { + "epoch": 2.0, + "learning_rate": 5.2834287097600545e-05, + "loss": 0.1315, + "step": 6170 + }, + { + "epoch": 2.0, + "learning_rate": 5.268006507721952e-05, + "loss": 0.1351, + "step": 6175 + }, + { + "epoch": 2.0, + "learning_rate": 5.252598794114002e-05, + "loss": 0.13, + "step": 6180 + }, + { + "epoch": 2.0, + "learning_rate": 5.2372056161115925e-05, + "loss": 0.1314, + "step": 6185 + }, + { + "epoch": 2.01, + "learning_rate": 5.2218270208455864e-05, + "loss": 0.1272, + "step": 6190 + }, + { + "epoch": 2.01, + "learning_rate": 5.206463055402218e-05, + "loss": 0.1357, + "step": 6195 + }, + { + "epoch": 2.01, + "learning_rate": 5.191113766822905e-05, + "loss": 0.132, + "step": 6200 + }, + { + "epoch": 2.01, + "learning_rate": 5.175779202104152e-05, + "loss": 0.1382, + "step": 6205 + }, + { + "epoch": 2.01, + "learning_rate": 5.160459408197358e-05, + "loss": 0.1361, + "step": 6210 + }, + { + "epoch": 2.01, + "learning_rate": 5.1451544320087185e-05, + "loss": 0.1371, + "step": 6215 + }, + { + "epoch": 2.02, + "learning_rate": 5.1298643203990424e-05, + "loss": 0.1347, + "step": 6220 + }, + { + "epoch": 2.02, + "learning_rate": 5.1145891201836414e-05, + "loss": 0.1325, + "step": 6225 + }, + { + "epoch": 2.02, + "learning_rate": 5.099328878132156e-05, + "loss": 0.1311, + "step": 6230 + }, + { + "epoch": 2.02, + "learning_rate": 5.084083640968442e-05, + "loss": 0.1396, + "step": 6235 + }, + { + "epoch": 2.02, + "learning_rate": 5.0688534553703995e-05, + "loss": 0.1322, + "step": 6240 + }, + { + "epoch": 2.02, + "learning_rate": 5.053638367969855e-05, + "loss": 0.1404, + "step": 6245 + }, + { + "epoch": 2.03, + "learning_rate": 5.038438425352405e-05, + "loss": 0.1383, + "step": 6250 + }, + { + "epoch": 2.03, + "learning_rate": 5.023253674057275e-05, + "loss": 0.1332, + "step": 6255 + }, + { + "epoch": 2.03, + "learning_rate": 5.008084160577171e-05, + "loss": 0.1369, + "step": 6260 + }, + { + "epoch": 2.03, + "learning_rate": 4.992929931358151e-05, + "loss": 0.1399, + "step": 6265 + }, + { + "epoch": 2.03, + "learning_rate": 4.9777910327994804e-05, + "loss": 0.1348, + "step": 6270 + }, + { + "epoch": 2.03, + "learning_rate": 4.962667511253469e-05, + "loss": 0.1333, + "step": 6275 + }, + { + "epoch": 2.04, + "learning_rate": 4.9475594130253645e-05, + "loss": 0.133, + "step": 6280 + }, + { + "epoch": 2.04, + "learning_rate": 4.932466784373172e-05, + "loss": 0.1312, + "step": 6285 + }, + { + "epoch": 2.04, + "learning_rate": 4.917389671507551e-05, + "loss": 0.1302, + "step": 6290 + }, + { + "epoch": 2.04, + "learning_rate": 4.902328120591637e-05, + "loss": 0.1337, + "step": 6295 + }, + { + "epoch": 2.04, + "learning_rate": 4.887282177740935e-05, + "loss": 0.1316, + "step": 6300 + }, + { + "epoch": 2.04, + "learning_rate": 4.872251889023144e-05, + "loss": 0.1324, + "step": 6305 + }, + { + "epoch": 2.04, + "learning_rate": 4.8572373004580465e-05, + "loss": 0.1344, + "step": 6310 + }, + { + "epoch": 2.05, + "learning_rate": 4.8422384580173505e-05, + "loss": 0.1281, + "step": 6315 + }, + { + "epoch": 2.05, + "learning_rate": 4.827255407624547e-05, + "loss": 0.1349, + "step": 6320 + }, + { + "epoch": 2.05, + "learning_rate": 4.812288195154785e-05, + "loss": 0.1305, + "step": 6325 + }, + { + "epoch": 2.05, + "learning_rate": 4.7973368664347086e-05, + "loss": 0.1273, + "step": 6330 + }, + { + "epoch": 2.05, + "learning_rate": 4.7824014672423414e-05, + "loss": 0.1309, + "step": 6335 + }, + { + "epoch": 2.05, + "learning_rate": 4.7674820433069224e-05, + "loss": 0.1333, + "step": 6340 + }, + { + "epoch": 2.06, + "learning_rate": 4.752578640308789e-05, + "loss": 0.1411, + "step": 6345 + }, + { + "epoch": 2.06, + "learning_rate": 4.7376913038792136e-05, + "loss": 0.1309, + "step": 6350 + }, + { + "epoch": 2.06, + "learning_rate": 4.7228200796002854e-05, + "loss": 0.1262, + "step": 6355 + }, + { + "epoch": 2.06, + "learning_rate": 4.707965013004758e-05, + "loss": 0.1319, + "step": 6360 + }, + { + "epoch": 2.06, + "learning_rate": 4.6931261495759184e-05, + "loss": 0.1297, + "step": 6365 + }, + { + "epoch": 2.06, + "learning_rate": 4.678303534747428e-05, + "loss": 0.1313, + "step": 6370 + }, + { + "epoch": 2.07, + "learning_rate": 4.663497213903213e-05, + "loss": 0.1244, + "step": 6375 + }, + { + "epoch": 2.07, + "learning_rate": 4.6487072323773093e-05, + "loss": 0.1292, + "step": 6380 + }, + { + "epoch": 2.07, + "learning_rate": 4.633933635453716e-05, + "loss": 0.1327, + "step": 6385 + }, + { + "epoch": 2.07, + "learning_rate": 4.6191764683662744e-05, + "loss": 0.1335, + "step": 6390 + }, + { + "epoch": 2.07, + "learning_rate": 4.6044357762985125e-05, + "loss": 0.1342, + "step": 6395 + }, + { + "epoch": 2.07, + "learning_rate": 4.589711604383528e-05, + "loss": 0.1348, + "step": 6400 + }, + { + "epoch": 2.08, + "learning_rate": 4.575003997703821e-05, + "loss": 0.1344, + "step": 6405 + }, + { + "epoch": 2.08, + "learning_rate": 4.560313001291188e-05, + "loss": 0.125, + "step": 6410 + }, + { + "epoch": 2.08, + "learning_rate": 4.5456386601265534e-05, + "loss": 0.1425, + "step": 6415 + }, + { + "epoch": 2.08, + "learning_rate": 4.530981019139862e-05, + "loss": 0.1286, + "step": 6420 + }, + { + "epoch": 2.08, + "learning_rate": 4.516340123209908e-05, + "loss": 0.1286, + "step": 6425 + }, + { + "epoch": 2.08, + "learning_rate": 4.5017160171642333e-05, + "loss": 0.1347, + "step": 6430 + }, + { + "epoch": 2.09, + "learning_rate": 4.4871087457789584e-05, + "loss": 0.1301, + "step": 6435 + }, + { + "epoch": 2.09, + "learning_rate": 4.4725183537786683e-05, + "loss": 0.1314, + "step": 6440 + }, + { + "epoch": 2.09, + "learning_rate": 4.457944885836267e-05, + "loss": 0.1341, + "step": 6445 + }, + { + "epoch": 2.09, + "learning_rate": 4.44338838657283e-05, + "loss": 0.1347, + "step": 6450 + }, + { + "epoch": 2.09, + "learning_rate": 4.428848900557492e-05, + "loss": 0.1308, + "step": 6455 + }, + { + "epoch": 2.09, + "learning_rate": 4.414326472307283e-05, + "loss": 0.1395, + "step": 6460 + }, + { + "epoch": 2.09, + "learning_rate": 4.3998211462870165e-05, + "loss": 0.1369, + "step": 6465 + }, + { + "epoch": 2.1, + "learning_rate": 4.385332966909136e-05, + "loss": 0.1334, + "step": 6470 + }, + { + "epoch": 2.1, + "learning_rate": 4.3708619785335923e-05, + "loss": 0.1296, + "step": 6475 + }, + { + "epoch": 2.1, + "learning_rate": 4.356408225467688e-05, + "loss": 0.1301, + "step": 6480 + }, + { + "epoch": 2.1, + "learning_rate": 4.3419717519659694e-05, + "loss": 0.125, + "step": 6485 + }, + { + "epoch": 2.1, + "learning_rate": 4.327552602230063e-05, + "loss": 0.1251, + "step": 6490 + }, + { + "epoch": 2.1, + "learning_rate": 4.313150820408566e-05, + "loss": 0.137, + "step": 6495 + }, + { + "epoch": 2.11, + "learning_rate": 4.298766450596885e-05, + "loss": 0.126, + "step": 6500 + }, + { + "epoch": 2.11, + "learning_rate": 4.284399536837124e-05, + "loss": 0.1331, + "step": 6505 + }, + { + "epoch": 2.11, + "learning_rate": 4.2700501231179427e-05, + "loss": 0.1322, + "step": 6510 + }, + { + "epoch": 2.11, + "learning_rate": 4.255718253374405e-05, + "loss": 0.1292, + "step": 6515 + }, + { + "epoch": 2.11, + "learning_rate": 4.241403971487876e-05, + "loss": 0.1354, + "step": 6520 + }, + { + "epoch": 2.11, + "learning_rate": 4.227107321285855e-05, + "loss": 0.1264, + "step": 6525 + }, + { + "epoch": 2.12, + "learning_rate": 4.212828346541868e-05, + "loss": 0.1336, + "step": 6530 + }, + { + "epoch": 2.12, + "learning_rate": 4.198567090975314e-05, + "loss": 0.1218, + "step": 6535 + }, + { + "epoch": 2.12, + "learning_rate": 4.1843235982513463e-05, + "loss": 0.1244, + "step": 6540 + }, + { + "epoch": 2.12, + "learning_rate": 4.1700979119807235e-05, + "loss": 0.1354, + "step": 6545 + }, + { + "epoch": 2.12, + "learning_rate": 4.1558900757196964e-05, + "loss": 0.1231, + "step": 6550 + }, + { + "epoch": 2.12, + "learning_rate": 4.141700132969848e-05, + "loss": 0.133, + "step": 6555 + }, + { + "epoch": 2.13, + "learning_rate": 4.1275281271779854e-05, + "loss": 0.1297, + "step": 6560 + }, + { + "epoch": 2.13, + "learning_rate": 4.113374101735998e-05, + "loss": 0.1359, + "step": 6565 + }, + { + "epoch": 2.13, + "learning_rate": 4.099238099980711e-05, + "loss": 0.1251, + "step": 6570 + }, + { + "epoch": 2.13, + "learning_rate": 4.0851201651937755e-05, + "loss": 0.1291, + "step": 6575 + }, + { + "epoch": 2.13, + "learning_rate": 4.0710203406015204e-05, + "loss": 0.1278, + "step": 6580 + }, + { + "epoch": 2.13, + "learning_rate": 4.05693866937483e-05, + "loss": 0.1191, + "step": 6585 + }, + { + "epoch": 2.14, + "learning_rate": 4.0428751946289956e-05, + "loss": 0.1256, + "step": 6590 + }, + { + "epoch": 2.14, + "learning_rate": 4.028829959423608e-05, + "loss": 0.1299, + "step": 6595 + }, + { + "epoch": 2.14, + "learning_rate": 4.014803006762398e-05, + "loss": 0.131, + "step": 6600 + }, + { + "epoch": 2.14, + "learning_rate": 4.0007943795931346e-05, + "loss": 0.1292, + "step": 6605 + }, + { + "epoch": 2.14, + "learning_rate": 3.9868041208074617e-05, + "loss": 0.1311, + "step": 6610 + }, + { + "epoch": 2.14, + "learning_rate": 3.972832273240796e-05, + "loss": 0.1273, + "step": 6615 + }, + { + "epoch": 2.15, + "learning_rate": 3.958878879672172e-05, + "loss": 0.1352, + "step": 6620 + }, + { + "epoch": 2.15, + "learning_rate": 3.9449439828241354e-05, + "loss": 0.129, + "step": 6625 + }, + { + "epoch": 2.15, + "learning_rate": 3.93102762536258e-05, + "loss": 0.1264, + "step": 6630 + }, + { + "epoch": 2.15, + "learning_rate": 3.917129849896653e-05, + "loss": 0.1248, + "step": 6635 + }, + { + "epoch": 2.15, + "learning_rate": 3.903250698978596e-05, + "loss": 0.1164, + "step": 6640 + }, + { + "epoch": 2.15, + "learning_rate": 3.889390215103631e-05, + "loss": 0.13, + "step": 6645 + }, + { + "epoch": 2.15, + "learning_rate": 3.875548440709827e-05, + "loss": 0.131, + "step": 6650 + }, + { + "epoch": 2.16, + "learning_rate": 3.861725418177961e-05, + "loss": 0.1253, + "step": 6655 + }, + { + "epoch": 2.16, + "learning_rate": 3.8479211898314035e-05, + "loss": 0.129, + "step": 6660 + }, + { + "epoch": 2.16, + "learning_rate": 3.8341357979359735e-05, + "loss": 0.1304, + "step": 6665 + }, + { + "epoch": 2.16, + "learning_rate": 3.820369284699823e-05, + "loss": 0.1252, + "step": 6670 + }, + { + "epoch": 2.16, + "learning_rate": 3.8066216922733e-05, + "loss": 0.129, + "step": 6675 + }, + { + "epoch": 2.16, + "learning_rate": 3.792893062748814e-05, + "loss": 0.1261, + "step": 6680 + }, + { + "epoch": 2.17, + "learning_rate": 3.7791834381607214e-05, + "loss": 0.1278, + "step": 6685 + }, + { + "epoch": 2.17, + "learning_rate": 3.7654928604851924e-05, + "loss": 0.1214, + "step": 6690 + }, + { + "epoch": 2.17, + "learning_rate": 3.751821371640065e-05, + "loss": 0.1227, + "step": 6695 + }, + { + "epoch": 2.17, + "learning_rate": 3.7381690134847483e-05, + "loss": 0.1272, + "step": 6700 + }, + { + "epoch": 2.17, + "learning_rate": 3.724535827820059e-05, + "loss": 0.1309, + "step": 6705 + }, + { + "epoch": 2.17, + "learning_rate": 3.7109218563881274e-05, + "loss": 0.1386, + "step": 6710 + }, + { + "epoch": 2.18, + "learning_rate": 3.6973271408722496e-05, + "loss": 0.1272, + "step": 6715 + }, + { + "epoch": 2.18, + "learning_rate": 3.6837517228967535e-05, + "loss": 0.1323, + "step": 6720 + }, + { + "epoch": 2.18, + "learning_rate": 3.670195644026898e-05, + "loss": 0.1259, + "step": 6725 + }, + { + "epoch": 2.18, + "learning_rate": 3.656658945768713e-05, + "loss": 0.131, + "step": 6730 + }, + { + "epoch": 2.18, + "learning_rate": 3.6431416695689046e-05, + "loss": 0.1209, + "step": 6735 + }, + { + "epoch": 2.18, + "learning_rate": 3.6296438568146974e-05, + "loss": 0.131, + "step": 6740 + }, + { + "epoch": 2.19, + "learning_rate": 3.616165548833738e-05, + "loss": 0.1306, + "step": 6745 + }, + { + "epoch": 2.19, + "learning_rate": 3.602706786893937e-05, + "loss": 0.1346, + "step": 6750 + }, + { + "epoch": 2.19, + "learning_rate": 3.589267612203374e-05, + "loss": 0.1231, + "step": 6755 + }, + { + "epoch": 2.19, + "learning_rate": 3.575848065910145e-05, + "loss": 0.1281, + "step": 6760 + }, + { + "epoch": 2.19, + "learning_rate": 3.5624481891022565e-05, + "loss": 0.1325, + "step": 6765 + }, + { + "epoch": 2.19, + "learning_rate": 3.54906802280748e-05, + "loss": 0.1274, + "step": 6770 + }, + { + "epoch": 2.2, + "learning_rate": 3.5357076079932495e-05, + "loss": 0.1223, + "step": 6775 + }, + { + "epoch": 2.2, + "learning_rate": 3.522366985566518e-05, + "loss": 0.1292, + "step": 6780 + }, + { + "epoch": 2.2, + "learning_rate": 3.509046196373639e-05, + "loss": 0.1293, + "step": 6785 + }, + { + "epoch": 2.2, + "learning_rate": 3.4957452812002355e-05, + "loss": 0.1217, + "step": 6790 + }, + { + "epoch": 2.2, + "learning_rate": 3.482464280771087e-05, + "loss": 0.1279, + "step": 6795 + }, + { + "epoch": 2.2, + "learning_rate": 3.4692032357500007e-05, + "loss": 0.1278, + "step": 6800 + }, + { + "epoch": 2.21, + "learning_rate": 3.455962186739671e-05, + "loss": 0.1266, + "step": 6805 + }, + { + "epoch": 2.21, + "learning_rate": 3.442741174281584e-05, + "loss": 0.1307, + "step": 6810 + }, + { + "epoch": 2.21, + "learning_rate": 3.429540238855863e-05, + "loss": 0.1263, + "step": 6815 + }, + { + "epoch": 2.21, + "learning_rate": 3.416359420881176e-05, + "loss": 0.1232, + "step": 6820 + }, + { + "epoch": 2.21, + "learning_rate": 3.4031987607145756e-05, + "loss": 0.1277, + "step": 6825 + }, + { + "epoch": 2.21, + "learning_rate": 3.390058298651415e-05, + "loss": 0.1179, + "step": 6830 + }, + { + "epoch": 2.21, + "learning_rate": 3.37693807492519e-05, + "loss": 0.1187, + "step": 6835 + }, + { + "epoch": 2.22, + "learning_rate": 3.363838129707437e-05, + "loss": 0.1285, + "step": 6840 + }, + { + "epoch": 2.22, + "learning_rate": 3.3507585031076083e-05, + "loss": 0.1241, + "step": 6845 + }, + { + "epoch": 2.22, + "learning_rate": 3.337699235172931e-05, + "loss": 0.133, + "step": 6850 + }, + { + "epoch": 2.22, + "learning_rate": 3.324660365888313e-05, + "loss": 0.1239, + "step": 6855 + }, + { + "epoch": 2.22, + "learning_rate": 3.311641935176192e-05, + "loss": 0.1297, + "step": 6860 + }, + { + "epoch": 2.22, + "learning_rate": 3.298643982896441e-05, + "loss": 0.1238, + "step": 6865 + }, + { + "epoch": 2.23, + "learning_rate": 3.285666548846217e-05, + "loss": 0.1242, + "step": 6870 + }, + { + "epoch": 2.23, + "learning_rate": 3.2727096727598696e-05, + "loss": 0.1297, + "step": 6875 + }, + { + "epoch": 2.23, + "learning_rate": 3.259773394308787e-05, + "loss": 0.1292, + "step": 6880 + }, + { + "epoch": 2.23, + "learning_rate": 3.246857753101308e-05, + "loss": 0.1298, + "step": 6885 + }, + { + "epoch": 2.23, + "learning_rate": 3.233962788682573e-05, + "loss": 0.1307, + "step": 6890 + }, + { + "epoch": 2.23, + "learning_rate": 3.2210885405344235e-05, + "loss": 0.1339, + "step": 6895 + }, + { + "epoch": 2.24, + "learning_rate": 3.20823504807526e-05, + "loss": 0.1233, + "step": 6900 + }, + { + "epoch": 2.24, + "learning_rate": 3.195402350659945e-05, + "loss": 0.1192, + "step": 6905 + }, + { + "epoch": 2.24, + "learning_rate": 3.182590487579659e-05, + "loss": 0.1307, + "step": 6910 + }, + { + "epoch": 2.24, + "learning_rate": 3.169799498061803e-05, + "loss": 0.1216, + "step": 6915 + }, + { + "epoch": 2.24, + "learning_rate": 3.157029421269863e-05, + "loss": 0.1245, + "step": 6920 + }, + { + "epoch": 2.24, + "learning_rate": 3.144280296303289e-05, + "loss": 0.1293, + "step": 6925 + }, + { + "epoch": 2.25, + "learning_rate": 3.1315521621973934e-05, + "loss": 0.122, + "step": 6930 + }, + { + "epoch": 2.25, + "learning_rate": 3.1188450579232007e-05, + "loss": 0.1244, + "step": 6935 + }, + { + "epoch": 2.25, + "learning_rate": 3.106159022387365e-05, + "loss": 0.1249, + "step": 6940 + }, + { + "epoch": 2.25, + "learning_rate": 3.093494094432018e-05, + "loss": 0.1282, + "step": 6945 + }, + { + "epoch": 2.25, + "learning_rate": 3.080850312834673e-05, + "loss": 0.128, + "step": 6950 + }, + { + "epoch": 2.25, + "learning_rate": 3.06822771630809e-05, + "loss": 0.1277, + "step": 6955 + }, + { + "epoch": 2.26, + "learning_rate": 3.055626343500173e-05, + "loss": 0.124, + "step": 6960 + }, + { + "epoch": 2.26, + "learning_rate": 3.0430462329938326e-05, + "loss": 0.1308, + "step": 6965 + }, + { + "epoch": 2.26, + "learning_rate": 3.0304874233068912e-05, + "loss": 0.1276, + "step": 6970 + }, + { + "epoch": 2.26, + "learning_rate": 3.01794995289194e-05, + "loss": 0.1255, + "step": 6975 + }, + { + "epoch": 2.26, + "learning_rate": 3.0054338601362396e-05, + "loss": 0.1222, + "step": 6980 + }, + { + "epoch": 2.26, + "learning_rate": 2.9929391833616003e-05, + "loss": 0.1231, + "step": 6985 + }, + { + "epoch": 2.27, + "learning_rate": 2.9804659608242512e-05, + "loss": 0.1263, + "step": 6990 + }, + { + "epoch": 2.27, + "learning_rate": 2.9680142307147397e-05, + "loss": 0.1235, + "step": 6995 + }, + { + "epoch": 2.27, + "learning_rate": 2.9555840311578066e-05, + "loss": 0.1241, + "step": 7000 + } + ], + "logging_steps": 5, + "max_steps": 9255, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 1000, + "total_flos": 1.4260852685208027e+19, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/save_lora_math_r8_2e4/checkpoint-7000/training_args.bin b/save_lora_math_r8_2e4/checkpoint-7000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..175cae96ce99657f1e2b29179eee355d72511307 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-7000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0cedf67f015d3f83c601cebdd463c220d745d8e241fab282366821363eb224d +size 5880 diff --git a/save_lora_math_r8_2e4/checkpoint-7000/zero_to_fp32.py b/save_lora_math_r8_2e4/checkpoint-7000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..c98caae31534368be22b67fc4ae906836c992a8d --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-7000/zero_to_fp32.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in param_shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, args.output_file, tag=args.tag) diff --git a/save_lora_math_r8_2e4/checkpoint-8000/README.md b/save_lora_math_r8_2e4/checkpoint-8000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d380bbbf0bf0fa1f211b53410559607e5007a209 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-8000/README.md @@ -0,0 +1,202 @@ +--- +library_name: peft +base_model: meta-llama/Llama-2-7b-hf +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.9.0 \ No newline at end of file diff --git a/save_lora_math_r8_2e4/checkpoint-8000/adapter_config.json b/save_lora_math_r8_2e4/checkpoint-8000/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9c81a592f0ff08374a4f61f6003029070a8429ae --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-8000/adapter_config.json @@ -0,0 +1,35 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "meta-llama/Llama-2-7b-hf", + "bias": "none", + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "mora_type": 1, + "peft_type": "LORA", + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "down_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "use_dora": false, + "use_mora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/save_lora_math_r8_2e4/checkpoint-8000/adapter_model.safetensors b/save_lora_math_r8_2e4/checkpoint-8000/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1dababed7c0f50ce466b7384554c6834123c12f0 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-8000/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3da2ef90ae44354f622d1ea0f009d812b15ce8291555c53022d07a414d1633ef +size 40036488 diff --git a/save_lora_math_r8_2e4/checkpoint-8000/global_step8000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/save_lora_math_r8_2e4/checkpoint-8000/global_step8000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..00552875befb1df8a770cfb22cd0d97482a55087 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-8000/global_step8000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbeddba8eecb104878c6960cafc1f602aa925ec9fb48a315eb36f65534ceab84 +size 119958768 diff --git a/save_lora_math_r8_2e4/checkpoint-8000/global_step8000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/save_lora_math_r8_2e4/checkpoint-8000/global_step8000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..669cb9415536bf970dba0b5916e4c246d6a4878e --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-8000/global_step8000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f99150a8db6891900639fa7e7465e94aad57ab370e6b5bb3cbffeee6ece5077 +size 119958896 diff --git a/save_lora_math_r8_2e4/checkpoint-8000/global_step8000/mp_rank_00_model_states.pt b/save_lora_math_r8_2e4/checkpoint-8000/global_step8000/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ba6a93402b05af6eef7bac844409f7094661d09f --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-8000/global_step8000/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c6704437b3d9bdd895c88d1aea443c290ed3f1d8f965715fde158bd40b1715b +size 40303404 diff --git a/save_lora_math_r8_2e4/checkpoint-8000/latest b/save_lora_math_r8_2e4/checkpoint-8000/latest new file mode 100644 index 0000000000000000000000000000000000000000..6c558c76ac9e7515267f79d24d9fde4a7f8688f1 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-8000/latest @@ -0,0 +1 @@ +global_step8000 \ No newline at end of file diff --git a/save_lora_math_r8_2e4/checkpoint-8000/rng_state_0.pth b/save_lora_math_r8_2e4/checkpoint-8000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..7747b1833f96ef8995834fa560b7a62eb9f51fc7 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-8000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c154cf8a94c9c77ba827ba26859ffec5ae733cda522ad44d13efc73af10322e +size 14512 diff --git a/save_lora_math_r8_2e4/checkpoint-8000/rng_state_1.pth b/save_lora_math_r8_2e4/checkpoint-8000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..26de42c19710d8f49ffcbced59895d69d4da3ea8 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-8000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cea02f86291ae1605d59789d2c2170ef74df30489ab33c523e3c5136adb708b4 +size 14512 diff --git a/save_lora_math_r8_2e4/checkpoint-8000/scheduler.pt b/save_lora_math_r8_2e4/checkpoint-8000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..8d96f5b9729762ba3fa045d4c1edeca98e6bf19e --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-8000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:841571816a8e9d017cf5da8136b4c887c70b81a9e86841ba1db1066a50e4b591 +size 1064 diff --git a/save_lora_math_r8_2e4/checkpoint-8000/trainer_state.json b/save_lora_math_r8_2e4/checkpoint-8000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..cf0567f20c7020c9f1f9994971ea3c4fd81bc3d7 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-8000/trainer_state.json @@ -0,0 +1,9621 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.5924050632911393, + "eval_steps": 500, + "global_step": 8000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 3.5971223021582732e-06, + "loss": 0.7628, + "step": 5 + }, + { + "epoch": 0.0, + "learning_rate": 7.1942446043165465e-06, + "loss": 0.769, + "step": 10 + }, + { + "epoch": 0.0, + "learning_rate": 1.0791366906474821e-05, + "loss": 0.7619, + "step": 15 + }, + { + "epoch": 0.01, + "learning_rate": 1.4388489208633093e-05, + "loss": 0.7564, + "step": 20 + }, + { + "epoch": 0.01, + "learning_rate": 1.7985611510791367e-05, + "loss": 0.7549, + "step": 25 + }, + { + "epoch": 0.01, + "learning_rate": 2.1582733812949642e-05, + "loss": 0.7001, + "step": 30 + }, + { + "epoch": 0.01, + "learning_rate": 2.5179856115107914e-05, + "loss": 0.6289, + "step": 35 + }, + { + "epoch": 0.01, + "learning_rate": 2.8776978417266186e-05, + "loss": 0.5632, + "step": 40 + }, + { + "epoch": 0.01, + "learning_rate": 3.237410071942446e-05, + "loss": 0.4923, + "step": 45 + }, + { + "epoch": 0.02, + "learning_rate": 3.597122302158273e-05, + "loss": 0.4582, + "step": 50 + }, + { + "epoch": 0.02, + "learning_rate": 3.956834532374101e-05, + "loss": 0.4357, + "step": 55 + }, + { + "epoch": 0.02, + "learning_rate": 4.3165467625899284e-05, + "loss": 0.4079, + "step": 60 + }, + { + "epoch": 0.02, + "learning_rate": 4.676258992805755e-05, + "loss": 0.4047, + "step": 65 + }, + { + "epoch": 0.02, + "learning_rate": 5.035971223021583e-05, + "loss": 0.3916, + "step": 70 + }, + { + "epoch": 0.02, + "learning_rate": 5.39568345323741e-05, + "loss": 0.3926, + "step": 75 + }, + { + "epoch": 0.03, + "learning_rate": 5.755395683453237e-05, + "loss": 0.3725, + "step": 80 + }, + { + "epoch": 0.03, + "learning_rate": 6.115107913669065e-05, + "loss": 0.3769, + "step": 85 + }, + { + "epoch": 0.03, + "learning_rate": 6.474820143884892e-05, + "loss": 0.352, + "step": 90 + }, + { + "epoch": 0.03, + "learning_rate": 6.83453237410072e-05, + "loss": 0.3534, + "step": 95 + }, + { + "epoch": 0.03, + "learning_rate": 7.194244604316547e-05, + "loss": 0.3432, + "step": 100 + }, + { + "epoch": 0.03, + "learning_rate": 7.553956834532374e-05, + "loss": 0.3286, + "step": 105 + }, + { + "epoch": 0.04, + "learning_rate": 7.913669064748202e-05, + "loss": 0.3298, + "step": 110 + }, + { + "epoch": 0.04, + "learning_rate": 8.273381294964029e-05, + "loss": 0.3223, + "step": 115 + }, + { + "epoch": 0.04, + "learning_rate": 8.633093525179857e-05, + "loss": 0.3181, + "step": 120 + }, + { + "epoch": 0.04, + "learning_rate": 8.992805755395684e-05, + "loss": 0.3225, + "step": 125 + }, + { + "epoch": 0.04, + "learning_rate": 9.35251798561151e-05, + "loss": 0.3145, + "step": 130 + }, + { + "epoch": 0.04, + "learning_rate": 9.712230215827339e-05, + "loss": 0.3137, + "step": 135 + }, + { + "epoch": 0.05, + "learning_rate": 0.00010071942446043166, + "loss": 0.3102, + "step": 140 + }, + { + "epoch": 0.05, + "learning_rate": 0.00010431654676258992, + "loss": 0.2951, + "step": 145 + }, + { + "epoch": 0.05, + "learning_rate": 0.0001079136690647482, + "loss": 0.3021, + "step": 150 + }, + { + "epoch": 0.05, + "learning_rate": 0.00011151079136690648, + "loss": 0.2923, + "step": 155 + }, + { + "epoch": 0.05, + "learning_rate": 0.00011510791366906474, + "loss": 0.2953, + "step": 160 + }, + { + "epoch": 0.05, + "learning_rate": 0.00011870503597122303, + "loss": 0.2893, + "step": 165 + }, + { + "epoch": 0.06, + "learning_rate": 0.0001223021582733813, + "loss": 0.2953, + "step": 170 + }, + { + "epoch": 0.06, + "learning_rate": 0.00012589928057553956, + "loss": 0.2931, + "step": 175 + }, + { + "epoch": 0.06, + "learning_rate": 0.00012949640287769783, + "loss": 0.2879, + "step": 180 + }, + { + "epoch": 0.06, + "learning_rate": 0.00013309352517985613, + "loss": 0.2642, + "step": 185 + }, + { + "epoch": 0.06, + "learning_rate": 0.0001366906474820144, + "loss": 0.2772, + "step": 190 + }, + { + "epoch": 0.06, + "learning_rate": 0.00014028776978417266, + "loss": 0.2858, + "step": 195 + }, + { + "epoch": 0.06, + "learning_rate": 0.00014388489208633093, + "loss": 0.2724, + "step": 200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0001474820143884892, + "loss": 0.2726, + "step": 205 + }, + { + "epoch": 0.07, + "learning_rate": 0.00015107913669064747, + "loss": 0.2678, + "step": 210 + }, + { + "epoch": 0.07, + "learning_rate": 0.00015467625899280577, + "loss": 0.2789, + "step": 215 + }, + { + "epoch": 0.07, + "learning_rate": 0.00015827338129496403, + "loss": 0.2742, + "step": 220 + }, + { + "epoch": 0.07, + "learning_rate": 0.0001618705035971223, + "loss": 0.2732, + "step": 225 + }, + { + "epoch": 0.07, + "learning_rate": 0.00016546762589928057, + "loss": 0.2741, + "step": 230 + }, + { + "epoch": 0.08, + "learning_rate": 0.00016906474820143884, + "loss": 0.2665, + "step": 235 + }, + { + "epoch": 0.08, + "learning_rate": 0.00017266187050359714, + "loss": 0.2771, + "step": 240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0001762589928057554, + "loss": 0.2751, + "step": 245 + }, + { + "epoch": 0.08, + "learning_rate": 0.00017985611510791367, + "loss": 0.2597, + "step": 250 + }, + { + "epoch": 0.08, + "learning_rate": 0.00018345323741007194, + "loss": 0.275, + "step": 255 + }, + { + "epoch": 0.08, + "learning_rate": 0.0001870503597122302, + "loss": 0.2791, + "step": 260 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019064748201438848, + "loss": 0.2493, + "step": 265 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019424460431654677, + "loss": 0.27, + "step": 270 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019784172661870504, + "loss": 0.2676, + "step": 275 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019999997550557367, + "loss": 0.2613, + "step": 280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019999969994341502, + "loss": 0.2747, + "step": 285 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019999911820191125, + "loss": 0.2699, + "step": 290 + }, + { + "epoch": 0.1, + "learning_rate": 0.0001999982302828436, + "loss": 0.2692, + "step": 295 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999703618893072, + "loss": 0.2522, + "step": 300 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999553592382863, + "loss": 0.2541, + "step": 305 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999372949213085, + "loss": 0.2614, + "step": 310 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999161689936837, + "loss": 0.2481, + "step": 315 + }, + { + "epoch": 0.1, + "learning_rate": 0.0001999891981520095, + "loss": 0.2497, + "step": 320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019998647325745995, + "loss": 0.2545, + "step": 325 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019998344222406283, + "loss": 0.2501, + "step": 330 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019998010506109854, + "loss": 0.2577, + "step": 335 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019997646177878487, + "loss": 0.2585, + "step": 340 + }, + { + "epoch": 0.11, + "learning_rate": 0.0001999725123882768, + "loss": 0.2494, + "step": 345 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019996825690166657, + "loss": 0.2538, + "step": 350 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019996369533198364, + "loss": 0.2469, + "step": 355 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019995882769319466, + "loss": 0.2502, + "step": 360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0001999536540002034, + "loss": 0.2361, + "step": 365 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019994817426885057, + "loss": 0.2446, + "step": 370 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019994238851591417, + "loss": 0.256, + "step": 375 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019993629675910896, + "loss": 0.2352, + "step": 380 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019992989901708672, + "loss": 0.2538, + "step": 385 + }, + { + "epoch": 0.13, + "learning_rate": 0.0001999231953094361, + "loss": 0.2523, + "step": 390 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019991618565668243, + "loss": 0.2512, + "step": 395 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019990887008028798, + "loss": 0.2402, + "step": 400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019990124860265156, + "loss": 0.2423, + "step": 405 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019989332124710862, + "loss": 0.2335, + "step": 410 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019988508803793124, + "loss": 0.2284, + "step": 415 + }, + { + "epoch": 0.14, + "learning_rate": 0.00019987654900032774, + "loss": 0.2382, + "step": 420 + }, + { + "epoch": 0.14, + "learning_rate": 0.0001998677041604431, + "loss": 0.2526, + "step": 425 + }, + { + "epoch": 0.14, + "learning_rate": 0.0001998585535453584, + "loss": 0.2417, + "step": 430 + }, + { + "epoch": 0.14, + "learning_rate": 0.00019984909718309105, + "loss": 0.2381, + "step": 435 + }, + { + "epoch": 0.14, + "learning_rate": 0.0001998393351025945, + "loss": 0.2418, + "step": 440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00019982926733375843, + "loss": 0.2409, + "step": 445 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001998188939074083, + "loss": 0.2467, + "step": 450 + }, + { + "epoch": 0.15, + "learning_rate": 0.00019980821485530544, + "loss": 0.2305, + "step": 455 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001997972302101471, + "loss": 0.232, + "step": 460 + }, + { + "epoch": 0.15, + "learning_rate": 0.00019978594000556602, + "loss": 0.225, + "step": 465 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001997743442761306, + "loss": 0.2259, + "step": 470 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001997624430573447, + "loss": 0.2414, + "step": 475 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019975023638564747, + "loss": 0.238, + "step": 480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019973772429841336, + "loss": 0.224, + "step": 485 + }, + { + "epoch": 0.16, + "learning_rate": 0.0001997249068339519, + "loss": 0.2369, + "step": 490 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019971178403150762, + "loss": 0.2393, + "step": 495 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019969835593125997, + "loss": 0.2249, + "step": 500 + }, + { + "epoch": 0.16, + "learning_rate": 0.0001996846225743231, + "loss": 0.2369, + "step": 505 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019967058400274592, + "loss": 0.2267, + "step": 510 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019965624025951168, + "loss": 0.2294, + "step": 515 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019964159138853815, + "loss": 0.2255, + "step": 520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019962663743467726, + "loss": 0.22, + "step": 525 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019961137844371507, + "loss": 0.2398, + "step": 530 + }, + { + "epoch": 0.17, + "learning_rate": 0.0001995958144623716, + "loss": 0.2431, + "step": 535 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019957994553830067, + "loss": 0.2245, + "step": 540 + }, + { + "epoch": 0.18, + "learning_rate": 0.0001995637717200898, + "loss": 0.2295, + "step": 545 + }, + { + "epoch": 0.18, + "learning_rate": 0.00019954729305726008, + "loss": 0.2288, + "step": 550 + }, + { + "epoch": 0.18, + "learning_rate": 0.00019953050960026586, + "loss": 0.2333, + "step": 555 + }, + { + "epoch": 0.18, + "learning_rate": 0.0001995134214004948, + "loss": 0.2316, + "step": 560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0001994960285102676, + "loss": 0.2285, + "step": 565 + }, + { + "epoch": 0.18, + "learning_rate": 0.00019947833098283782, + "loss": 0.2273, + "step": 570 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019946032887239182, + "loss": 0.2344, + "step": 575 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019944202223404854, + "loss": 0.2251, + "step": 580 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019942341112385922, + "loss": 0.226, + "step": 585 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019940449559880752, + "loss": 0.2353, + "step": 590 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019938527571680892, + "loss": 0.226, + "step": 595 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019936575153671097, + "loss": 0.221, + "step": 600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019934592311829285, + "loss": 0.2219, + "step": 605 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019932579052226528, + "loss": 0.234, + "step": 610 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019930535381027027, + "loss": 0.2181, + "step": 615 + }, + { + "epoch": 0.2, + "learning_rate": 0.000199284613044881, + "loss": 0.2191, + "step": 620 + }, + { + "epoch": 0.2, + "learning_rate": 0.0001992635682896016, + "loss": 0.235, + "step": 625 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019924221960886698, + "loss": 0.2256, + "step": 630 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019922056706804257, + "loss": 0.2265, + "step": 635 + }, + { + "epoch": 0.21, + "learning_rate": 0.0001991986107334242, + "loss": 0.2158, + "step": 640 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019917635067223777, + "loss": 0.2238, + "step": 645 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019915378695263924, + "loss": 0.2182, + "step": 650 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019913091964371423, + "loss": 0.2185, + "step": 655 + }, + { + "epoch": 0.21, + "learning_rate": 0.000199107748815478, + "loss": 0.2063, + "step": 660 + }, + { + "epoch": 0.22, + "learning_rate": 0.000199084274538875, + "loss": 0.2162, + "step": 665 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019906049688577887, + "loss": 0.2185, + "step": 670 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019903641592899205, + "loss": 0.2186, + "step": 675 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019901203174224564, + "loss": 0.2292, + "step": 680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019898734440019933, + "loss": 0.2162, + "step": 685 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019896235397844078, + "loss": 0.219, + "step": 690 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019893706055348578, + "loss": 0.218, + "step": 695 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019891146420277777, + "loss": 0.2211, + "step": 700 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019888556500468775, + "loss": 0.2227, + "step": 705 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019885936303851395, + "loss": 0.2214, + "step": 710 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019883285838448163, + "loss": 0.2204, + "step": 715 + }, + { + "epoch": 0.23, + "learning_rate": 0.0001988060511237428, + "loss": 0.2228, + "step": 720 + }, + { + "epoch": 0.23, + "learning_rate": 0.000198778941338376, + "loss": 0.2259, + "step": 725 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019875152911138602, + "loss": 0.2062, + "step": 730 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019872381452670372, + "loss": 0.2073, + "step": 735 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019869579766918568, + "loss": 0.2172, + "step": 740 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019866747862461396, + "loss": 0.21, + "step": 745 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019863885747969598, + "loss": 0.2195, + "step": 750 + }, + { + "epoch": 0.24, + "learning_rate": 0.0001986099343220639, + "loss": 0.2129, + "step": 755 + }, + { + "epoch": 0.25, + "learning_rate": 0.0001985807092402748, + "loss": 0.215, + "step": 760 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019855118232381015, + "loss": 0.2085, + "step": 765 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019852135366307545, + "loss": 0.2196, + "step": 770 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019849122334940022, + "loss": 0.2129, + "step": 775 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019846079147503755, + "loss": 0.2221, + "step": 780 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019843005813316377, + "loss": 0.2133, + "step": 785 + }, + { + "epoch": 0.26, + "learning_rate": 0.0001983990234178784, + "loss": 0.2066, + "step": 790 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019836768742420352, + "loss": 0.2188, + "step": 795 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019833605024808382, + "loss": 0.2128, + "step": 800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019830411198638607, + "loss": 0.2109, + "step": 805 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019827187273689894, + "loss": 0.2141, + "step": 810 + }, + { + "epoch": 0.26, + "learning_rate": 0.0001982393325983326, + "loss": 0.2098, + "step": 815 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019820649167031856, + "loss": 0.2188, + "step": 820 + }, + { + "epoch": 0.27, + "learning_rate": 0.0001981733500534093, + "loss": 0.2107, + "step": 825 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019813990784907785, + "loss": 0.2162, + "step": 830 + }, + { + "epoch": 0.27, + "learning_rate": 0.0001981061651597177, + "loss": 0.2189, + "step": 835 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019807212208864227, + "loss": 0.2117, + "step": 840 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019803777874008468, + "loss": 0.209, + "step": 845 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019800313521919753, + "loss": 0.206, + "step": 850 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019796819163205246, + "loss": 0.2104, + "step": 855 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019793294808563984, + "loss": 0.2016, + "step": 860 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019789740468786841, + "loss": 0.212, + "step": 865 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019786156154756506, + "loss": 0.2051, + "step": 870 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019782541877447443, + "loss": 0.21, + "step": 875 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019778897647925855, + "loss": 0.2104, + "step": 880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0001977522347734965, + "loss": 0.2179, + "step": 885 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019771519376968416, + "loss": 0.2139, + "step": 890 + }, + { + "epoch": 0.29, + "learning_rate": 0.0001976778535812338, + "loss": 0.2034, + "step": 895 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019764021432247368, + "loss": 0.2067, + "step": 900 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019760227610864776, + "loss": 0.2189, + "step": 905 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019756403905591538, + "loss": 0.2119, + "step": 910 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019752550328135086, + "loss": 0.2004, + "step": 915 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019748666890294315, + "loss": 0.2073, + "step": 920 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019744753603959545, + "loss": 0.1995, + "step": 925 + }, + { + "epoch": 0.3, + "learning_rate": 0.0001974081048111248, + "loss": 0.2034, + "step": 930 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019736837533826192, + "loss": 0.2015, + "step": 935 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019732834774265057, + "loss": 0.2166, + "step": 940 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019728802214684734, + "loss": 0.1987, + "step": 945 + }, + { + "epoch": 0.31, + "learning_rate": 0.0001972473986743213, + "loss": 0.2051, + "step": 950 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019720647744945344, + "loss": 0.2055, + "step": 955 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019716525859753652, + "loss": 0.2005, + "step": 960 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019712374224477452, + "loss": 0.1995, + "step": 965 + }, + { + "epoch": 0.31, + "learning_rate": 0.0001970819285182823, + "loss": 0.2096, + "step": 970 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019703981754608525, + "loss": 0.2024, + "step": 975 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019699740945711884, + "loss": 0.2006, + "step": 980 + }, + { + "epoch": 0.32, + "learning_rate": 0.0001969547043812283, + "loss": 0.2077, + "step": 985 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019691170244916814, + "loss": 0.2094, + "step": 990 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019686840379260176, + "loss": 0.2051, + "step": 995 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019682480854410115, + "loss": 0.2158, + "step": 1000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0001967809168371463, + "loss": 0.2035, + "step": 1005 + }, + { + "epoch": 0.33, + "learning_rate": 0.000196736728806125, + "loss": 0.2104, + "step": 1010 + }, + { + "epoch": 0.33, + "learning_rate": 0.00019669224458633222, + "loss": 0.1969, + "step": 1015 + }, + { + "epoch": 0.33, + "learning_rate": 0.0001966474643139699, + "loss": 0.2058, + "step": 1020 + }, + { + "epoch": 0.33, + "learning_rate": 0.00019660238812614647, + "loss": 0.2045, + "step": 1025 + }, + { + "epoch": 0.33, + "learning_rate": 0.0001965570161608762, + "loss": 0.1916, + "step": 1030 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019651134855707913, + "loss": 0.2008, + "step": 1035 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019646538545458047, + "loss": 0.2095, + "step": 1040 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019641912699411016, + "loss": 0.2009, + "step": 1045 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019637257331730248, + "loss": 0.2059, + "step": 1050 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019632572456669558, + "loss": 0.2084, + "step": 1055 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019627858088573112, + "loss": 0.1997, + "step": 1060 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001962311424187537, + "loss": 0.2068, + "step": 1065 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001961834093110106, + "loss": 0.1978, + "step": 1070 + }, + { + "epoch": 0.35, + "learning_rate": 0.00019613538170865118, + "loss": 0.2028, + "step": 1075 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001960870597587264, + "loss": 0.1996, + "step": 1080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001960384436091886, + "loss": 0.2002, + "step": 1085 + }, + { + "epoch": 0.35, + "learning_rate": 0.00019598953340889082, + "loss": 0.198, + "step": 1090 + }, + { + "epoch": 0.35, + "learning_rate": 0.00019594032930758643, + "loss": 0.1974, + "step": 1095 + }, + { + "epoch": 0.36, + "learning_rate": 0.0001958908314559287, + "loss": 0.1983, + "step": 1100 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019584104000547025, + "loss": 0.1987, + "step": 1105 + }, + { + "epoch": 0.36, + "learning_rate": 0.0001957909551086627, + "loss": 0.2003, + "step": 1110 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019574057691885613, + "loss": 0.1982, + "step": 1115 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019568990559029858, + "loss": 0.2031, + "step": 1120 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019563894127813564, + "loss": 0.1969, + "step": 1125 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019558768413841006, + "loss": 0.1916, + "step": 1130 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019553613432806102, + "loss": 0.2124, + "step": 1135 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019548429200492388, + "loss": 0.2042, + "step": 1140 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019543215732772956, + "loss": 0.201, + "step": 1145 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019537973045610424, + "loss": 0.2001, + "step": 1150 + }, + { + "epoch": 0.37, + "learning_rate": 0.0001953270115505686, + "loss": 0.1978, + "step": 1155 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019527400077253755, + "loss": 0.192, + "step": 1160 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019522069828431957, + "loss": 0.1946, + "step": 1165 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019516710424911642, + "loss": 0.1848, + "step": 1170 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019511321883102244, + "loss": 0.2005, + "step": 1175 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019505904219502414, + "loss": 0.1966, + "step": 1180 + }, + { + "epoch": 0.38, + "learning_rate": 0.0001950045745069997, + "loss": 0.1863, + "step": 1185 + }, + { + "epoch": 0.39, + "learning_rate": 0.0001949498159337184, + "loss": 0.1983, + "step": 1190 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019489476664284023, + "loss": 0.2113, + "step": 1195 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019483942680291523, + "loss": 0.1956, + "step": 1200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019478379658338305, + "loss": 0.2055, + "step": 1205 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019472787615457246, + "loss": 0.1879, + "step": 1210 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019467166568770078, + "loss": 0.1924, + "step": 1215 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019461516535487338, + "loss": 0.1971, + "step": 1220 + }, + { + "epoch": 0.4, + "learning_rate": 0.0001945583753290831, + "loss": 0.1952, + "step": 1225 + }, + { + "epoch": 0.4, + "learning_rate": 0.0001945012957842098, + "loss": 0.1838, + "step": 1230 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019444392689501978, + "loss": 0.1931, + "step": 1235 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019438626883716526, + "loss": 0.2093, + "step": 1240 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019432832178718385, + "loss": 0.1992, + "step": 1245 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019427008592249795, + "loss": 0.1963, + "step": 1250 + }, + { + "epoch": 0.41, + "learning_rate": 0.0001942115614214143, + "loss": 0.2003, + "step": 1255 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019415274846312338, + "loss": 0.1914, + "step": 1260 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019409364722769882, + "loss": 0.1917, + "step": 1265 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019403425789609697, + "loss": 0.2033, + "step": 1270 + }, + { + "epoch": 0.41, + "learning_rate": 0.0001939745806501563, + "loss": 0.1975, + "step": 1275 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019391461567259662, + "loss": 0.1922, + "step": 1280 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019385436314701897, + "loss": 0.1999, + "step": 1285 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019379382325790468, + "loss": 0.1997, + "step": 1290 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019373299619061487, + "loss": 0.1872, + "step": 1295 + }, + { + "epoch": 0.42, + "learning_rate": 0.0001936718821313901, + "loss": 0.1936, + "step": 1300 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019361048126734952, + "loss": 0.1798, + "step": 1305 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019354879378649046, + "loss": 0.1959, + "step": 1310 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019348681987768783, + "loss": 0.1924, + "step": 1315 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019342455973069345, + "loss": 0.1967, + "step": 1320 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019336201353613568, + "loss": 0.1925, + "step": 1325 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019329918148551856, + "loss": 0.1878, + "step": 1330 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019323606377122146, + "loss": 0.1979, + "step": 1335 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019317266058649834, + "loss": 0.189, + "step": 1340 + }, + { + "epoch": 0.44, + "learning_rate": 0.0001931089721254773, + "loss": 0.1995, + "step": 1345 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019304499858315976, + "loss": 0.1874, + "step": 1350 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019298074015542015, + "loss": 0.1991, + "step": 1355 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019291619703900502, + "loss": 0.188, + "step": 1360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0001928513694315327, + "loss": 0.1914, + "step": 1365 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019278625753149254, + "loss": 0.1843, + "step": 1370 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019272086153824427, + "loss": 0.1959, + "step": 1375 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019265518165201758, + "loss": 0.1867, + "step": 1380 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019258921807391131, + "loss": 0.1956, + "step": 1385 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019252297100589286, + "loss": 0.1991, + "step": 1390 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019245644065079775, + "loss": 0.1934, + "step": 1395 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019238962721232876, + "loss": 0.1953, + "step": 1400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019232253089505546, + "loss": 0.1865, + "step": 1405 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019225515190441354, + "loss": 0.1864, + "step": 1410 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019218749044670413, + "loss": 0.194, + "step": 1415 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019211954672909327, + "loss": 0.1853, + "step": 1420 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019205132095961126, + "loss": 0.1967, + "step": 1425 + }, + { + "epoch": 0.46, + "learning_rate": 0.0001919828133471519, + "loss": 0.1852, + "step": 1430 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019191402410147204, + "loss": 0.1913, + "step": 1435 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019184495343319068, + "loss": 0.1911, + "step": 1440 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019177560155378865, + "loss": 0.189, + "step": 1445 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019170596867560766, + "loss": 0.1854, + "step": 1450 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019163605501184986, + "loss": 0.1974, + "step": 1455 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019156586077657712, + "loss": 0.1933, + "step": 1460 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019149538618471032, + "loss": 0.1877, + "step": 1465 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001914246314520287, + "loss": 0.1904, + "step": 1470 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001913535967951694, + "loss": 0.1841, + "step": 1475 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001912822824316264, + "loss": 0.1876, + "step": 1480 + }, + { + "epoch": 0.48, + "learning_rate": 0.00019121068857975027, + "loss": 0.1858, + "step": 1485 + }, + { + "epoch": 0.48, + "learning_rate": 0.00019113881545874722, + "loss": 0.1898, + "step": 1490 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001910666632886786, + "loss": 0.1923, + "step": 1495 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019099423229046014, + "loss": 0.1803, + "step": 1500 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019092152268586118, + "loss": 0.1945, + "step": 1505 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019084853469750424, + "loss": 0.1802, + "step": 1510 + }, + { + "epoch": 0.49, + "learning_rate": 0.0001907752685488641, + "loss": 0.197, + "step": 1515 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019070172446426724, + "loss": 0.1791, + "step": 1520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019062790266889117, + "loss": 0.1874, + "step": 1525 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019055380338876363, + "loss": 0.1806, + "step": 1530 + }, + { + "epoch": 0.5, + "learning_rate": 0.000190479426850762, + "loss": 0.1834, + "step": 1535 + }, + { + "epoch": 0.5, + "learning_rate": 0.0001904047732826126, + "loss": 0.1921, + "step": 1540 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019032984291288986, + "loss": 0.1826, + "step": 1545 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019025463597101584, + "loss": 0.1966, + "step": 1550 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019017915268725925, + "loss": 0.1867, + "step": 1555 + }, + { + "epoch": 0.51, + "learning_rate": 0.00019010339329273514, + "loss": 0.196, + "step": 1560 + }, + { + "epoch": 0.51, + "learning_rate": 0.00019002735801940373, + "loss": 0.1971, + "step": 1565 + }, + { + "epoch": 0.51, + "learning_rate": 0.00018995104710007004, + "loss": 0.187, + "step": 1570 + }, + { + "epoch": 0.51, + "learning_rate": 0.00018987446076838312, + "loss": 0.1897, + "step": 1575 + }, + { + "epoch": 0.51, + "learning_rate": 0.00018979759925883509, + "loss": 0.1887, + "step": 1580 + }, + { + "epoch": 0.51, + "learning_rate": 0.0001897204628067608, + "loss": 0.1849, + "step": 1585 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018964305164833678, + "loss": 0.1795, + "step": 1590 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018956536602058078, + "loss": 0.1984, + "step": 1595 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018948740616135085, + "loss": 0.1876, + "step": 1600 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018940917230934464, + "loss": 0.1879, + "step": 1605 + }, + { + "epoch": 0.52, + "learning_rate": 0.0001893306647040988, + "loss": 0.1906, + "step": 1610 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018925188358598813, + "loss": 0.1917, + "step": 1615 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018917282919622488, + "loss": 0.1912, + "step": 1620 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018909350177685795, + "loss": 0.1813, + "step": 1625 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018901390157077225, + "loss": 0.183, + "step": 1630 + }, + { + "epoch": 0.53, + "learning_rate": 0.0001889340288216879, + "loss": 0.1782, + "step": 1635 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018885388377415957, + "loss": 0.1866, + "step": 1640 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018877346667357548, + "loss": 0.1915, + "step": 1645 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018869277776615691, + "loss": 0.1902, + "step": 1650 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018861181729895745, + "loss": 0.1829, + "step": 1655 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018853058551986202, + "loss": 0.1855, + "step": 1660 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018844908267758633, + "loss": 0.1845, + "step": 1665 + }, + { + "epoch": 0.54, + "learning_rate": 0.000188367309021676, + "loss": 0.1794, + "step": 1670 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018828526480250575, + "loss": 0.1796, + "step": 1675 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018820295027127885, + "loss": 0.1806, + "step": 1680 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018812036568002617, + "loss": 0.1846, + "step": 1685 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018803751128160538, + "loss": 0.189, + "step": 1690 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018795438732970032, + "loss": 0.1856, + "step": 1695 + }, + { + "epoch": 0.55, + "learning_rate": 0.0001878709940788201, + "loss": 0.1797, + "step": 1700 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018778733178429844, + "loss": 0.1876, + "step": 1705 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018770340070229272, + "loss": 0.1775, + "step": 1710 + }, + { + "epoch": 0.56, + "learning_rate": 0.0001876192010897834, + "loss": 0.1862, + "step": 1715 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018753473320457306, + "loss": 0.1835, + "step": 1720 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018744999730528568, + "loss": 0.1909, + "step": 1725 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018736499365136584, + "loss": 0.189, + "step": 1730 + }, + { + "epoch": 0.56, + "learning_rate": 0.000187279722503078, + "loss": 0.1861, + "step": 1735 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018719418412150557, + "loss": 0.1809, + "step": 1740 + }, + { + "epoch": 0.57, + "learning_rate": 0.00018710837876855017, + "loss": 0.1829, + "step": 1745 + }, + { + "epoch": 0.57, + "learning_rate": 0.0001870223067069309, + "loss": 0.1886, + "step": 1750 + }, + { + "epoch": 0.57, + "learning_rate": 0.00018693596820018336, + "loss": 0.1787, + "step": 1755 + }, + { + "epoch": 0.57, + "learning_rate": 0.0001868493635126591, + "loss": 0.1816, + "step": 1760 + }, + { + "epoch": 0.57, + "learning_rate": 0.0001867624929095245, + "loss": 0.1759, + "step": 1765 + }, + { + "epoch": 0.57, + "learning_rate": 0.00018667535665676025, + "loss": 0.1702, + "step": 1770 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018658795502116035, + "loss": 0.1802, + "step": 1775 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018650028827033136, + "loss": 0.1836, + "step": 1780 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018641235667269155, + "loss": 0.1787, + "step": 1785 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018632416049747016, + "loss": 0.1811, + "step": 1790 + }, + { + "epoch": 0.58, + "learning_rate": 0.0001862357000147064, + "loss": 0.1835, + "step": 1795 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018614697549524889, + "loss": 0.1857, + "step": 1800 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018605798721075455, + "loss": 0.1867, + "step": 1805 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018596873543368797, + "loss": 0.1766, + "step": 1810 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018587922043732046, + "loss": 0.1895, + "step": 1815 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018578944249572925, + "loss": 0.1888, + "step": 1820 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018569940188379672, + "loss": 0.1871, + "step": 1825 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018560909887720944, + "loss": 0.1804, + "step": 1830 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018551853375245737, + "loss": 0.1877, + "step": 1835 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018542770678683308, + "loss": 0.182, + "step": 1840 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001853366182584308, + "loss": 0.1809, + "step": 1845 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018524526844614564, + "loss": 0.1913, + "step": 1850 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018515365762967266, + "loss": 0.185, + "step": 1855 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018506178608950613, + "loss": 0.1753, + "step": 1860 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018496965410693856, + "loss": 0.1786, + "step": 1865 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001848772619640599, + "loss": 0.1841, + "step": 1870 + }, + { + "epoch": 0.61, + "learning_rate": 0.00018478460994375667, + "loss": 0.184, + "step": 1875 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001846916983297111, + "loss": 0.179, + "step": 1880 + }, + { + "epoch": 0.61, + "learning_rate": 0.00018459852740640014, + "loss": 0.1833, + "step": 1885 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001845050974590948, + "loss": 0.186, + "step": 1890 + }, + { + "epoch": 0.61, + "learning_rate": 0.00018441140877385913, + "loss": 0.1778, + "step": 1895 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018431746163754943, + "loss": 0.1786, + "step": 1900 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018422325633781327, + "loss": 0.1826, + "step": 1905 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018412879316308866, + "loss": 0.1738, + "step": 1910 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018403407240260327, + "loss": 0.1852, + "step": 1915 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018393909434637329, + "loss": 0.1821, + "step": 1920 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001838438592852028, + "loss": 0.1773, + "step": 1925 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018374836751068282, + "loss": 0.1772, + "step": 1930 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018365261931519022, + "loss": 0.1766, + "step": 1935 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018355661499188717, + "loss": 0.1858, + "step": 1940 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018346035483471994, + "loss": 0.1786, + "step": 1945 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001833638391384181, + "loss": 0.1745, + "step": 1950 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018326706819849365, + "loss": 0.1713, + "step": 1955 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018317004231124017, + "loss": 0.1738, + "step": 1960 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018307276177373174, + "loss": 0.1807, + "step": 1965 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018297522688382218, + "loss": 0.1806, + "step": 1970 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018287743794014407, + "loss": 0.1735, + "step": 1975 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018277939524210783, + "loss": 0.1812, + "step": 1980 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001826810990899009, + "loss": 0.1734, + "step": 1985 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001825825497844867, + "loss": 0.1703, + "step": 1990 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018248374762760365, + "loss": 0.1823, + "step": 1995 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018238469292176457, + "loss": 0.1837, + "step": 2000 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018228538597025537, + "loss": 0.1807, + "step": 2005 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018218582707713434, + "loss": 0.1904, + "step": 2010 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001820860165472312, + "loss": 0.1844, + "step": 2015 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018198595468614604, + "loss": 0.1792, + "step": 2020 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018188564180024854, + "loss": 0.1845, + "step": 2025 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018178507819667698, + "loss": 0.1853, + "step": 2030 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018168426418333725, + "loss": 0.1875, + "step": 2035 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018158320006890198, + "loss": 0.1759, + "step": 2040 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018148188616280955, + "loss": 0.1816, + "step": 2045 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001813803227752631, + "loss": 0.1787, + "step": 2050 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001812785102172297, + "loss": 0.1758, + "step": 2055 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018117644880043932, + "loss": 0.1802, + "step": 2060 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018107413883738387, + "loss": 0.186, + "step": 2065 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018097158064131626, + "loss": 0.1743, + "step": 2070 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018086877452624944, + "loss": 0.1877, + "step": 2075 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018076572080695543, + "loss": 0.1742, + "step": 2080 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018066241979896444, + "loss": 0.1727, + "step": 2085 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018055887181856375, + "loss": 0.1806, + "step": 2090 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018045507718279677, + "loss": 0.175, + "step": 2095 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018035103620946225, + "loss": 0.1727, + "step": 2100 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001802467492171131, + "loss": 0.1779, + "step": 2105 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018014221652505556, + "loss": 0.1823, + "step": 2110 + }, + { + "epoch": 0.69, + "learning_rate": 0.000180037438453348, + "loss": 0.1814, + "step": 2115 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001799324153228003, + "loss": 0.1707, + "step": 2120 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001798271474549725, + "loss": 0.1778, + "step": 2125 + }, + { + "epoch": 0.69, + "learning_rate": 0.000179721635172174, + "loss": 0.1888, + "step": 2130 + }, + { + "epoch": 0.69, + "learning_rate": 0.00017961587879746266, + "loss": 0.1785, + "step": 2135 + }, + { + "epoch": 0.69, + "learning_rate": 0.00017950987865464365, + "loss": 0.1661, + "step": 2140 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017940363506826836, + "loss": 0.1792, + "step": 2145 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001792971483636338, + "loss": 0.1732, + "step": 2150 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017919041886678118, + "loss": 0.1771, + "step": 2155 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001790834469044952, + "loss": 0.173, + "step": 2160 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017897623280430284, + "loss": 0.1768, + "step": 2165 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017886877689447258, + "loss": 0.175, + "step": 2170 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001787610795040132, + "loss": 0.1691, + "step": 2175 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001786531409626728, + "loss": 0.1774, + "step": 2180 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001785449616009379, + "loss": 0.1709, + "step": 2185 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001784365417500324, + "loss": 0.1809, + "step": 2190 + }, + { + "epoch": 0.71, + "learning_rate": 0.00017832788174191645, + "loss": 0.1764, + "step": 2195 + }, + { + "epoch": 0.71, + "learning_rate": 0.00017821898190928558, + "loss": 0.1739, + "step": 2200 + }, + { + "epoch": 0.71, + "learning_rate": 0.00017810984258556957, + "loss": 0.1772, + "step": 2205 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017800046410493145, + "loss": 0.1779, + "step": 2210 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001778908468022666, + "loss": 0.1828, + "step": 2215 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017778099101320156, + "loss": 0.1722, + "step": 2220 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017767089707409304, + "loss": 0.1737, + "step": 2225 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017756056532202696, + "loss": 0.1688, + "step": 2230 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017744999609481738, + "loss": 0.1784, + "step": 2235 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017733918973100545, + "loss": 0.1747, + "step": 2240 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017722814656985841, + "loss": 0.1667, + "step": 2245 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001771168669513685, + "loss": 0.1648, + "step": 2250 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017700535121625187, + "loss": 0.1777, + "step": 2255 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017689359970594779, + "loss": 0.175, + "step": 2260 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001767816127626173, + "loss": 0.1715, + "step": 2265 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001766693907291423, + "loss": 0.1742, + "step": 2270 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017655693394912452, + "loss": 0.1868, + "step": 2275 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017644424276688441, + "loss": 0.1757, + "step": 2280 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017633131752746017, + "loss": 0.1803, + "step": 2285 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017621815857660656, + "loss": 0.1812, + "step": 2290 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017610476626079395, + "loss": 0.1691, + "step": 2295 + }, + { + "epoch": 0.75, + "learning_rate": 0.00017599114092720726, + "loss": 0.1662, + "step": 2300 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001758772829237448, + "loss": 0.1709, + "step": 2305 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001757631925990174, + "loss": 0.1822, + "step": 2310 + }, + { + "epoch": 0.75, + "learning_rate": 0.00017564887030234696, + "loss": 0.1732, + "step": 2315 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001755343163837659, + "loss": 0.1854, + "step": 2320 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001754195311940157, + "loss": 0.1768, + "step": 2325 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017530451508454594, + "loss": 0.1785, + "step": 2330 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017518926840751325, + "loss": 0.1724, + "step": 2335 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017507379151578017, + "loss": 0.1812, + "step": 2340 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017495808476291423, + "loss": 0.1754, + "step": 2345 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017484214850318664, + "loss": 0.1799, + "step": 2350 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001747259830915713, + "loss": 0.1769, + "step": 2355 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017460958888374385, + "loss": 0.1764, + "step": 2360 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001744929662360803, + "loss": 0.1716, + "step": 2365 + }, + { + "epoch": 0.77, + "learning_rate": 0.00017437611550565624, + "loss": 0.1699, + "step": 2370 + }, + { + "epoch": 0.77, + "learning_rate": 0.00017425903705024553, + "loss": 0.1661, + "step": 2375 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001741417312283193, + "loss": 0.1699, + "step": 2380 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001740241983990448, + "loss": 0.1682, + "step": 2385 + }, + { + "epoch": 0.77, + "learning_rate": 0.00017390643892228438, + "loss": 0.173, + "step": 2390 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001737884531585943, + "loss": 0.1852, + "step": 2395 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017367024146922372, + "loss": 0.1681, + "step": 2400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017355180421611347, + "loss": 0.1798, + "step": 2405 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017343314176189502, + "loss": 0.1705, + "step": 2410 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017331425446988946, + "loss": 0.177, + "step": 2415 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017319514270410614, + "loss": 0.1754, + "step": 2420 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017307580682924178, + "loss": 0.1679, + "step": 2425 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017295624721067932, + "loss": 0.1763, + "step": 2430 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017283646421448673, + "loss": 0.1769, + "step": 2435 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017271645820741583, + "loss": 0.1753, + "step": 2440 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001725962295569014, + "loss": 0.1715, + "step": 2445 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017247577863105983, + "loss": 0.1784, + "step": 2450 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017235510579868802, + "loss": 0.17, + "step": 2455 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017223421142926245, + "loss": 0.1686, + "step": 2460 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017211309589293774, + "loss": 0.166, + "step": 2465 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017199175956054584, + "loss": 0.1652, + "step": 2470 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017187020280359455, + "loss": 0.1732, + "step": 2475 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001717484259942668, + "loss": 0.1754, + "step": 2480 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017162642950541902, + "loss": 0.1748, + "step": 2485 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017150421371058049, + "loss": 0.1733, + "step": 2490 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017138177898395177, + "loss": 0.1737, + "step": 2495 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017125912570040383, + "loss": 0.175, + "step": 2500 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001711362542354769, + "loss": 0.173, + "step": 2505 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017101316496537905, + "loss": 0.1796, + "step": 2510 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017088985826698542, + "loss": 0.1722, + "step": 2515 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017076633451783675, + "loss": 0.1738, + "step": 2520 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017064259409613834, + "loss": 0.1743, + "step": 2525 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017051863738075902, + "loss": 0.1653, + "step": 2530 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001703944647512297, + "loss": 0.1761, + "step": 2535 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017027007658774256, + "loss": 0.1649, + "step": 2540 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017014547327114956, + "loss": 0.1738, + "step": 2545 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001700206551829614, + "loss": 0.1585, + "step": 2550 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016989562270534655, + "loss": 0.183, + "step": 2555 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001697703762211297, + "loss": 0.1693, + "step": 2560 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001696449161137909, + "loss": 0.173, + "step": 2565 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016951924276746425, + "loss": 0.1691, + "step": 2570 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016939335656693667, + "loss": 0.1698, + "step": 2575 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016926725789764691, + "loss": 0.1706, + "step": 2580 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016914094714568417, + "loss": 0.1691, + "step": 2585 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016901442469778713, + "loss": 0.1665, + "step": 2590 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016888769094134247, + "loss": 0.1831, + "step": 2595 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016876074626438395, + "loss": 0.1637, + "step": 2600 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016863359105559117, + "loss": 0.1757, + "step": 2605 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016850622570428826, + "loss": 0.1721, + "step": 2610 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016837865060044272, + "loss": 0.1647, + "step": 2615 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016825086613466442, + "loss": 0.1734, + "step": 2620 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016812287269820416, + "loss": 0.1847, + "step": 2625 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016799467068295257, + "loss": 0.1682, + "step": 2630 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001678662604814389, + "loss": 0.1741, + "step": 2635 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016773764248682988, + "loss": 0.1692, + "step": 2640 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016760881709292842, + "loss": 0.1805, + "step": 2645 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001674797846941724, + "loss": 0.1685, + "step": 2650 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016735054568563364, + "loss": 0.1673, + "step": 2655 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001672211004630164, + "loss": 0.1686, + "step": 2660 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016709144942265644, + "loss": 0.1749, + "step": 2665 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001669615929615197, + "loss": 0.1644, + "step": 2670 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016683153147720097, + "loss": 0.1673, + "step": 2675 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016670126536792287, + "loss": 0.1722, + "step": 2680 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001665707950325345, + "loss": 0.1588, + "step": 2685 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001664401208705103, + "loss": 0.164, + "step": 2690 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016630924328194877, + "loss": 0.1708, + "step": 2695 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016617816266757123, + "loss": 0.1711, + "step": 2700 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016604687942872067, + "loss": 0.1701, + "step": 2705 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016591539396736046, + "loss": 0.1708, + "step": 2710 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016578370668607307, + "loss": 0.1669, + "step": 2715 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016565181798805907, + "loss": 0.1601, + "step": 2720 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016551972827713555, + "loss": 0.1733, + "step": 2725 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016538743795773515, + "loss": 0.1641, + "step": 2730 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016525494743490474, + "loss": 0.1723, + "step": 2735 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016512225711430415, + "loss": 0.1655, + "step": 2740 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016498936740220492, + "loss": 0.172, + "step": 2745 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016485627870548917, + "loss": 0.1686, + "step": 2750 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016472299143164816, + "loss": 0.1709, + "step": 2755 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016458950598878126, + "loss": 0.1648, + "step": 2760 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016445582278559458, + "loss": 0.1641, + "step": 2765 + }, + { + "epoch": 0.9, + "learning_rate": 0.0001643219422313996, + "loss": 0.1662, + "step": 2770 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016418786473611227, + "loss": 0.1763, + "step": 2775 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016405359071025138, + "loss": 0.1773, + "step": 2780 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016391912056493743, + "loss": 0.1678, + "step": 2785 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016378445471189156, + "loss": 0.1632, + "step": 2790 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016364959356343398, + "loss": 0.1671, + "step": 2795 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016351453753248296, + "loss": 0.1739, + "step": 2800 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016337928703255336, + "loss": 0.1727, + "step": 2805 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016324384247775554, + "loss": 0.1756, + "step": 2810 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016310820428279397, + "loss": 0.1671, + "step": 2815 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016297237286296606, + "loss": 0.169, + "step": 2820 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016283634863416082, + "loss": 0.1693, + "step": 2825 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016270013201285757, + "loss": 0.164, + "step": 2830 + }, + { + "epoch": 0.92, + "learning_rate": 0.0001625637234161247, + "loss": 0.1663, + "step": 2835 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016242712326161845, + "loss": 0.1742, + "step": 2840 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016229033196758154, + "loss": 0.169, + "step": 2845 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016215334995284188, + "loss": 0.1703, + "step": 2850 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001620161776368114, + "loss": 0.1669, + "step": 2855 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001618788154394846, + "loss": 0.1634, + "step": 2860 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001617412637814375, + "loss": 0.163, + "step": 2865 + }, + { + "epoch": 0.93, + "learning_rate": 0.00016160352308382608, + "loss": 0.1651, + "step": 2870 + }, + { + "epoch": 0.93, + "learning_rate": 0.00016146559376838514, + "loss": 0.1657, + "step": 2875 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001613274762574271, + "loss": 0.1638, + "step": 2880 + }, + { + "epoch": 0.93, + "learning_rate": 0.00016118917097384048, + "loss": 0.1691, + "step": 2885 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016105067834108874, + "loss": 0.1627, + "step": 2890 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016091199878320902, + "loss": 0.1608, + "step": 2895 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016077313272481076, + "loss": 0.1617, + "step": 2900 + }, + { + "epoch": 0.94, + "learning_rate": 0.0001606340805910744, + "loss": 0.1696, + "step": 2905 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016049484280775012, + "loss": 0.1604, + "step": 2910 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016035541980115653, + "loss": 0.161, + "step": 2915 + }, + { + "epoch": 0.95, + "learning_rate": 0.00016021581199817943, + "loss": 0.1703, + "step": 2920 + }, + { + "epoch": 0.95, + "learning_rate": 0.0001600760198262703, + "loss": 0.1671, + "step": 2925 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015993604371344517, + "loss": 0.1603, + "step": 2930 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015979588408828333, + "loss": 0.1655, + "step": 2935 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015965554137992584, + "loss": 0.1675, + "step": 2940 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015951501601807437, + "loss": 0.1641, + "step": 2945 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015937430843298986, + "loss": 0.165, + "step": 2950 + }, + { + "epoch": 0.96, + "learning_rate": 0.0001592334190554912, + "loss": 0.1758, + "step": 2955 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015909234831695379, + "loss": 0.1703, + "step": 2960 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015895109664930846, + "loss": 0.1644, + "step": 2965 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015880966448503987, + "loss": 0.1638, + "step": 2970 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015866805225718542, + "loss": 0.1659, + "step": 2975 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015852626039933383, + "loss": 0.1676, + "step": 2980 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015838428934562373, + "loss": 0.1794, + "step": 2985 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015824213953074254, + "loss": 0.1562, + "step": 2990 + }, + { + "epoch": 0.97, + "learning_rate": 0.0001580998113899249, + "loss": 0.171, + "step": 2995 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015795730535895147, + "loss": 0.1681, + "step": 3000 + }, + { + "epoch": 0.97, + "learning_rate": 0.0001578146218741476, + "loss": 0.1617, + "step": 3005 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015767176137238196, + "loss": 0.1708, + "step": 3010 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015752872429106523, + "loss": 0.1636, + "step": 3015 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015738551106814872, + "loss": 0.1678, + "step": 3020 + }, + { + "epoch": 0.98, + "learning_rate": 0.000157242122142123, + "loss": 0.1612, + "step": 3025 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015709855795201675, + "loss": 0.1658, + "step": 3030 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015695481893739514, + "loss": 0.1621, + "step": 3035 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015681090553835864, + "loss": 0.1718, + "step": 3040 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015666681819554175, + "loss": 0.1712, + "step": 3045 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015652255735011135, + "loss": 0.1693, + "step": 3050 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015637812344376583, + "loss": 0.1714, + "step": 3055 + }, + { + "epoch": 0.99, + "learning_rate": 0.0001562335169187332, + "loss": 0.1661, + "step": 3060 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015608873821777012, + "loss": 0.1676, + "step": 3065 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015594378778416044, + "loss": 0.1654, + "step": 3070 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015579866606171372, + "loss": 0.1644, + "step": 3075 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015565337349476404, + "loss": 0.1744, + "step": 3080 + }, + { + "epoch": 1.0, + "learning_rate": 0.0001555079105281686, + "loss": 0.1606, + "step": 3085 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015536227760730632, + "loss": 0.1659, + "step": 3090 + }, + { + "epoch": 1.0, + "learning_rate": 0.0001552164751780764, + "loss": 0.1602, + "step": 3095 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015507050368689714, + "loss": 0.1636, + "step": 3100 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015492436358070438, + "loss": 0.1555, + "step": 3105 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015477805530695035, + "loss": 0.1661, + "step": 3110 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015463157931360207, + "loss": 0.1628, + "step": 3115 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015448493604914014, + "loss": 0.1696, + "step": 3120 + }, + { + "epoch": 1.01, + "learning_rate": 0.0001543381259625573, + "loss": 0.1685, + "step": 3125 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015419114950335704, + "loss": 0.1635, + "step": 3130 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015404400712155228, + "loss": 0.1631, + "step": 3135 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015389669926766395, + "loss": 0.1664, + "step": 3140 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015374922639271963, + "loss": 0.1617, + "step": 3145 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015360158894825216, + "loss": 0.174, + "step": 3150 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015345378738629826, + "loss": 0.1591, + "step": 3155 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015330582215939713, + "loss": 0.1729, + "step": 3160 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015315769372058907, + "loss": 0.1732, + "step": 3165 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015300940252341416, + "loss": 0.1608, + "step": 3170 + }, + { + "epoch": 1.03, + "learning_rate": 0.0001528609490219108, + "loss": 0.168, + "step": 3175 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015271233367061425, + "loss": 0.1731, + "step": 3180 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015256355692455542, + "loss": 0.1656, + "step": 3185 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015241461923925934, + "loss": 0.1591, + "step": 3190 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015226552107074385, + "loss": 0.1635, + "step": 3195 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015211626287551803, + "loss": 0.1622, + "step": 3200 + }, + { + "epoch": 1.04, + "learning_rate": 0.0001519668451105811, + "loss": 0.162, + "step": 3205 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015181726823342072, + "loss": 0.1614, + "step": 3210 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015166753270201176, + "loss": 0.1657, + "step": 3215 + }, + { + "epoch": 1.04, + "learning_rate": 0.0001515176389748149, + "loss": 0.1623, + "step": 3220 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015136758751077504, + "loss": 0.1629, + "step": 3225 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015121737876932034, + "loss": 0.1556, + "step": 3230 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015106701321036017, + "loss": 0.1651, + "step": 3235 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015091649129428424, + "loss": 0.1575, + "step": 3240 + }, + { + "epoch": 1.05, + "learning_rate": 0.000150765813481961, + "loss": 0.1594, + "step": 3245 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015061498023473608, + "loss": 0.1624, + "step": 3250 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015046399201443114, + "loss": 0.1616, + "step": 3255 + }, + { + "epoch": 1.06, + "learning_rate": 0.00015031284928334238, + "loss": 0.1693, + "step": 3260 + }, + { + "epoch": 1.06, + "learning_rate": 0.00015016155250423892, + "loss": 0.1592, + "step": 3265 + }, + { + "epoch": 1.06, + "learning_rate": 0.00015001010214036174, + "loss": 0.1542, + "step": 3270 + }, + { + "epoch": 1.06, + "learning_rate": 0.0001498584986554219, + "loss": 0.1631, + "step": 3275 + }, + { + "epoch": 1.06, + "learning_rate": 0.0001497067425135993, + "loss": 0.1612, + "step": 3280 + }, + { + "epoch": 1.06, + "learning_rate": 0.00014955483417954137, + "loss": 0.1551, + "step": 3285 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014940277411836146, + "loss": 0.155, + "step": 3290 + }, + { + "epoch": 1.07, + "learning_rate": 0.0001492505627956374, + "loss": 0.1593, + "step": 3295 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014909820067741028, + "loss": 0.1622, + "step": 3300 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014894568823018277, + "loss": 0.1637, + "step": 3305 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014879302592091798, + "loss": 0.1643, + "step": 3310 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014864021421703768, + "loss": 0.1642, + "step": 3315 + }, + { + "epoch": 1.08, + "learning_rate": 0.00014848725358642114, + "loss": 0.1607, + "step": 3320 + }, + { + "epoch": 1.08, + "learning_rate": 0.00014833414449740373, + "loss": 0.1591, + "step": 3325 + }, + { + "epoch": 1.08, + "learning_rate": 0.0001481808874187752, + "loss": 0.1682, + "step": 3330 + }, + { + "epoch": 1.08, + "learning_rate": 0.00014802748281977842, + "loss": 0.1572, + "step": 3335 + }, + { + "epoch": 1.08, + "learning_rate": 0.0001478739311701081, + "loss": 0.1595, + "step": 3340 + }, + { + "epoch": 1.08, + "learning_rate": 0.000147720232939909, + "loss": 0.1683, + "step": 3345 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014756638859977483, + "loss": 0.1539, + "step": 3350 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014741239862074655, + "loss": 0.1594, + "step": 3355 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014725826347431113, + "loss": 0.1658, + "step": 3360 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014710398363239994, + "loss": 0.1621, + "step": 3365 + }, + { + "epoch": 1.09, + "learning_rate": 0.0001469495595673874, + "loss": 0.1651, + "step": 3370 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014679499175208953, + "loss": 0.1655, + "step": 3375 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014664028065976246, + "loss": 0.1669, + "step": 3380 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014648542676410105, + "loss": 0.1593, + "step": 3385 + }, + { + "epoch": 1.1, + "learning_rate": 0.0001463304305392373, + "loss": 0.157, + "step": 3390 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014617529245973917, + "loss": 0.1593, + "step": 3395 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014602001300060877, + "loss": 0.1516, + "step": 3400 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014586459263728115, + "loss": 0.156, + "step": 3405 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014570903184562282, + "loss": 0.1637, + "step": 3410 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014555333110193022, + "loss": 0.1552, + "step": 3415 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014539749088292826, + "loss": 0.1624, + "step": 3420 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014524151166576897, + "loss": 0.1568, + "step": 3425 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014508539392802994, + "loss": 0.1583, + "step": 3430 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014492913814771283, + "loss": 0.1625, + "step": 3435 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014477274480324202, + "loss": 0.1563, + "step": 3440 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014461621437346307, + "loss": 0.1634, + "step": 3445 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014445954733764127, + "loss": 0.1452, + "step": 3450 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014430274417546012, + "loss": 0.1539, + "step": 3455 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014414580536702, + "loss": 0.1636, + "step": 3460 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014398873139283657, + "loss": 0.1509, + "step": 3465 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014383152273383934, + "loss": 0.1636, + "step": 3470 + }, + { + "epoch": 1.13, + "learning_rate": 0.00014367417987137015, + "loss": 0.1619, + "step": 3475 + }, + { + "epoch": 1.13, + "learning_rate": 0.00014351670328718183, + "loss": 0.1648, + "step": 3480 + }, + { + "epoch": 1.13, + "learning_rate": 0.00014335909346343658, + "loss": 0.1508, + "step": 3485 + }, + { + "epoch": 1.13, + "learning_rate": 0.0001432013508827046, + "loss": 0.1546, + "step": 3490 + }, + { + "epoch": 1.13, + "learning_rate": 0.0001430434760279625, + "loss": 0.1549, + "step": 3495 + }, + { + "epoch": 1.13, + "learning_rate": 0.000142885469382592, + "loss": 0.1412, + "step": 3500 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001427273314303782, + "loss": 0.1574, + "step": 3505 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001425690626555083, + "loss": 0.16, + "step": 3510 + }, + { + "epoch": 1.14, + "learning_rate": 0.00014241066354257008, + "loss": 0.1633, + "step": 3515 + }, + { + "epoch": 1.14, + "learning_rate": 0.00014225213457655034, + "loss": 0.1499, + "step": 3520 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001420934762428335, + "loss": 0.159, + "step": 3525 + }, + { + "epoch": 1.14, + "learning_rate": 0.00014193468902720007, + "loss": 0.1561, + "step": 3530 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014177577341582516, + "loss": 0.1619, + "step": 3535 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014161672989527698, + "loss": 0.16, + "step": 3540 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014145755895251546, + "loss": 0.148, + "step": 3545 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014129826107489054, + "loss": 0.1502, + "step": 3550 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014113883675014098, + "loss": 0.1427, + "step": 3555 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001409792864663925, + "loss": 0.1613, + "step": 3560 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001408196107121566, + "loss": 0.1597, + "step": 3565 + }, + { + "epoch": 1.16, + "learning_rate": 0.00014065980997632896, + "loss": 0.1501, + "step": 3570 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001404998847481879, + "loss": 0.1537, + "step": 3575 + }, + { + "epoch": 1.16, + "learning_rate": 0.00014033983551739286, + "loss": 0.1608, + "step": 3580 + }, + { + "epoch": 1.16, + "learning_rate": 0.00014017966277398304, + "loss": 0.1488, + "step": 3585 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001400193670083757, + "loss": 0.1593, + "step": 3590 + }, + { + "epoch": 1.16, + "learning_rate": 0.00013985894871136492, + "loss": 0.1525, + "step": 3595 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013969840837411982, + "loss": 0.1529, + "step": 3600 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013953774648818324, + "loss": 0.1489, + "step": 3605 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013937696354547022, + "loss": 0.144, + "step": 3610 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013921606003826627, + "loss": 0.1584, + "step": 3615 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013905503645922628, + "loss": 0.1657, + "step": 3620 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013889389330137264, + "loss": 0.1557, + "step": 3625 + }, + { + "epoch": 1.18, + "learning_rate": 0.00013873263105809388, + "loss": 0.1551, + "step": 3630 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001385712502231432, + "loss": 0.157, + "step": 3635 + }, + { + "epoch": 1.18, + "learning_rate": 0.00013840975129063684, + "loss": 0.1548, + "step": 3640 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001382481347550526, + "loss": 0.1565, + "step": 3645 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001380864011112285, + "loss": 0.1497, + "step": 3650 + }, + { + "epoch": 1.18, + "learning_rate": 0.000137924550854361, + "loss": 0.1569, + "step": 3655 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001377625844800037, + "loss": 0.1582, + "step": 3660 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013760050248406557, + "loss": 0.1571, + "step": 3665 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001374383053628098, + "loss": 0.1524, + "step": 3670 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013727599361285187, + "loss": 0.1567, + "step": 3675 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013711356773115838, + "loss": 0.1548, + "step": 3680 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013695102821504533, + "loss": 0.1557, + "step": 3685 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001367883755621766, + "loss": 0.148, + "step": 3690 + }, + { + "epoch": 1.2, + "learning_rate": 0.00013662561027056252, + "loss": 0.1604, + "step": 3695 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001364627328385583, + "loss": 0.15, + "step": 3700 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001362997437648625, + "loss": 0.1476, + "step": 3705 + }, + { + "epoch": 1.2, + "learning_rate": 0.00013613664354851547, + "loss": 0.1586, + "step": 3710 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001359734326888979, + "loss": 0.1536, + "step": 3715 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013581011168572923, + "loss": 0.1564, + "step": 3720 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013564668103906616, + "loss": 0.1534, + "step": 3725 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013548314124930104, + "loss": 0.1521, + "step": 3730 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013531949281716044, + "loss": 0.1489, + "step": 3735 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013515573624370357, + "loss": 0.1528, + "step": 3740 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001349918720303207, + "loss": 0.1405, + "step": 3745 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013482790067873178, + "loss": 0.1457, + "step": 3750 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013466382269098468, + "loss": 0.1554, + "step": 3755 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013449963856945383, + "loss": 0.1478, + "step": 3760 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013433534881683858, + "loss": 0.1633, + "step": 3765 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013417095393616178, + "loss": 0.15, + "step": 3770 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013400645443076804, + "loss": 0.1552, + "step": 3775 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013384185080432244, + "loss": 0.1485, + "step": 3780 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001336771435608088, + "loss": 0.1531, + "step": 3785 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013351233320452812, + "loss": 0.158, + "step": 3790 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013334742024009725, + "loss": 0.1554, + "step": 3795 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013318240517244713, + "loss": 0.1537, + "step": 3800 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001330172885068213, + "loss": 0.1584, + "step": 3805 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013285207074877447, + "loss": 0.1612, + "step": 3810 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013268675240417073, + "loss": 0.1449, + "step": 3815 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013252133397918226, + "loss": 0.1456, + "step": 3820 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001323558159802876, + "loss": 0.1541, + "step": 3825 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013219019891427025, + "loss": 0.1492, + "step": 3830 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013202448328821694, + "loss": 0.1528, + "step": 3835 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013185866960951622, + "loss": 0.1516, + "step": 3840 + }, + { + "epoch": 1.25, + "learning_rate": 0.0001316927583858569, + "loss": 0.1484, + "step": 3845 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013152675012522629, + "loss": 0.1509, + "step": 3850 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013136064533590897, + "loss": 0.1516, + "step": 3855 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013119444452648504, + "loss": 0.15, + "step": 3860 + }, + { + "epoch": 1.25, + "learning_rate": 0.0001310281482058286, + "loss": 0.1571, + "step": 3865 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013086175688310612, + "loss": 0.1526, + "step": 3870 + }, + { + "epoch": 1.26, + "learning_rate": 0.000130695271067775, + "loss": 0.1445, + "step": 3875 + }, + { + "epoch": 1.26, + "learning_rate": 0.00013052869126958198, + "loss": 0.1598, + "step": 3880 + }, + { + "epoch": 1.26, + "learning_rate": 0.00013036201799856149, + "loss": 0.1499, + "step": 3885 + }, + { + "epoch": 1.26, + "learning_rate": 0.0001301952517650342, + "loss": 0.1507, + "step": 3890 + }, + { + "epoch": 1.26, + "learning_rate": 0.0001300283930796054, + "loss": 0.1489, + "step": 3895 + }, + { + "epoch": 1.26, + "learning_rate": 0.00012986144245316346, + "loss": 0.1477, + "step": 3900 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012969440039687822, + "loss": 0.1528, + "step": 3905 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012952726742219953, + "loss": 0.1514, + "step": 3910 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012936004404085557, + "loss": 0.1482, + "step": 3915 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012919273076485133, + "loss": 0.1558, + "step": 3920 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012902532810646697, + "loss": 0.1539, + "step": 3925 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012885783657825647, + "loss": 0.1479, + "step": 3930 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012869025669304585, + "loss": 0.1449, + "step": 3935 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012852258896393158, + "loss": 0.1489, + "step": 3940 + }, + { + "epoch": 1.28, + "learning_rate": 0.0001283548339042792, + "loss": 0.1479, + "step": 3945 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012818699202772152, + "loss": 0.149, + "step": 3950 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012801906384815733, + "loss": 0.1472, + "step": 3955 + }, + { + "epoch": 1.28, + "learning_rate": 0.0001278510498797495, + "loss": 0.1526, + "step": 3960 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012768295063692376, + "loss": 0.1469, + "step": 3965 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012751476663436668, + "loss": 0.1576, + "step": 3970 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012734649838702455, + "loss": 0.1583, + "step": 3975 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012717814641010153, + "loss": 0.1434, + "step": 3980 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012700971121905812, + "loss": 0.1463, + "step": 3985 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012684119332960963, + "loss": 0.1543, + "step": 3990 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012667259325772464, + "loss": 0.1501, + "step": 3995 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012650391151962324, + "loss": 0.1475, + "step": 4000 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012633514863177566, + "loss": 0.147, + "step": 4005 + }, + { + "epoch": 1.3, + "learning_rate": 0.0001261663051109005, + "loss": 0.1454, + "step": 4010 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012599738147396332, + "loss": 0.1459, + "step": 4015 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012582837823817497, + "loss": 0.1479, + "step": 4020 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012565929592099002, + "loss": 0.1541, + "step": 4025 + }, + { + "epoch": 1.31, + "learning_rate": 0.00012549013504010512, + "loss": 0.145, + "step": 4030 + }, + { + "epoch": 1.31, + "learning_rate": 0.0001253208961134575, + "loss": 0.1467, + "step": 4035 + }, + { + "epoch": 1.31, + "learning_rate": 0.0001251515796592234, + "loss": 0.1497, + "step": 4040 + }, + { + "epoch": 1.31, + "learning_rate": 0.00012498218619581632, + "loss": 0.1439, + "step": 4045 + }, + { + "epoch": 1.31, + "learning_rate": 0.00012481271624188568, + "loss": 0.1419, + "step": 4050 + }, + { + "epoch": 1.31, + "learning_rate": 0.00012464317031631494, + "loss": 0.1518, + "step": 4055 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012447354893822034, + "loss": 0.1462, + "step": 4060 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012430385262694906, + "loss": 0.1473, + "step": 4065 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012413408190207768, + "loss": 0.1495, + "step": 4070 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012396423728341067, + "loss": 0.1566, + "step": 4075 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012379431929097873, + "loss": 0.1507, + "step": 4080 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012362432844503725, + "loss": 0.1544, + "step": 4085 + }, + { + "epoch": 1.33, + "learning_rate": 0.0001234542652660646, + "loss": 0.1528, + "step": 4090 + }, + { + "epoch": 1.33, + "learning_rate": 0.00012328413027476066, + "loss": 0.153, + "step": 4095 + }, + { + "epoch": 1.33, + "learning_rate": 0.00012311392399204526, + "loss": 0.1438, + "step": 4100 + }, + { + "epoch": 1.33, + "learning_rate": 0.0001229436469390564, + "loss": 0.1497, + "step": 4105 + }, + { + "epoch": 1.33, + "learning_rate": 0.00012277329963714882, + "loss": 0.1471, + "step": 4110 + }, + { + "epoch": 1.33, + "learning_rate": 0.0001226028826078923, + "loss": 0.1442, + "step": 4115 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012243239637307023, + "loss": 0.1455, + "step": 4120 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012226184145467775, + "loss": 0.1509, + "step": 4125 + }, + { + "epoch": 1.34, + "learning_rate": 0.0001220912183749204, + "loss": 0.1504, + "step": 4130 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012192052765621234, + "loss": 0.1524, + "step": 4135 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012174976982117488, + "loss": 0.1526, + "step": 4140 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012157894539263482, + "loss": 0.1501, + "step": 4145 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012140805489362281, + "loss": 0.1464, + "step": 4150 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012123709884737189, + "loss": 0.1481, + "step": 4155 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012106607777731567, + "loss": 0.1483, + "step": 4160 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012089499220708696, + "loss": 0.1456, + "step": 4165 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012072384266051595, + "loss": 0.1479, + "step": 4170 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012055262966162883, + "loss": 0.1488, + "step": 4175 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012038135373464602, + "loss": 0.1468, + "step": 4180 + }, + { + "epoch": 1.36, + "learning_rate": 0.00012021001540398058, + "loss": 0.1412, + "step": 4185 + }, + { + "epoch": 1.36, + "learning_rate": 0.00012003861519423662, + "loss": 0.1503, + "step": 4190 + }, + { + "epoch": 1.36, + "learning_rate": 0.00011986715363020784, + "loss": 0.1497, + "step": 4195 + }, + { + "epoch": 1.36, + "learning_rate": 0.00011969563123687566, + "loss": 0.1457, + "step": 4200 + }, + { + "epoch": 1.36, + "learning_rate": 0.00011952404853940782, + "loss": 0.1469, + "step": 4205 + }, + { + "epoch": 1.36, + "learning_rate": 0.00011935240606315666, + "loss": 0.1485, + "step": 4210 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011918070433365754, + "loss": 0.1413, + "step": 4215 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011900894387662731, + "loss": 0.1565, + "step": 4220 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011883712521796253, + "loss": 0.1516, + "step": 4225 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011866524888373811, + "loss": 0.146, + "step": 4230 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011849331540020543, + "loss": 0.1507, + "step": 4235 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011832132529379088, + "loss": 0.1493, + "step": 4240 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011814927909109419, + "loss": 0.1419, + "step": 4245 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011797717731888691, + "loss": 0.1414, + "step": 4250 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011780502050411069, + "loss": 0.1385, + "step": 4255 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011763280917387572, + "loss": 0.1473, + "step": 4260 + }, + { + "epoch": 1.38, + "learning_rate": 0.0001174605438554591, + "loss": 0.1496, + "step": 4265 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011728822507630326, + "loss": 0.1369, + "step": 4270 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011711585336401429, + "loss": 0.1437, + "step": 4275 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011694342924636031, + "loss": 0.1564, + "step": 4280 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011677095325126996, + "loss": 0.1443, + "step": 4285 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011659842590683074, + "loss": 0.1577, + "step": 4290 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011642584774128728, + "loss": 0.1416, + "step": 4295 + }, + { + "epoch": 1.39, + "learning_rate": 0.0001162532192830399, + "loss": 0.145, + "step": 4300 + }, + { + "epoch": 1.4, + "learning_rate": 0.00011608054106064281, + "loss": 0.1458, + "step": 4305 + }, + { + "epoch": 1.4, + "learning_rate": 0.0001159078136028027, + "loss": 0.1445, + "step": 4310 + }, + { + "epoch": 1.4, + "learning_rate": 0.00011573503743837693, + "loss": 0.139, + "step": 4315 + }, + { + "epoch": 1.4, + "learning_rate": 0.00011556221309637203, + "loss": 0.1441, + "step": 4320 + }, + { + "epoch": 1.4, + "learning_rate": 0.000115389341105942, + "loss": 0.1567, + "step": 4325 + }, + { + "epoch": 1.4, + "learning_rate": 0.0001152164219963868, + "loss": 0.1499, + "step": 4330 + }, + { + "epoch": 1.4, + "learning_rate": 0.00011504345629715057, + "loss": 0.15, + "step": 4335 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011487044453782016, + "loss": 0.1449, + "step": 4340 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011469738724812346, + "loss": 0.1423, + "step": 4345 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011452428495792773, + "loss": 0.1468, + "step": 4350 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011435113819723797, + "loss": 0.1519, + "step": 4355 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011417794749619544, + "loss": 0.1498, + "step": 4360 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011400471338507585, + "loss": 0.1406, + "step": 4365 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011383143639428791, + "loss": 0.1485, + "step": 4370 + }, + { + "epoch": 1.42, + "learning_rate": 0.0001136581170543715, + "loss": 0.1502, + "step": 4375 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011348475589599625, + "loss": 0.139, + "step": 4380 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011331135344995985, + "loss": 0.1485, + "step": 4385 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011313791024718634, + "loss": 0.133, + "step": 4390 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011296442681872457, + "loss": 0.1473, + "step": 4395 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011279090369574658, + "loss": 0.1401, + "step": 4400 + }, + { + "epoch": 1.43, + "learning_rate": 0.0001126173414095459, + "loss": 0.1483, + "step": 4405 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011244374049153603, + "loss": 0.1436, + "step": 4410 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011227010147324868, + "loss": 0.1445, + "step": 4415 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011209642488633227, + "loss": 0.1451, + "step": 4420 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011192271126255024, + "loss": 0.1438, + "step": 4425 + }, + { + "epoch": 1.44, + "learning_rate": 0.00011174896113377945, + "loss": 0.1474, + "step": 4430 + }, + { + "epoch": 1.44, + "learning_rate": 0.0001115751750320084, + "loss": 0.1444, + "step": 4435 + }, + { + "epoch": 1.44, + "learning_rate": 0.00011140135348933593, + "loss": 0.147, + "step": 4440 + }, + { + "epoch": 1.44, + "learning_rate": 0.00011122749703796929, + "loss": 0.1447, + "step": 4445 + }, + { + "epoch": 1.44, + "learning_rate": 0.0001110536062102226, + "loss": 0.1423, + "step": 4450 + }, + { + "epoch": 1.44, + "learning_rate": 0.00011087968153851525, + "loss": 0.1405, + "step": 4455 + }, + { + "epoch": 1.45, + "learning_rate": 0.00011070572355537026, + "loss": 0.1468, + "step": 4460 + }, + { + "epoch": 1.45, + "learning_rate": 0.00011053173279341265, + "loss": 0.1442, + "step": 4465 + }, + { + "epoch": 1.45, + "learning_rate": 0.0001103577097853678, + "loss": 0.1447, + "step": 4470 + }, + { + "epoch": 1.45, + "learning_rate": 0.00011018365506405978, + "loss": 0.1514, + "step": 4475 + }, + { + "epoch": 1.45, + "learning_rate": 0.00011000956916240985, + "loss": 0.148, + "step": 4480 + }, + { + "epoch": 1.45, + "learning_rate": 0.00010983545261343464, + "loss": 0.1499, + "step": 4485 + }, + { + "epoch": 1.45, + "learning_rate": 0.00010966130595024466, + "loss": 0.1398, + "step": 4490 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010948712970604266, + "loss": 0.1379, + "step": 4495 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010931292441412191, + "loss": 0.1438, + "step": 4500 + }, + { + "epoch": 1.46, + "learning_rate": 0.0001091386906078646, + "loss": 0.1453, + "step": 4505 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010896442882074029, + "loss": 0.1439, + "step": 4510 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010879013958630418, + "loss": 0.1436, + "step": 4515 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010861582343819548, + "loss": 0.144, + "step": 4520 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010844148091013581, + "loss": 0.1439, + "step": 4525 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010826711253592768, + "loss": 0.1441, + "step": 4530 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010809271884945256, + "loss": 0.1421, + "step": 4535 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010791830038466952, + "loss": 0.1476, + "step": 4540 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010774385767561347, + "loss": 0.1484, + "step": 4545 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010756939125639358, + "loss": 0.1437, + "step": 4550 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010739490166119155, + "loss": 0.1403, + "step": 4555 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010722038942426014, + "loss": 0.1437, + "step": 4560 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010704585507992131, + "loss": 0.1429, + "step": 4565 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010687129916256482, + "loss": 0.1401, + "step": 4570 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010669672220664644, + "loss": 0.1448, + "step": 4575 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010652212474668635, + "loss": 0.1449, + "step": 4580 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010634750731726748, + "loss": 0.1431, + "step": 4585 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010617287045303395, + "loss": 0.1467, + "step": 4590 + }, + { + "epoch": 1.49, + "learning_rate": 0.0001059982146886894, + "loss": 0.1403, + "step": 4595 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010582354055899528, + "loss": 0.1467, + "step": 4600 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010564884859876934, + "loss": 0.1399, + "step": 4605 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010547413934288386, + "loss": 0.1408, + "step": 4610 + }, + { + "epoch": 1.5, + "learning_rate": 0.0001052994133262641, + "loss": 0.1411, + "step": 4615 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010512467108388661, + "loss": 0.1398, + "step": 4620 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010494991315077772, + "loss": 0.1444, + "step": 4625 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010477514006201166, + "loss": 0.1438, + "step": 4630 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010460035235270918, + "loss": 0.1499, + "step": 4635 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010442555055803572, + "loss": 0.1451, + "step": 4640 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010425073521319986, + "loss": 0.15, + "step": 4645 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010407590685345174, + "loss": 0.1471, + "step": 4650 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010390106601408125, + "loss": 0.1454, + "step": 4655 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010372621323041656, + "loss": 0.1476, + "step": 4660 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010355134903782235, + "loss": 0.142, + "step": 4665 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010337647397169832, + "loss": 0.1391, + "step": 4670 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010320158856747737, + "loss": 0.1398, + "step": 4675 + }, + { + "epoch": 1.52, + "learning_rate": 0.0001030266933606241, + "loss": 0.15, + "step": 4680 + }, + { + "epoch": 1.52, + "learning_rate": 0.0001028517888866331, + "loss": 0.1387, + "step": 4685 + }, + { + "epoch": 1.52, + "learning_rate": 0.00010267687568102737, + "loss": 0.1484, + "step": 4690 + }, + { + "epoch": 1.52, + "learning_rate": 0.00010250195427935663, + "loss": 0.1483, + "step": 4695 + }, + { + "epoch": 1.52, + "learning_rate": 0.00010232702521719564, + "loss": 0.1452, + "step": 4700 + }, + { + "epoch": 1.52, + "learning_rate": 0.00010215208903014272, + "loss": 0.1481, + "step": 4705 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010197714625381795, + "loss": 0.1406, + "step": 4710 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010180219742386157, + "loss": 0.139, + "step": 4715 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010162724307593234, + "loss": 0.1354, + "step": 4720 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010145228374570599, + "loss": 0.1379, + "step": 4725 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010127731996887339, + "loss": 0.1481, + "step": 4730 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010110235228113918, + "loss": 0.1464, + "step": 4735 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010092738121821985, + "loss": 0.1426, + "step": 4740 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010075240731584223, + "loss": 0.1415, + "step": 4745 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010057743110974194, + "loss": 0.1393, + "step": 4750 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010040245313566153, + "loss": 0.1408, + "step": 4755 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010022747392934905, + "loss": 0.1358, + "step": 4760 + }, + { + "epoch": 1.54, + "learning_rate": 0.0001000524940265563, + "loss": 0.1401, + "step": 4765 + }, + { + "epoch": 1.55, + "learning_rate": 9.98775139630372e-05, + "loss": 0.1379, + "step": 4770 + }, + { + "epoch": 1.55, + "learning_rate": 9.970253427454616e-05, + "loss": 0.1425, + "step": 4775 + }, + { + "epoch": 1.55, + "learning_rate": 9.95275554968365e-05, + "loss": 0.1475, + "step": 4780 + }, + { + "epoch": 1.55, + "learning_rate": 9.935257816565861e-05, + "loss": 0.1364, + "step": 4785 + }, + { + "epoch": 1.55, + "learning_rate": 9.917760281675866e-05, + "loss": 0.1443, + "step": 4790 + }, + { + "epoch": 1.55, + "learning_rate": 9.900262998587653e-05, + "loss": 0.1388, + "step": 4795 + }, + { + "epoch": 1.56, + "learning_rate": 9.882766020874456e-05, + "loss": 0.1375, + "step": 4800 + }, + { + "epoch": 1.56, + "learning_rate": 9.865269402108565e-05, + "loss": 0.1413, + "step": 4805 + }, + { + "epoch": 1.56, + "learning_rate": 9.847773195861175e-05, + "loss": 0.1499, + "step": 4810 + }, + { + "epoch": 1.56, + "learning_rate": 9.830277455702213e-05, + "loss": 0.1421, + "step": 4815 + }, + { + "epoch": 1.56, + "learning_rate": 9.812782235200189e-05, + "loss": 0.146, + "step": 4820 + }, + { + "epoch": 1.56, + "learning_rate": 9.795287587922006e-05, + "loss": 0.1343, + "step": 4825 + }, + { + "epoch": 1.57, + "learning_rate": 9.777793567432828e-05, + "loss": 0.1451, + "step": 4830 + }, + { + "epoch": 1.57, + "learning_rate": 9.760300227295894e-05, + "loss": 0.1438, + "step": 4835 + }, + { + "epoch": 1.57, + "learning_rate": 9.742807621072353e-05, + "loss": 0.1436, + "step": 4840 + }, + { + "epoch": 1.57, + "learning_rate": 9.725315802321121e-05, + "loss": 0.1384, + "step": 4845 + }, + { + "epoch": 1.57, + "learning_rate": 9.707824824598684e-05, + "loss": 0.1368, + "step": 4850 + }, + { + "epoch": 1.57, + "learning_rate": 9.690334741458974e-05, + "loss": 0.1309, + "step": 4855 + }, + { + "epoch": 1.57, + "learning_rate": 9.672845606453168e-05, + "loss": 0.1384, + "step": 4860 + }, + { + "epoch": 1.58, + "learning_rate": 9.655357473129548e-05, + "loss": 0.1387, + "step": 4865 + }, + { + "epoch": 1.58, + "learning_rate": 9.637870395033324e-05, + "loss": 0.1426, + "step": 4870 + }, + { + "epoch": 1.58, + "learning_rate": 9.620384425706484e-05, + "loss": 0.1407, + "step": 4875 + }, + { + "epoch": 1.58, + "learning_rate": 9.602899618687606e-05, + "loss": 0.1413, + "step": 4880 + }, + { + "epoch": 1.58, + "learning_rate": 9.585416027511727e-05, + "loss": 0.1434, + "step": 4885 + }, + { + "epoch": 1.58, + "learning_rate": 9.567933705710143e-05, + "loss": 0.146, + "step": 4890 + }, + { + "epoch": 1.59, + "learning_rate": 9.550452706810281e-05, + "loss": 0.1384, + "step": 4895 + }, + { + "epoch": 1.59, + "learning_rate": 9.532973084335511e-05, + "loss": 0.1447, + "step": 4900 + }, + { + "epoch": 1.59, + "learning_rate": 9.515494891804979e-05, + "loss": 0.1418, + "step": 4905 + }, + { + "epoch": 1.59, + "learning_rate": 9.498018182733466e-05, + "loss": 0.144, + "step": 4910 + }, + { + "epoch": 1.59, + "learning_rate": 9.480543010631207e-05, + "loss": 0.1449, + "step": 4915 + }, + { + "epoch": 1.59, + "learning_rate": 9.463069429003728e-05, + "loss": 0.1391, + "step": 4920 + }, + { + "epoch": 1.6, + "learning_rate": 9.445597491351685e-05, + "loss": 0.1484, + "step": 4925 + }, + { + "epoch": 1.6, + "learning_rate": 9.42812725117071e-05, + "loss": 0.1402, + "step": 4930 + }, + { + "epoch": 1.6, + "learning_rate": 9.410658761951221e-05, + "loss": 0.1478, + "step": 4935 + }, + { + "epoch": 1.6, + "learning_rate": 9.393192077178295e-05, + "loss": 0.1431, + "step": 4940 + }, + { + "epoch": 1.6, + "learning_rate": 9.375727250331462e-05, + "loss": 0.1405, + "step": 4945 + }, + { + "epoch": 1.6, + "learning_rate": 9.358264334884585e-05, + "loss": 0.1359, + "step": 4950 + }, + { + "epoch": 1.61, + "learning_rate": 9.340803384305657e-05, + "loss": 0.1464, + "step": 4955 + }, + { + "epoch": 1.61, + "learning_rate": 9.323344452056668e-05, + "loss": 0.1408, + "step": 4960 + }, + { + "epoch": 1.61, + "learning_rate": 9.305887591593419e-05, + "loss": 0.1435, + "step": 4965 + }, + { + "epoch": 1.61, + "learning_rate": 9.288432856365374e-05, + "loss": 0.1449, + "step": 4970 + }, + { + "epoch": 1.61, + "learning_rate": 9.270980299815486e-05, + "loss": 0.1403, + "step": 4975 + }, + { + "epoch": 1.61, + "learning_rate": 9.253529975380039e-05, + "loss": 0.1434, + "step": 4980 + }, + { + "epoch": 1.62, + "learning_rate": 9.236081936488485e-05, + "loss": 0.1405, + "step": 4985 + }, + { + "epoch": 1.62, + "learning_rate": 9.21863623656327e-05, + "loss": 0.1394, + "step": 4990 + }, + { + "epoch": 1.62, + "learning_rate": 9.20119292901969e-05, + "loss": 0.1352, + "step": 4995 + }, + { + "epoch": 1.62, + "learning_rate": 9.183752067265703e-05, + "loss": 0.1452, + "step": 5000 + }, + { + "epoch": 1.62, + "learning_rate": 9.166313704701795e-05, + "loss": 0.1423, + "step": 5005 + }, + { + "epoch": 1.62, + "learning_rate": 9.14887789472078e-05, + "loss": 0.1378, + "step": 5010 + }, + { + "epoch": 1.63, + "learning_rate": 9.131444690707673e-05, + "loss": 0.1379, + "step": 5015 + }, + { + "epoch": 1.63, + "learning_rate": 9.114014146039503e-05, + "loss": 0.137, + "step": 5020 + }, + { + "epoch": 1.63, + "learning_rate": 9.096586314085162e-05, + "loss": 0.1459, + "step": 5025 + }, + { + "epoch": 1.63, + "learning_rate": 9.079161248205222e-05, + "loss": 0.1417, + "step": 5030 + }, + { + "epoch": 1.63, + "learning_rate": 9.061739001751803e-05, + "loss": 0.139, + "step": 5035 + }, + { + "epoch": 1.63, + "learning_rate": 9.044319628068387e-05, + "loss": 0.1322, + "step": 5040 + }, + { + "epoch": 1.63, + "learning_rate": 9.02690318048965e-05, + "loss": 0.1324, + "step": 5045 + }, + { + "epoch": 1.64, + "learning_rate": 9.009489712341327e-05, + "loss": 0.1403, + "step": 5050 + }, + { + "epoch": 1.64, + "learning_rate": 8.992079276940011e-05, + "loss": 0.1394, + "step": 5055 + }, + { + "epoch": 1.64, + "learning_rate": 8.974671927593028e-05, + "loss": 0.1389, + "step": 5060 + }, + { + "epoch": 1.64, + "learning_rate": 8.957267717598236e-05, + "loss": 0.141, + "step": 5065 + }, + { + "epoch": 1.64, + "learning_rate": 8.939866700243901e-05, + "loss": 0.1334, + "step": 5070 + }, + { + "epoch": 1.64, + "learning_rate": 8.922468928808497e-05, + "loss": 0.1311, + "step": 5075 + }, + { + "epoch": 1.65, + "learning_rate": 8.905074456560574e-05, + "loss": 0.1439, + "step": 5080 + }, + { + "epoch": 1.65, + "learning_rate": 8.887683336758563e-05, + "loss": 0.1424, + "step": 5085 + }, + { + "epoch": 1.65, + "learning_rate": 8.870295622650651e-05, + "loss": 0.1476, + "step": 5090 + }, + { + "epoch": 1.65, + "learning_rate": 8.852911367474577e-05, + "loss": 0.1436, + "step": 5095 + }, + { + "epoch": 1.65, + "learning_rate": 8.835530624457503e-05, + "loss": 0.1442, + "step": 5100 + }, + { + "epoch": 1.65, + "learning_rate": 8.81815344681584e-05, + "loss": 0.1467, + "step": 5105 + }, + { + "epoch": 1.66, + "learning_rate": 8.800779887755066e-05, + "loss": 0.1409, + "step": 5110 + }, + { + "epoch": 1.66, + "learning_rate": 8.783410000469601e-05, + "loss": 0.1487, + "step": 5115 + }, + { + "epoch": 1.66, + "learning_rate": 8.7660438381426e-05, + "loss": 0.1463, + "step": 5120 + }, + { + "epoch": 1.66, + "learning_rate": 8.748681453945833e-05, + "loss": 0.1398, + "step": 5125 + }, + { + "epoch": 1.66, + "learning_rate": 8.73132290103949e-05, + "loss": 0.1358, + "step": 5130 + }, + { + "epoch": 1.66, + "learning_rate": 8.713968232572039e-05, + "loss": 0.143, + "step": 5135 + }, + { + "epoch": 1.67, + "learning_rate": 8.69661750168004e-05, + "loss": 0.137, + "step": 5140 + }, + { + "epoch": 1.67, + "learning_rate": 8.679270761488019e-05, + "loss": 0.1393, + "step": 5145 + }, + { + "epoch": 1.67, + "learning_rate": 8.661928065108259e-05, + "loss": 0.1472, + "step": 5150 + }, + { + "epoch": 1.67, + "learning_rate": 8.644589465640684e-05, + "loss": 0.1389, + "step": 5155 + }, + { + "epoch": 1.67, + "learning_rate": 8.627255016172652e-05, + "loss": 0.1487, + "step": 5160 + }, + { + "epoch": 1.67, + "learning_rate": 8.609924769778835e-05, + "loss": 0.1403, + "step": 5165 + }, + { + "epoch": 1.68, + "learning_rate": 8.59259877952103e-05, + "loss": 0.1351, + "step": 5170 + }, + { + "epoch": 1.68, + "learning_rate": 8.575277098447987e-05, + "loss": 0.1406, + "step": 5175 + }, + { + "epoch": 1.68, + "learning_rate": 8.557959779595286e-05, + "loss": 0.1378, + "step": 5180 + }, + { + "epoch": 1.68, + "learning_rate": 8.540646875985134e-05, + "loss": 0.1352, + "step": 5185 + }, + { + "epoch": 1.68, + "learning_rate": 8.523338440626232e-05, + "loss": 0.1406, + "step": 5190 + }, + { + "epoch": 1.68, + "learning_rate": 8.50603452651358e-05, + "loss": 0.1432, + "step": 5195 + }, + { + "epoch": 1.69, + "learning_rate": 8.488735186628364e-05, + "loss": 0.1433, + "step": 5200 + }, + { + "epoch": 1.69, + "learning_rate": 8.471440473937732e-05, + "loss": 0.1368, + "step": 5205 + }, + { + "epoch": 1.69, + "learning_rate": 8.454150441394697e-05, + "loss": 0.136, + "step": 5210 + }, + { + "epoch": 1.69, + "learning_rate": 8.436865141937912e-05, + "loss": 0.1502, + "step": 5215 + }, + { + "epoch": 1.69, + "learning_rate": 8.419584628491562e-05, + "loss": 0.1395, + "step": 5220 + }, + { + "epoch": 1.69, + "learning_rate": 8.402308953965165e-05, + "loss": 0.1294, + "step": 5225 + }, + { + "epoch": 1.69, + "learning_rate": 8.385038171253424e-05, + "loss": 0.1438, + "step": 5230 + }, + { + "epoch": 1.7, + "learning_rate": 8.367772333236073e-05, + "loss": 0.1371, + "step": 5235 + }, + { + "epoch": 1.7, + "learning_rate": 8.350511492777698e-05, + "loss": 0.138, + "step": 5240 + }, + { + "epoch": 1.7, + "learning_rate": 8.333255702727588e-05, + "loss": 0.1389, + "step": 5245 + }, + { + "epoch": 1.7, + "learning_rate": 8.316005015919559e-05, + "loss": 0.1384, + "step": 5250 + }, + { + "epoch": 1.7, + "learning_rate": 8.298759485171819e-05, + "loss": 0.1384, + "step": 5255 + }, + { + "epoch": 1.7, + "learning_rate": 8.281519163286772e-05, + "loss": 0.1307, + "step": 5260 + }, + { + "epoch": 1.71, + "learning_rate": 8.264284103050888e-05, + "loss": 0.1413, + "step": 5265 + }, + { + "epoch": 1.71, + "learning_rate": 8.247054357234511e-05, + "loss": 0.1322, + "step": 5270 + }, + { + "epoch": 1.71, + "learning_rate": 8.229829978591729e-05, + "loss": 0.1433, + "step": 5275 + }, + { + "epoch": 1.71, + "learning_rate": 8.212611019860188e-05, + "loss": 0.1384, + "step": 5280 + }, + { + "epoch": 1.71, + "learning_rate": 8.195397533760943e-05, + "loss": 0.1396, + "step": 5285 + }, + { + "epoch": 1.71, + "learning_rate": 8.178189572998284e-05, + "loss": 0.1403, + "step": 5290 + }, + { + "epoch": 1.72, + "learning_rate": 8.160987190259605e-05, + "loss": 0.1429, + "step": 5295 + }, + { + "epoch": 1.72, + "learning_rate": 8.14379043821519e-05, + "loss": 0.1384, + "step": 5300 + }, + { + "epoch": 1.72, + "learning_rate": 8.12659936951811e-05, + "loss": 0.1422, + "step": 5305 + }, + { + "epoch": 1.72, + "learning_rate": 8.109414036804026e-05, + "loss": 0.1355, + "step": 5310 + }, + { + "epoch": 1.72, + "learning_rate": 8.092234492691027e-05, + "loss": 0.1357, + "step": 5315 + }, + { + "epoch": 1.72, + "learning_rate": 8.075060789779494e-05, + "loss": 0.1388, + "step": 5320 + }, + { + "epoch": 1.73, + "learning_rate": 8.057892980651909e-05, + "loss": 0.1349, + "step": 5325 + }, + { + "epoch": 1.73, + "learning_rate": 8.040731117872721e-05, + "loss": 0.1328, + "step": 5330 + }, + { + "epoch": 1.73, + "learning_rate": 8.023575253988165e-05, + "loss": 0.1301, + "step": 5335 + }, + { + "epoch": 1.73, + "learning_rate": 8.00642544152611e-05, + "loss": 0.1376, + "step": 5340 + }, + { + "epoch": 1.73, + "learning_rate": 7.989281732995895e-05, + "loss": 0.1389, + "step": 5345 + }, + { + "epoch": 1.73, + "learning_rate": 7.972144180888177e-05, + "loss": 0.1337, + "step": 5350 + }, + { + "epoch": 1.74, + "learning_rate": 7.955012837674753e-05, + "loss": 0.141, + "step": 5355 + }, + { + "epoch": 1.74, + "learning_rate": 7.93788775580842e-05, + "loss": 0.1456, + "step": 5360 + }, + { + "epoch": 1.74, + "learning_rate": 7.920768987722791e-05, + "loss": 0.1459, + "step": 5365 + }, + { + "epoch": 1.74, + "learning_rate": 7.903656585832162e-05, + "loss": 0.1332, + "step": 5370 + }, + { + "epoch": 1.74, + "learning_rate": 7.886550602531331e-05, + "loss": 0.1445, + "step": 5375 + }, + { + "epoch": 1.74, + "learning_rate": 7.869451090195437e-05, + "loss": 0.1349, + "step": 5380 + }, + { + "epoch": 1.75, + "learning_rate": 7.852358101179817e-05, + "loss": 0.1327, + "step": 5385 + }, + { + "epoch": 1.75, + "learning_rate": 7.835271687819827e-05, + "loss": 0.1305, + "step": 5390 + }, + { + "epoch": 1.75, + "learning_rate": 7.818191902430696e-05, + "loss": 0.1446, + "step": 5395 + }, + { + "epoch": 1.75, + "learning_rate": 7.801118797307349e-05, + "loss": 0.139, + "step": 5400 + }, + { + "epoch": 1.75, + "learning_rate": 7.784052424724276e-05, + "loss": 0.1432, + "step": 5405 + }, + { + "epoch": 1.75, + "learning_rate": 7.766992836935327e-05, + "loss": 0.1439, + "step": 5410 + }, + { + "epoch": 1.75, + "learning_rate": 7.749940086173607e-05, + "loss": 0.1415, + "step": 5415 + }, + { + "epoch": 1.76, + "learning_rate": 7.732894224651261e-05, + "loss": 0.1404, + "step": 5420 + }, + { + "epoch": 1.76, + "learning_rate": 7.715855304559362e-05, + "loss": 0.1415, + "step": 5425 + }, + { + "epoch": 1.76, + "learning_rate": 7.698823378067714e-05, + "loss": 0.142, + "step": 5430 + }, + { + "epoch": 1.76, + "learning_rate": 7.681798497324716e-05, + "loss": 0.1428, + "step": 5435 + }, + { + "epoch": 1.76, + "learning_rate": 7.664780714457198e-05, + "loss": 0.1405, + "step": 5440 + }, + { + "epoch": 1.76, + "learning_rate": 7.647770081570246e-05, + "loss": 0.1355, + "step": 5445 + }, + { + "epoch": 1.77, + "learning_rate": 7.630766650747066e-05, + "loss": 0.1371, + "step": 5450 + }, + { + "epoch": 1.77, + "learning_rate": 7.613770474048802e-05, + "loss": 0.1388, + "step": 5455 + }, + { + "epoch": 1.77, + "learning_rate": 7.596781603514398e-05, + "loss": 0.1306, + "step": 5460 + }, + { + "epoch": 1.77, + "learning_rate": 7.579800091160417e-05, + "loss": 0.1351, + "step": 5465 + }, + { + "epoch": 1.77, + "learning_rate": 7.562825988980904e-05, + "loss": 0.1322, + "step": 5470 + }, + { + "epoch": 1.77, + "learning_rate": 7.545859348947204e-05, + "loss": 0.1389, + "step": 5475 + }, + { + "epoch": 1.78, + "learning_rate": 7.528900223007824e-05, + "loss": 0.1432, + "step": 5480 + }, + { + "epoch": 1.78, + "learning_rate": 7.511948663088261e-05, + "loss": 0.1359, + "step": 5485 + }, + { + "epoch": 1.78, + "learning_rate": 7.495004721090843e-05, + "loss": 0.1411, + "step": 5490 + }, + { + "epoch": 1.78, + "learning_rate": 7.478068448894577e-05, + "loss": 0.1373, + "step": 5495 + }, + { + "epoch": 1.78, + "learning_rate": 7.461139898354986e-05, + "loss": 0.1406, + "step": 5500 + }, + { + "epoch": 1.78, + "learning_rate": 7.444219121303957e-05, + "loss": 0.1388, + "step": 5505 + }, + { + "epoch": 1.79, + "learning_rate": 7.42730616954956e-05, + "loss": 0.1344, + "step": 5510 + }, + { + "epoch": 1.79, + "learning_rate": 7.410401094875921e-05, + "loss": 0.1416, + "step": 5515 + }, + { + "epoch": 1.79, + "learning_rate": 7.393503949043039e-05, + "loss": 0.1417, + "step": 5520 + }, + { + "epoch": 1.79, + "learning_rate": 7.376614783786646e-05, + "loss": 0.1425, + "step": 5525 + }, + { + "epoch": 1.79, + "learning_rate": 7.359733650818024e-05, + "loss": 0.1326, + "step": 5530 + }, + { + "epoch": 1.79, + "learning_rate": 7.342860601823875e-05, + "loss": 0.1449, + "step": 5535 + }, + { + "epoch": 1.8, + "learning_rate": 7.325995688466146e-05, + "loss": 0.1339, + "step": 5540 + }, + { + "epoch": 1.8, + "learning_rate": 7.309138962381872e-05, + "loss": 0.1341, + "step": 5545 + }, + { + "epoch": 1.8, + "learning_rate": 7.29229047518302e-05, + "loss": 0.1324, + "step": 5550 + }, + { + "epoch": 1.8, + "learning_rate": 7.275450278456335e-05, + "loss": 0.129, + "step": 5555 + }, + { + "epoch": 1.8, + "learning_rate": 7.258618423763171e-05, + "loss": 0.1388, + "step": 5560 + }, + { + "epoch": 1.8, + "learning_rate": 7.24179496263935e-05, + "loss": 0.1406, + "step": 5565 + }, + { + "epoch": 1.8, + "learning_rate": 7.224979946594992e-05, + "loss": 0.138, + "step": 5570 + }, + { + "epoch": 1.81, + "learning_rate": 7.20817342711435e-05, + "loss": 0.1412, + "step": 5575 + }, + { + "epoch": 1.81, + "learning_rate": 7.191375455655676e-05, + "loss": 0.1377, + "step": 5580 + }, + { + "epoch": 1.81, + "learning_rate": 7.174586083651038e-05, + "loss": 0.1387, + "step": 5585 + }, + { + "epoch": 1.81, + "learning_rate": 7.157805362506184e-05, + "loss": 0.1352, + "step": 5590 + }, + { + "epoch": 1.81, + "learning_rate": 7.14103334360037e-05, + "loss": 0.1457, + "step": 5595 + }, + { + "epoch": 1.81, + "learning_rate": 7.124270078286203e-05, + "loss": 0.1359, + "step": 5600 + }, + { + "epoch": 1.82, + "learning_rate": 7.107515617889498e-05, + "loss": 0.138, + "step": 5605 + }, + { + "epoch": 1.82, + "learning_rate": 7.090770013709105e-05, + "loss": 0.1402, + "step": 5610 + }, + { + "epoch": 1.82, + "learning_rate": 7.074033317016755e-05, + "loss": 0.133, + "step": 5615 + }, + { + "epoch": 1.82, + "learning_rate": 7.057305579056919e-05, + "loss": 0.1419, + "step": 5620 + }, + { + "epoch": 1.82, + "learning_rate": 7.040586851046618e-05, + "loss": 0.1282, + "step": 5625 + }, + { + "epoch": 1.82, + "learning_rate": 7.023877184175307e-05, + "loss": 0.1393, + "step": 5630 + }, + { + "epoch": 1.83, + "learning_rate": 7.007176629604681e-05, + "loss": 0.1273, + "step": 5635 + }, + { + "epoch": 1.83, + "learning_rate": 6.990485238468545e-05, + "loss": 0.1417, + "step": 5640 + }, + { + "epoch": 1.83, + "learning_rate": 6.973803061872646e-05, + "loss": 0.1376, + "step": 5645 + }, + { + "epoch": 1.83, + "learning_rate": 6.957130150894516e-05, + "loss": 0.1421, + "step": 5650 + }, + { + "epoch": 1.83, + "learning_rate": 6.940466556583316e-05, + "loss": 0.1311, + "step": 5655 + }, + { + "epoch": 1.83, + "learning_rate": 6.923812329959683e-05, + "loss": 0.1376, + "step": 5660 + }, + { + "epoch": 1.84, + "learning_rate": 6.907167522015577e-05, + "loss": 0.1375, + "step": 5665 + }, + { + "epoch": 1.84, + "learning_rate": 6.890532183714105e-05, + "loss": 0.1304, + "step": 5670 + }, + { + "epoch": 1.84, + "learning_rate": 6.8739063659894e-05, + "loss": 0.1358, + "step": 5675 + }, + { + "epoch": 1.84, + "learning_rate": 6.857290119746425e-05, + "loss": 0.1418, + "step": 5680 + }, + { + "epoch": 1.84, + "learning_rate": 6.840683495860857e-05, + "loss": 0.1367, + "step": 5685 + }, + { + "epoch": 1.84, + "learning_rate": 6.82408654517889e-05, + "loss": 0.1373, + "step": 5690 + }, + { + "epoch": 1.85, + "learning_rate": 6.80749931851712e-05, + "loss": 0.1408, + "step": 5695 + }, + { + "epoch": 1.85, + "learning_rate": 6.790921866662355e-05, + "loss": 0.1326, + "step": 5700 + }, + { + "epoch": 1.85, + "learning_rate": 6.774354240371487e-05, + "loss": 0.137, + "step": 5705 + }, + { + "epoch": 1.85, + "learning_rate": 6.757796490371314e-05, + "loss": 0.1509, + "step": 5710 + }, + { + "epoch": 1.85, + "learning_rate": 6.741248667358397e-05, + "loss": 0.1317, + "step": 5715 + }, + { + "epoch": 1.85, + "learning_rate": 6.724710821998912e-05, + "loss": 0.1395, + "step": 5720 + }, + { + "epoch": 1.86, + "learning_rate": 6.708183004928468e-05, + "loss": 0.1359, + "step": 5725 + }, + { + "epoch": 1.86, + "learning_rate": 6.691665266751985e-05, + "loss": 0.1448, + "step": 5730 + }, + { + "epoch": 1.86, + "learning_rate": 6.675157658043513e-05, + "loss": 0.1365, + "step": 5735 + }, + { + "epoch": 1.86, + "learning_rate": 6.658660229346097e-05, + "loss": 0.1341, + "step": 5740 + }, + { + "epoch": 1.86, + "learning_rate": 6.642173031171603e-05, + "loss": 0.137, + "step": 5745 + }, + { + "epoch": 1.86, + "learning_rate": 6.62569611400058e-05, + "loss": 0.1376, + "step": 5750 + }, + { + "epoch": 1.86, + "learning_rate": 6.609229528282094e-05, + "loss": 0.1297, + "step": 5755 + }, + { + "epoch": 1.87, + "learning_rate": 6.592773324433588e-05, + "loss": 0.1344, + "step": 5760 + }, + { + "epoch": 1.87, + "learning_rate": 6.5763275528407e-05, + "loss": 0.1359, + "step": 5765 + }, + { + "epoch": 1.87, + "learning_rate": 6.559892263857144e-05, + "loss": 0.1302, + "step": 5770 + }, + { + "epoch": 1.87, + "learning_rate": 6.54346750780453e-05, + "loss": 0.132, + "step": 5775 + }, + { + "epoch": 1.87, + "learning_rate": 6.527053334972218e-05, + "loss": 0.1349, + "step": 5780 + }, + { + "epoch": 1.87, + "learning_rate": 6.510649795617168e-05, + "loss": 0.1406, + "step": 5785 + }, + { + "epoch": 1.88, + "learning_rate": 6.494256939963775e-05, + "loss": 0.1345, + "step": 5790 + }, + { + "epoch": 1.88, + "learning_rate": 6.477874818203736e-05, + "loss": 0.1392, + "step": 5795 + }, + { + "epoch": 1.88, + "learning_rate": 6.461503480495864e-05, + "loss": 0.1359, + "step": 5800 + }, + { + "epoch": 1.88, + "learning_rate": 6.445142976965972e-05, + "loss": 0.1284, + "step": 5805 + }, + { + "epoch": 1.88, + "learning_rate": 6.428793357706689e-05, + "loss": 0.1343, + "step": 5810 + }, + { + "epoch": 1.88, + "learning_rate": 6.412454672777328e-05, + "loss": 0.1387, + "step": 5815 + }, + { + "epoch": 1.89, + "learning_rate": 6.396126972203708e-05, + "loss": 0.1334, + "step": 5820 + }, + { + "epoch": 1.89, + "learning_rate": 6.379810305978038e-05, + "loss": 0.1329, + "step": 5825 + }, + { + "epoch": 1.89, + "learning_rate": 6.363504724058717e-05, + "loss": 0.1378, + "step": 5830 + }, + { + "epoch": 1.89, + "learning_rate": 6.347210276370224e-05, + "loss": 0.1358, + "step": 5835 + }, + { + "epoch": 1.89, + "learning_rate": 6.330927012802947e-05, + "loss": 0.1412, + "step": 5840 + }, + { + "epoch": 1.89, + "learning_rate": 6.314654983213018e-05, + "loss": 0.1308, + "step": 5845 + }, + { + "epoch": 1.9, + "learning_rate": 6.298394237422186e-05, + "loss": 0.131, + "step": 5850 + }, + { + "epoch": 1.9, + "learning_rate": 6.282144825217639e-05, + "loss": 0.1341, + "step": 5855 + }, + { + "epoch": 1.9, + "learning_rate": 6.265906796351875e-05, + "loss": 0.1435, + "step": 5860 + }, + { + "epoch": 1.9, + "learning_rate": 6.24968020054253e-05, + "loss": 0.1417, + "step": 5865 + }, + { + "epoch": 1.9, + "learning_rate": 6.233465087472244e-05, + "loss": 0.1376, + "step": 5870 + }, + { + "epoch": 1.9, + "learning_rate": 6.217261506788485e-05, + "loss": 0.1268, + "step": 5875 + }, + { + "epoch": 1.91, + "learning_rate": 6.201069508103426e-05, + "loss": 0.1389, + "step": 5880 + }, + { + "epoch": 1.91, + "learning_rate": 6.184889140993765e-05, + "loss": 0.14, + "step": 5885 + }, + { + "epoch": 1.91, + "learning_rate": 6.168720455000601e-05, + "loss": 0.1394, + "step": 5890 + }, + { + "epoch": 1.91, + "learning_rate": 6.152563499629249e-05, + "loss": 0.1413, + "step": 5895 + }, + { + "epoch": 1.91, + "learning_rate": 6.136418324349125e-05, + "loss": 0.1354, + "step": 5900 + }, + { + "epoch": 1.91, + "learning_rate": 6.120284978593569e-05, + "loss": 0.1335, + "step": 5905 + }, + { + "epoch": 1.92, + "learning_rate": 6.104163511759696e-05, + "loss": 0.1381, + "step": 5910 + }, + { + "epoch": 1.92, + "learning_rate": 6.088053973208262e-05, + "loss": 0.1313, + "step": 5915 + }, + { + "epoch": 1.92, + "learning_rate": 6.071956412263491e-05, + "loss": 0.1314, + "step": 5920 + }, + { + "epoch": 1.92, + "learning_rate": 6.0558708782129416e-05, + "loss": 0.1432, + "step": 5925 + }, + { + "epoch": 1.92, + "learning_rate": 6.039797420307338e-05, + "loss": 0.1398, + "step": 5930 + }, + { + "epoch": 1.92, + "learning_rate": 6.023736087760443e-05, + "loss": 0.1334, + "step": 5935 + }, + { + "epoch": 1.92, + "learning_rate": 6.00768692974888e-05, + "loss": 0.1393, + "step": 5940 + }, + { + "epoch": 1.93, + "learning_rate": 5.991649995412011e-05, + "loss": 0.1319, + "step": 5945 + }, + { + "epoch": 1.93, + "learning_rate": 5.975625333851755e-05, + "loss": 0.1304, + "step": 5950 + }, + { + "epoch": 1.93, + "learning_rate": 5.9596129941324705e-05, + "loss": 0.1332, + "step": 5955 + }, + { + "epoch": 1.93, + "learning_rate": 5.9436130252807785e-05, + "loss": 0.1344, + "step": 5960 + }, + { + "epoch": 1.93, + "learning_rate": 5.927625476285426e-05, + "loss": 0.1329, + "step": 5965 + }, + { + "epoch": 1.93, + "learning_rate": 5.91165039609713e-05, + "loss": 0.1359, + "step": 5970 + }, + { + "epoch": 1.94, + "learning_rate": 5.895687833628437e-05, + "loss": 0.1311, + "step": 5975 + }, + { + "epoch": 1.94, + "learning_rate": 5.879737837753565e-05, + "loss": 0.1324, + "step": 5980 + }, + { + "epoch": 1.94, + "learning_rate": 5.863800457308246e-05, + "loss": 0.1286, + "step": 5985 + }, + { + "epoch": 1.94, + "learning_rate": 5.847875741089604e-05, + "loss": 0.134, + "step": 5990 + }, + { + "epoch": 1.94, + "learning_rate": 5.8319637378559635e-05, + "loss": 0.1312, + "step": 5995 + }, + { + "epoch": 1.94, + "learning_rate": 5.816064496326752e-05, + "loss": 0.1277, + "step": 6000 + }, + { + "epoch": 1.95, + "learning_rate": 5.800178065182296e-05, + "loss": 0.1377, + "step": 6005 + }, + { + "epoch": 1.95, + "learning_rate": 5.784304493063725e-05, + "loss": 0.138, + "step": 6010 + }, + { + "epoch": 1.95, + "learning_rate": 5.768443828572772e-05, + "loss": 0.1283, + "step": 6015 + }, + { + "epoch": 1.95, + "learning_rate": 5.7525961202716714e-05, + "loss": 0.1307, + "step": 6020 + }, + { + "epoch": 1.95, + "learning_rate": 5.7367614166829706e-05, + "loss": 0.136, + "step": 6025 + }, + { + "epoch": 1.95, + "learning_rate": 5.720939766289415e-05, + "loss": 0.137, + "step": 6030 + }, + { + "epoch": 1.96, + "learning_rate": 5.7051312175337656e-05, + "loss": 0.1341, + "step": 6035 + }, + { + "epoch": 1.96, + "learning_rate": 5.689335818818684e-05, + "loss": 0.1384, + "step": 6040 + }, + { + "epoch": 1.96, + "learning_rate": 5.673553618506562e-05, + "loss": 0.1378, + "step": 6045 + }, + { + "epoch": 1.96, + "learning_rate": 5.65778466491939e-05, + "loss": 0.1348, + "step": 6050 + }, + { + "epoch": 1.96, + "learning_rate": 5.6420290063385805e-05, + "loss": 0.1344, + "step": 6055 + }, + { + "epoch": 1.96, + "learning_rate": 5.626286691004854e-05, + "loss": 0.1354, + "step": 6060 + }, + { + "epoch": 1.97, + "learning_rate": 5.61055776711808e-05, + "loss": 0.1345, + "step": 6065 + }, + { + "epoch": 1.97, + "learning_rate": 5.594842282837109e-05, + "loss": 0.1497, + "step": 6070 + }, + { + "epoch": 1.97, + "learning_rate": 5.5791402862796604e-05, + "loss": 0.1257, + "step": 6075 + }, + { + "epoch": 1.97, + "learning_rate": 5.5634518255221415e-05, + "loss": 0.1384, + "step": 6080 + }, + { + "epoch": 1.97, + "learning_rate": 5.54777694859953e-05, + "loss": 0.1344, + "step": 6085 + }, + { + "epoch": 1.97, + "learning_rate": 5.532115703505198e-05, + "loss": 0.1321, + "step": 6090 + }, + { + "epoch": 1.98, + "learning_rate": 5.516468138190796e-05, + "loss": 0.1389, + "step": 6095 + }, + { + "epoch": 1.98, + "learning_rate": 5.500834300566069e-05, + "loss": 0.1357, + "step": 6100 + }, + { + "epoch": 1.98, + "learning_rate": 5.4852142384987504e-05, + "loss": 0.1333, + "step": 6105 + }, + { + "epoch": 1.98, + "learning_rate": 5.46960799981439e-05, + "loss": 0.1296, + "step": 6110 + }, + { + "epoch": 1.98, + "learning_rate": 5.454015632296203e-05, + "loss": 0.1352, + "step": 6115 + }, + { + "epoch": 1.98, + "learning_rate": 5.438437183684948e-05, + "loss": 0.132, + "step": 6120 + }, + { + "epoch": 1.98, + "learning_rate": 5.422872701678754e-05, + "loss": 0.1382, + "step": 6125 + }, + { + "epoch": 1.99, + "learning_rate": 5.407322233932999e-05, + "loss": 0.1363, + "step": 6130 + }, + { + "epoch": 1.99, + "learning_rate": 5.391785828060139e-05, + "loss": 0.138, + "step": 6135 + }, + { + "epoch": 1.99, + "learning_rate": 5.376263531629591e-05, + "loss": 0.1357, + "step": 6140 + }, + { + "epoch": 1.99, + "learning_rate": 5.360755392167555e-05, + "loss": 0.1376, + "step": 6145 + }, + { + "epoch": 1.99, + "learning_rate": 5.345261457156897e-05, + "loss": 0.1372, + "step": 6150 + }, + { + "epoch": 1.99, + "learning_rate": 5.329781774036988e-05, + "loss": 0.1362, + "step": 6155 + }, + { + "epoch": 2.0, + "learning_rate": 5.3143163902035664e-05, + "loss": 0.1321, + "step": 6160 + }, + { + "epoch": 2.0, + "learning_rate": 5.2988653530085796e-05, + "loss": 0.1423, + "step": 6165 + }, + { + "epoch": 2.0, + "learning_rate": 5.2834287097600545e-05, + "loss": 0.1315, + "step": 6170 + }, + { + "epoch": 2.0, + "learning_rate": 5.268006507721952e-05, + "loss": 0.1351, + "step": 6175 + }, + { + "epoch": 2.0, + "learning_rate": 5.252598794114002e-05, + "loss": 0.13, + "step": 6180 + }, + { + "epoch": 2.0, + "learning_rate": 5.2372056161115925e-05, + "loss": 0.1314, + "step": 6185 + }, + { + "epoch": 2.01, + "learning_rate": 5.2218270208455864e-05, + "loss": 0.1272, + "step": 6190 + }, + { + "epoch": 2.01, + "learning_rate": 5.206463055402218e-05, + "loss": 0.1357, + "step": 6195 + }, + { + "epoch": 2.01, + "learning_rate": 5.191113766822905e-05, + "loss": 0.132, + "step": 6200 + }, + { + "epoch": 2.01, + "learning_rate": 5.175779202104152e-05, + "loss": 0.1382, + "step": 6205 + }, + { + "epoch": 2.01, + "learning_rate": 5.160459408197358e-05, + "loss": 0.1361, + "step": 6210 + }, + { + "epoch": 2.01, + "learning_rate": 5.1451544320087185e-05, + "loss": 0.1371, + "step": 6215 + }, + { + "epoch": 2.02, + "learning_rate": 5.1298643203990424e-05, + "loss": 0.1347, + "step": 6220 + }, + { + "epoch": 2.02, + "learning_rate": 5.1145891201836414e-05, + "loss": 0.1325, + "step": 6225 + }, + { + "epoch": 2.02, + "learning_rate": 5.099328878132156e-05, + "loss": 0.1311, + "step": 6230 + }, + { + "epoch": 2.02, + "learning_rate": 5.084083640968442e-05, + "loss": 0.1396, + "step": 6235 + }, + { + "epoch": 2.02, + "learning_rate": 5.0688534553703995e-05, + "loss": 0.1322, + "step": 6240 + }, + { + "epoch": 2.02, + "learning_rate": 5.053638367969855e-05, + "loss": 0.1404, + "step": 6245 + }, + { + "epoch": 2.03, + "learning_rate": 5.038438425352405e-05, + "loss": 0.1383, + "step": 6250 + }, + { + "epoch": 2.03, + "learning_rate": 5.023253674057275e-05, + "loss": 0.1332, + "step": 6255 + }, + { + "epoch": 2.03, + "learning_rate": 5.008084160577171e-05, + "loss": 0.1369, + "step": 6260 + }, + { + "epoch": 2.03, + "learning_rate": 4.992929931358151e-05, + "loss": 0.1399, + "step": 6265 + }, + { + "epoch": 2.03, + "learning_rate": 4.9777910327994804e-05, + "loss": 0.1348, + "step": 6270 + }, + { + "epoch": 2.03, + "learning_rate": 4.962667511253469e-05, + "loss": 0.1333, + "step": 6275 + }, + { + "epoch": 2.04, + "learning_rate": 4.9475594130253645e-05, + "loss": 0.133, + "step": 6280 + }, + { + "epoch": 2.04, + "learning_rate": 4.932466784373172e-05, + "loss": 0.1312, + "step": 6285 + }, + { + "epoch": 2.04, + "learning_rate": 4.917389671507551e-05, + "loss": 0.1302, + "step": 6290 + }, + { + "epoch": 2.04, + "learning_rate": 4.902328120591637e-05, + "loss": 0.1337, + "step": 6295 + }, + { + "epoch": 2.04, + "learning_rate": 4.887282177740935e-05, + "loss": 0.1316, + "step": 6300 + }, + { + "epoch": 2.04, + "learning_rate": 4.872251889023144e-05, + "loss": 0.1324, + "step": 6305 + }, + { + "epoch": 2.04, + "learning_rate": 4.8572373004580465e-05, + "loss": 0.1344, + "step": 6310 + }, + { + "epoch": 2.05, + "learning_rate": 4.8422384580173505e-05, + "loss": 0.1281, + "step": 6315 + }, + { + "epoch": 2.05, + "learning_rate": 4.827255407624547e-05, + "loss": 0.1349, + "step": 6320 + }, + { + "epoch": 2.05, + "learning_rate": 4.812288195154785e-05, + "loss": 0.1305, + "step": 6325 + }, + { + "epoch": 2.05, + "learning_rate": 4.7973368664347086e-05, + "loss": 0.1273, + "step": 6330 + }, + { + "epoch": 2.05, + "learning_rate": 4.7824014672423414e-05, + "loss": 0.1309, + "step": 6335 + }, + { + "epoch": 2.05, + "learning_rate": 4.7674820433069224e-05, + "loss": 0.1333, + "step": 6340 + }, + { + "epoch": 2.06, + "learning_rate": 4.752578640308789e-05, + "loss": 0.1411, + "step": 6345 + }, + { + "epoch": 2.06, + "learning_rate": 4.7376913038792136e-05, + "loss": 0.1309, + "step": 6350 + }, + { + "epoch": 2.06, + "learning_rate": 4.7228200796002854e-05, + "loss": 0.1262, + "step": 6355 + }, + { + "epoch": 2.06, + "learning_rate": 4.707965013004758e-05, + "loss": 0.1319, + "step": 6360 + }, + { + "epoch": 2.06, + "learning_rate": 4.6931261495759184e-05, + "loss": 0.1297, + "step": 6365 + }, + { + "epoch": 2.06, + "learning_rate": 4.678303534747428e-05, + "loss": 0.1313, + "step": 6370 + }, + { + "epoch": 2.07, + "learning_rate": 4.663497213903213e-05, + "loss": 0.1244, + "step": 6375 + }, + { + "epoch": 2.07, + "learning_rate": 4.6487072323773093e-05, + "loss": 0.1292, + "step": 6380 + }, + { + "epoch": 2.07, + "learning_rate": 4.633933635453716e-05, + "loss": 0.1327, + "step": 6385 + }, + { + "epoch": 2.07, + "learning_rate": 4.6191764683662744e-05, + "loss": 0.1335, + "step": 6390 + }, + { + "epoch": 2.07, + "learning_rate": 4.6044357762985125e-05, + "loss": 0.1342, + "step": 6395 + }, + { + "epoch": 2.07, + "learning_rate": 4.589711604383528e-05, + "loss": 0.1348, + "step": 6400 + }, + { + "epoch": 2.08, + "learning_rate": 4.575003997703821e-05, + "loss": 0.1344, + "step": 6405 + }, + { + "epoch": 2.08, + "learning_rate": 4.560313001291188e-05, + "loss": 0.125, + "step": 6410 + }, + { + "epoch": 2.08, + "learning_rate": 4.5456386601265534e-05, + "loss": 0.1425, + "step": 6415 + }, + { + "epoch": 2.08, + "learning_rate": 4.530981019139862e-05, + "loss": 0.1286, + "step": 6420 + }, + { + "epoch": 2.08, + "learning_rate": 4.516340123209908e-05, + "loss": 0.1286, + "step": 6425 + }, + { + "epoch": 2.08, + "learning_rate": 4.5017160171642333e-05, + "loss": 0.1347, + "step": 6430 + }, + { + "epoch": 2.09, + "learning_rate": 4.4871087457789584e-05, + "loss": 0.1301, + "step": 6435 + }, + { + "epoch": 2.09, + "learning_rate": 4.4725183537786683e-05, + "loss": 0.1314, + "step": 6440 + }, + { + "epoch": 2.09, + "learning_rate": 4.457944885836267e-05, + "loss": 0.1341, + "step": 6445 + }, + { + "epoch": 2.09, + "learning_rate": 4.44338838657283e-05, + "loss": 0.1347, + "step": 6450 + }, + { + "epoch": 2.09, + "learning_rate": 4.428848900557492e-05, + "loss": 0.1308, + "step": 6455 + }, + { + "epoch": 2.09, + "learning_rate": 4.414326472307283e-05, + "loss": 0.1395, + "step": 6460 + }, + { + "epoch": 2.09, + "learning_rate": 4.3998211462870165e-05, + "loss": 0.1369, + "step": 6465 + }, + { + "epoch": 2.1, + "learning_rate": 4.385332966909136e-05, + "loss": 0.1334, + "step": 6470 + }, + { + "epoch": 2.1, + "learning_rate": 4.3708619785335923e-05, + "loss": 0.1296, + "step": 6475 + }, + { + "epoch": 2.1, + "learning_rate": 4.356408225467688e-05, + "loss": 0.1301, + "step": 6480 + }, + { + "epoch": 2.1, + "learning_rate": 4.3419717519659694e-05, + "loss": 0.125, + "step": 6485 + }, + { + "epoch": 2.1, + "learning_rate": 4.327552602230063e-05, + "loss": 0.1251, + "step": 6490 + }, + { + "epoch": 2.1, + "learning_rate": 4.313150820408566e-05, + "loss": 0.137, + "step": 6495 + }, + { + "epoch": 2.11, + "learning_rate": 4.298766450596885e-05, + "loss": 0.126, + "step": 6500 + }, + { + "epoch": 2.11, + "learning_rate": 4.284399536837124e-05, + "loss": 0.1331, + "step": 6505 + }, + { + "epoch": 2.11, + "learning_rate": 4.2700501231179427e-05, + "loss": 0.1322, + "step": 6510 + }, + { + "epoch": 2.11, + "learning_rate": 4.255718253374405e-05, + "loss": 0.1292, + "step": 6515 + }, + { + "epoch": 2.11, + "learning_rate": 4.241403971487876e-05, + "loss": 0.1354, + "step": 6520 + }, + { + "epoch": 2.11, + "learning_rate": 4.227107321285855e-05, + "loss": 0.1264, + "step": 6525 + }, + { + "epoch": 2.12, + "learning_rate": 4.212828346541868e-05, + "loss": 0.1336, + "step": 6530 + }, + { + "epoch": 2.12, + "learning_rate": 4.198567090975314e-05, + "loss": 0.1218, + "step": 6535 + }, + { + "epoch": 2.12, + "learning_rate": 4.1843235982513463e-05, + "loss": 0.1244, + "step": 6540 + }, + { + "epoch": 2.12, + "learning_rate": 4.1700979119807235e-05, + "loss": 0.1354, + "step": 6545 + }, + { + "epoch": 2.12, + "learning_rate": 4.1558900757196964e-05, + "loss": 0.1231, + "step": 6550 + }, + { + "epoch": 2.12, + "learning_rate": 4.141700132969848e-05, + "loss": 0.133, + "step": 6555 + }, + { + "epoch": 2.13, + "learning_rate": 4.1275281271779854e-05, + "loss": 0.1297, + "step": 6560 + }, + { + "epoch": 2.13, + "learning_rate": 4.113374101735998e-05, + "loss": 0.1359, + "step": 6565 + }, + { + "epoch": 2.13, + "learning_rate": 4.099238099980711e-05, + "loss": 0.1251, + "step": 6570 + }, + { + "epoch": 2.13, + "learning_rate": 4.0851201651937755e-05, + "loss": 0.1291, + "step": 6575 + }, + { + "epoch": 2.13, + "learning_rate": 4.0710203406015204e-05, + "loss": 0.1278, + "step": 6580 + }, + { + "epoch": 2.13, + "learning_rate": 4.05693866937483e-05, + "loss": 0.1191, + "step": 6585 + }, + { + "epoch": 2.14, + "learning_rate": 4.0428751946289956e-05, + "loss": 0.1256, + "step": 6590 + }, + { + "epoch": 2.14, + "learning_rate": 4.028829959423608e-05, + "loss": 0.1299, + "step": 6595 + }, + { + "epoch": 2.14, + "learning_rate": 4.014803006762398e-05, + "loss": 0.131, + "step": 6600 + }, + { + "epoch": 2.14, + "learning_rate": 4.0007943795931346e-05, + "loss": 0.1292, + "step": 6605 + }, + { + "epoch": 2.14, + "learning_rate": 3.9868041208074617e-05, + "loss": 0.1311, + "step": 6610 + }, + { + "epoch": 2.14, + "learning_rate": 3.972832273240796e-05, + "loss": 0.1273, + "step": 6615 + }, + { + "epoch": 2.15, + "learning_rate": 3.958878879672172e-05, + "loss": 0.1352, + "step": 6620 + }, + { + "epoch": 2.15, + "learning_rate": 3.9449439828241354e-05, + "loss": 0.129, + "step": 6625 + }, + { + "epoch": 2.15, + "learning_rate": 3.93102762536258e-05, + "loss": 0.1264, + "step": 6630 + }, + { + "epoch": 2.15, + "learning_rate": 3.917129849896653e-05, + "loss": 0.1248, + "step": 6635 + }, + { + "epoch": 2.15, + "learning_rate": 3.903250698978596e-05, + "loss": 0.1164, + "step": 6640 + }, + { + "epoch": 2.15, + "learning_rate": 3.889390215103631e-05, + "loss": 0.13, + "step": 6645 + }, + { + "epoch": 2.15, + "learning_rate": 3.875548440709827e-05, + "loss": 0.131, + "step": 6650 + }, + { + "epoch": 2.16, + "learning_rate": 3.861725418177961e-05, + "loss": 0.1253, + "step": 6655 + }, + { + "epoch": 2.16, + "learning_rate": 3.8479211898314035e-05, + "loss": 0.129, + "step": 6660 + }, + { + "epoch": 2.16, + "learning_rate": 3.8341357979359735e-05, + "loss": 0.1304, + "step": 6665 + }, + { + "epoch": 2.16, + "learning_rate": 3.820369284699823e-05, + "loss": 0.1252, + "step": 6670 + }, + { + "epoch": 2.16, + "learning_rate": 3.8066216922733e-05, + "loss": 0.129, + "step": 6675 + }, + { + "epoch": 2.16, + "learning_rate": 3.792893062748814e-05, + "loss": 0.1261, + "step": 6680 + }, + { + "epoch": 2.17, + "learning_rate": 3.7791834381607214e-05, + "loss": 0.1278, + "step": 6685 + }, + { + "epoch": 2.17, + "learning_rate": 3.7654928604851924e-05, + "loss": 0.1214, + "step": 6690 + }, + { + "epoch": 2.17, + "learning_rate": 3.751821371640065e-05, + "loss": 0.1227, + "step": 6695 + }, + { + "epoch": 2.17, + "learning_rate": 3.7381690134847483e-05, + "loss": 0.1272, + "step": 6700 + }, + { + "epoch": 2.17, + "learning_rate": 3.724535827820059e-05, + "loss": 0.1309, + "step": 6705 + }, + { + "epoch": 2.17, + "learning_rate": 3.7109218563881274e-05, + "loss": 0.1386, + "step": 6710 + }, + { + "epoch": 2.18, + "learning_rate": 3.6973271408722496e-05, + "loss": 0.1272, + "step": 6715 + }, + { + "epoch": 2.18, + "learning_rate": 3.6837517228967535e-05, + "loss": 0.1323, + "step": 6720 + }, + { + "epoch": 2.18, + "learning_rate": 3.670195644026898e-05, + "loss": 0.1259, + "step": 6725 + }, + { + "epoch": 2.18, + "learning_rate": 3.656658945768713e-05, + "loss": 0.131, + "step": 6730 + }, + { + "epoch": 2.18, + "learning_rate": 3.6431416695689046e-05, + "loss": 0.1209, + "step": 6735 + }, + { + "epoch": 2.18, + "learning_rate": 3.6296438568146974e-05, + "loss": 0.131, + "step": 6740 + }, + { + "epoch": 2.19, + "learning_rate": 3.616165548833738e-05, + "loss": 0.1306, + "step": 6745 + }, + { + "epoch": 2.19, + "learning_rate": 3.602706786893937e-05, + "loss": 0.1346, + "step": 6750 + }, + { + "epoch": 2.19, + "learning_rate": 3.589267612203374e-05, + "loss": 0.1231, + "step": 6755 + }, + { + "epoch": 2.19, + "learning_rate": 3.575848065910145e-05, + "loss": 0.1281, + "step": 6760 + }, + { + "epoch": 2.19, + "learning_rate": 3.5624481891022565e-05, + "loss": 0.1325, + "step": 6765 + }, + { + "epoch": 2.19, + "learning_rate": 3.54906802280748e-05, + "loss": 0.1274, + "step": 6770 + }, + { + "epoch": 2.2, + "learning_rate": 3.5357076079932495e-05, + "loss": 0.1223, + "step": 6775 + }, + { + "epoch": 2.2, + "learning_rate": 3.522366985566518e-05, + "loss": 0.1292, + "step": 6780 + }, + { + "epoch": 2.2, + "learning_rate": 3.509046196373639e-05, + "loss": 0.1293, + "step": 6785 + }, + { + "epoch": 2.2, + "learning_rate": 3.4957452812002355e-05, + "loss": 0.1217, + "step": 6790 + }, + { + "epoch": 2.2, + "learning_rate": 3.482464280771087e-05, + "loss": 0.1279, + "step": 6795 + }, + { + "epoch": 2.2, + "learning_rate": 3.4692032357500007e-05, + "loss": 0.1278, + "step": 6800 + }, + { + "epoch": 2.21, + "learning_rate": 3.455962186739671e-05, + "loss": 0.1266, + "step": 6805 + }, + { + "epoch": 2.21, + "learning_rate": 3.442741174281584e-05, + "loss": 0.1307, + "step": 6810 + }, + { + "epoch": 2.21, + "learning_rate": 3.429540238855863e-05, + "loss": 0.1263, + "step": 6815 + }, + { + "epoch": 2.21, + "learning_rate": 3.416359420881176e-05, + "loss": 0.1232, + "step": 6820 + }, + { + "epoch": 2.21, + "learning_rate": 3.4031987607145756e-05, + "loss": 0.1277, + "step": 6825 + }, + { + "epoch": 2.21, + "learning_rate": 3.390058298651415e-05, + "loss": 0.1179, + "step": 6830 + }, + { + "epoch": 2.21, + "learning_rate": 3.37693807492519e-05, + "loss": 0.1187, + "step": 6835 + }, + { + "epoch": 2.22, + "learning_rate": 3.363838129707437e-05, + "loss": 0.1285, + "step": 6840 + }, + { + "epoch": 2.22, + "learning_rate": 3.3507585031076083e-05, + "loss": 0.1241, + "step": 6845 + }, + { + "epoch": 2.22, + "learning_rate": 3.337699235172931e-05, + "loss": 0.133, + "step": 6850 + }, + { + "epoch": 2.22, + "learning_rate": 3.324660365888313e-05, + "loss": 0.1239, + "step": 6855 + }, + { + "epoch": 2.22, + "learning_rate": 3.311641935176192e-05, + "loss": 0.1297, + "step": 6860 + }, + { + "epoch": 2.22, + "learning_rate": 3.298643982896441e-05, + "loss": 0.1238, + "step": 6865 + }, + { + "epoch": 2.23, + "learning_rate": 3.285666548846217e-05, + "loss": 0.1242, + "step": 6870 + }, + { + "epoch": 2.23, + "learning_rate": 3.2727096727598696e-05, + "loss": 0.1297, + "step": 6875 + }, + { + "epoch": 2.23, + "learning_rate": 3.259773394308787e-05, + "loss": 0.1292, + "step": 6880 + }, + { + "epoch": 2.23, + "learning_rate": 3.246857753101308e-05, + "loss": 0.1298, + "step": 6885 + }, + { + "epoch": 2.23, + "learning_rate": 3.233962788682573e-05, + "loss": 0.1307, + "step": 6890 + }, + { + "epoch": 2.23, + "learning_rate": 3.2210885405344235e-05, + "loss": 0.1339, + "step": 6895 + }, + { + "epoch": 2.24, + "learning_rate": 3.20823504807526e-05, + "loss": 0.1233, + "step": 6900 + }, + { + "epoch": 2.24, + "learning_rate": 3.195402350659945e-05, + "loss": 0.1192, + "step": 6905 + }, + { + "epoch": 2.24, + "learning_rate": 3.182590487579659e-05, + "loss": 0.1307, + "step": 6910 + }, + { + "epoch": 2.24, + "learning_rate": 3.169799498061803e-05, + "loss": 0.1216, + "step": 6915 + }, + { + "epoch": 2.24, + "learning_rate": 3.157029421269863e-05, + "loss": 0.1245, + "step": 6920 + }, + { + "epoch": 2.24, + "learning_rate": 3.144280296303289e-05, + "loss": 0.1293, + "step": 6925 + }, + { + "epoch": 2.25, + "learning_rate": 3.1315521621973934e-05, + "loss": 0.122, + "step": 6930 + }, + { + "epoch": 2.25, + "learning_rate": 3.1188450579232007e-05, + "loss": 0.1244, + "step": 6935 + }, + { + "epoch": 2.25, + "learning_rate": 3.106159022387365e-05, + "loss": 0.1249, + "step": 6940 + }, + { + "epoch": 2.25, + "learning_rate": 3.093494094432018e-05, + "loss": 0.1282, + "step": 6945 + }, + { + "epoch": 2.25, + "learning_rate": 3.080850312834673e-05, + "loss": 0.128, + "step": 6950 + }, + { + "epoch": 2.25, + "learning_rate": 3.06822771630809e-05, + "loss": 0.1277, + "step": 6955 + }, + { + "epoch": 2.26, + "learning_rate": 3.055626343500173e-05, + "loss": 0.124, + "step": 6960 + }, + { + "epoch": 2.26, + "learning_rate": 3.0430462329938326e-05, + "loss": 0.1308, + "step": 6965 + }, + { + "epoch": 2.26, + "learning_rate": 3.0304874233068912e-05, + "loss": 0.1276, + "step": 6970 + }, + { + "epoch": 2.26, + "learning_rate": 3.01794995289194e-05, + "loss": 0.1255, + "step": 6975 + }, + { + "epoch": 2.26, + "learning_rate": 3.0054338601362396e-05, + "loss": 0.1222, + "step": 6980 + }, + { + "epoch": 2.26, + "learning_rate": 2.9929391833616003e-05, + "loss": 0.1231, + "step": 6985 + }, + { + "epoch": 2.27, + "learning_rate": 2.9804659608242512e-05, + "loss": 0.1263, + "step": 6990 + }, + { + "epoch": 2.27, + "learning_rate": 2.9680142307147397e-05, + "loss": 0.1235, + "step": 6995 + }, + { + "epoch": 2.27, + "learning_rate": 2.9555840311578066e-05, + "loss": 0.1241, + "step": 7000 + }, + { + "epoch": 2.27, + "learning_rate": 2.9431754002122715e-05, + "loss": 0.1274, + "step": 7005 + }, + { + "epoch": 2.27, + "learning_rate": 2.9307883758709086e-05, + "loss": 0.1318, + "step": 7010 + }, + { + "epoch": 2.27, + "learning_rate": 2.918422996060347e-05, + "loss": 0.1199, + "step": 7015 + }, + { + "epoch": 2.27, + "learning_rate": 2.9060792986409345e-05, + "loss": 0.1234, + "step": 7020 + }, + { + "epoch": 2.28, + "learning_rate": 2.8937573214066417e-05, + "loss": 0.1223, + "step": 7025 + }, + { + "epoch": 2.28, + "learning_rate": 2.8814571020849236e-05, + "loss": 0.1244, + "step": 7030 + }, + { + "epoch": 2.28, + "learning_rate": 2.8691786783366316e-05, + "loss": 0.1202, + "step": 7035 + }, + { + "epoch": 2.28, + "learning_rate": 2.85692208775587e-05, + "loss": 0.1221, + "step": 7040 + }, + { + "epoch": 2.28, + "learning_rate": 2.8446873678699027e-05, + "loss": 0.1288, + "step": 7045 + }, + { + "epoch": 2.28, + "learning_rate": 2.8324745561390296e-05, + "loss": 0.1221, + "step": 7050 + }, + { + "epoch": 2.29, + "learning_rate": 2.8202836899564666e-05, + "loss": 0.1304, + "step": 7055 + }, + { + "epoch": 2.29, + "learning_rate": 2.8081148066482433e-05, + "loss": 0.1337, + "step": 7060 + }, + { + "epoch": 2.29, + "learning_rate": 2.7959679434730724e-05, + "loss": 0.1212, + "step": 7065 + }, + { + "epoch": 2.29, + "learning_rate": 2.7838431376222595e-05, + "loss": 0.1185, + "step": 7070 + }, + { + "epoch": 2.29, + "learning_rate": 2.77174042621956e-05, + "loss": 0.1305, + "step": 7075 + }, + { + "epoch": 2.29, + "learning_rate": 2.7596598463210955e-05, + "loss": 0.123, + "step": 7080 + }, + { + "epoch": 2.3, + "learning_rate": 2.747601434915208e-05, + "loss": 0.1219, + "step": 7085 + }, + { + "epoch": 2.3, + "learning_rate": 2.735565228922379e-05, + "loss": 0.1224, + "step": 7090 + }, + { + "epoch": 2.3, + "learning_rate": 2.723551265195098e-05, + "loss": 0.1228, + "step": 7095 + }, + { + "epoch": 2.3, + "learning_rate": 2.711559580517744e-05, + "loss": 0.1202, + "step": 7100 + }, + { + "epoch": 2.3, + "learning_rate": 2.699590211606493e-05, + "loss": 0.1219, + "step": 7105 + }, + { + "epoch": 2.3, + "learning_rate": 2.6876431951091897e-05, + "loss": 0.1274, + "step": 7110 + }, + { + "epoch": 2.31, + "learning_rate": 2.6757185676052443e-05, + "loss": 0.1195, + "step": 7115 + }, + { + "epoch": 2.31, + "learning_rate": 2.663816365605506e-05, + "loss": 0.1241, + "step": 7120 + }, + { + "epoch": 2.31, + "learning_rate": 2.651936625552175e-05, + "loss": 0.1256, + "step": 7125 + }, + { + "epoch": 2.31, + "learning_rate": 2.6400793838186644e-05, + "loss": 0.1192, + "step": 7130 + }, + { + "epoch": 2.31, + "learning_rate": 2.6282446767095138e-05, + "loss": 0.1165, + "step": 7135 + }, + { + "epoch": 2.31, + "learning_rate": 2.616432540460255e-05, + "loss": 0.1253, + "step": 7140 + }, + { + "epoch": 2.32, + "learning_rate": 2.6046430112373233e-05, + "loss": 0.1204, + "step": 7145 + }, + { + "epoch": 2.32, + "learning_rate": 2.5928761251379265e-05, + "loss": 0.1262, + "step": 7150 + }, + { + "epoch": 2.32, + "learning_rate": 2.5811319181899505e-05, + "loss": 0.1204, + "step": 7155 + }, + { + "epoch": 2.32, + "learning_rate": 2.569410426351836e-05, + "loss": 0.1335, + "step": 7160 + }, + { + "epoch": 2.32, + "learning_rate": 2.5577116855124816e-05, + "loss": 0.1286, + "step": 7165 + }, + { + "epoch": 2.32, + "learning_rate": 2.5460357314911176e-05, + "loss": 0.1253, + "step": 7170 + }, + { + "epoch": 2.33, + "learning_rate": 2.5343826000372163e-05, + "loss": 0.1281, + "step": 7175 + }, + { + "epoch": 2.33, + "learning_rate": 2.522752326830361e-05, + "loss": 0.1247, + "step": 7180 + }, + { + "epoch": 2.33, + "learning_rate": 2.5111449474801553e-05, + "loss": 0.1256, + "step": 7185 + }, + { + "epoch": 2.33, + "learning_rate": 2.4995604975261077e-05, + "loss": 0.1245, + "step": 7190 + }, + { + "epoch": 2.33, + "learning_rate": 2.4879990124375118e-05, + "loss": 0.1236, + "step": 7195 + }, + { + "epoch": 2.33, + "learning_rate": 2.4764605276133544e-05, + "loss": 0.1211, + "step": 7200 + }, + { + "epoch": 2.33, + "learning_rate": 2.4649450783822038e-05, + "loss": 0.1199, + "step": 7205 + }, + { + "epoch": 2.34, + "learning_rate": 2.4534527000020858e-05, + "loss": 0.1253, + "step": 7210 + }, + { + "epoch": 2.34, + "learning_rate": 2.441983427660397e-05, + "loss": 0.1285, + "step": 7215 + }, + { + "epoch": 2.34, + "learning_rate": 2.4305372964737905e-05, + "loss": 0.126, + "step": 7220 + }, + { + "epoch": 2.34, + "learning_rate": 2.4191143414880546e-05, + "loss": 0.1287, + "step": 7225 + }, + { + "epoch": 2.34, + "learning_rate": 2.407714597678028e-05, + "loss": 0.1259, + "step": 7230 + }, + { + "epoch": 2.34, + "learning_rate": 2.3963380999474704e-05, + "loss": 0.1232, + "step": 7235 + }, + { + "epoch": 2.35, + "learning_rate": 2.38498488312898e-05, + "loss": 0.124, + "step": 7240 + }, + { + "epoch": 2.35, + "learning_rate": 2.373654981983858e-05, + "loss": 0.1247, + "step": 7245 + }, + { + "epoch": 2.35, + "learning_rate": 2.3623484312020305e-05, + "loss": 0.1194, + "step": 7250 + }, + { + "epoch": 2.35, + "learning_rate": 2.3510652654019272e-05, + "loss": 0.1238, + "step": 7255 + }, + { + "epoch": 2.35, + "learning_rate": 2.339805519130367e-05, + "loss": 0.1225, + "step": 7260 + }, + { + "epoch": 2.35, + "learning_rate": 2.328569226862479e-05, + "loss": 0.1256, + "step": 7265 + }, + { + "epoch": 2.36, + "learning_rate": 2.3173564230015655e-05, + "loss": 0.1179, + "step": 7270 + }, + { + "epoch": 2.36, + "learning_rate": 2.3061671418790254e-05, + "loss": 0.1271, + "step": 7275 + }, + { + "epoch": 2.36, + "learning_rate": 2.2950014177542233e-05, + "loss": 0.1221, + "step": 7280 + }, + { + "epoch": 2.36, + "learning_rate": 2.2838592848144068e-05, + "loss": 0.1226, + "step": 7285 + }, + { + "epoch": 2.36, + "learning_rate": 2.2727407771745844e-05, + "loss": 0.1236, + "step": 7290 + }, + { + "epoch": 2.36, + "learning_rate": 2.2616459288774373e-05, + "loss": 0.1229, + "step": 7295 + }, + { + "epoch": 2.37, + "learning_rate": 2.250574773893196e-05, + "loss": 0.1145, + "step": 7300 + }, + { + "epoch": 2.37, + "learning_rate": 2.2395273461195533e-05, + "loss": 0.1352, + "step": 7305 + }, + { + "epoch": 2.37, + "learning_rate": 2.2285036793815527e-05, + "loss": 0.1273, + "step": 7310 + }, + { + "epoch": 2.37, + "learning_rate": 2.2175038074314914e-05, + "loss": 0.1199, + "step": 7315 + }, + { + "epoch": 2.37, + "learning_rate": 2.206527763948798e-05, + "loss": 0.1262, + "step": 7320 + }, + { + "epoch": 2.37, + "learning_rate": 2.195575582539956e-05, + "loss": 0.1241, + "step": 7325 + }, + { + "epoch": 2.38, + "learning_rate": 2.184647296738386e-05, + "loss": 0.1192, + "step": 7330 + }, + { + "epoch": 2.38, + "learning_rate": 2.1737429400043353e-05, + "loss": 0.1171, + "step": 7335 + }, + { + "epoch": 2.38, + "learning_rate": 2.1628625457247985e-05, + "loss": 0.122, + "step": 7340 + }, + { + "epoch": 2.38, + "learning_rate": 2.1520061472133902e-05, + "loss": 0.1182, + "step": 7345 + }, + { + "epoch": 2.38, + "learning_rate": 2.1411737777102647e-05, + "loss": 0.1261, + "step": 7350 + }, + { + "epoch": 2.38, + "learning_rate": 2.1303654703819966e-05, + "loss": 0.1185, + "step": 7355 + }, + { + "epoch": 2.39, + "learning_rate": 2.1195812583214914e-05, + "loss": 0.1183, + "step": 7360 + }, + { + "epoch": 2.39, + "learning_rate": 2.1088211745478748e-05, + "loss": 0.1291, + "step": 7365 + }, + { + "epoch": 2.39, + "learning_rate": 2.0980852520064053e-05, + "loss": 0.1215, + "step": 7370 + }, + { + "epoch": 2.39, + "learning_rate": 2.0873735235683535e-05, + "loss": 0.1258, + "step": 7375 + }, + { + "epoch": 2.39, + "learning_rate": 2.0766860220309194e-05, + "loss": 0.1274, + "step": 7380 + }, + { + "epoch": 2.39, + "learning_rate": 2.0660227801171263e-05, + "loss": 0.1194, + "step": 7385 + }, + { + "epoch": 2.39, + "learning_rate": 2.055383830475711e-05, + "loss": 0.1212, + "step": 7390 + }, + { + "epoch": 2.4, + "learning_rate": 2.044769205681045e-05, + "loss": 0.124, + "step": 7395 + }, + { + "epoch": 2.4, + "learning_rate": 2.0341789382330068e-05, + "loss": 0.1174, + "step": 7400 + }, + { + "epoch": 2.4, + "learning_rate": 2.0236130605569102e-05, + "loss": 0.1213, + "step": 7405 + }, + { + "epoch": 2.4, + "learning_rate": 2.0130716050033837e-05, + "loss": 0.1325, + "step": 7410 + }, + { + "epoch": 2.4, + "learning_rate": 2.0025546038482857e-05, + "loss": 0.1244, + "step": 7415 + }, + { + "epoch": 2.4, + "learning_rate": 1.992062089292599e-05, + "loss": 0.1285, + "step": 7420 + }, + { + "epoch": 2.41, + "learning_rate": 1.9815940934623324e-05, + "loss": 0.12, + "step": 7425 + }, + { + "epoch": 2.41, + "learning_rate": 1.9711506484084207e-05, + "loss": 0.1194, + "step": 7430 + }, + { + "epoch": 2.41, + "learning_rate": 1.9607317861066353e-05, + "loss": 0.1254, + "step": 7435 + }, + { + "epoch": 2.41, + "learning_rate": 1.950337538457472e-05, + "loss": 0.1249, + "step": 7440 + }, + { + "epoch": 2.41, + "learning_rate": 1.9399679372860667e-05, + "loss": 0.1236, + "step": 7445 + }, + { + "epoch": 2.41, + "learning_rate": 1.9296230143420956e-05, + "loss": 0.1181, + "step": 7450 + }, + { + "epoch": 2.42, + "learning_rate": 1.919302801299665e-05, + "loss": 0.1244, + "step": 7455 + }, + { + "epoch": 2.42, + "learning_rate": 1.9090073297572376e-05, + "loss": 0.1266, + "step": 7460 + }, + { + "epoch": 2.42, + "learning_rate": 1.8987366312375066e-05, + "loss": 0.1194, + "step": 7465 + }, + { + "epoch": 2.42, + "learning_rate": 1.8884907371873317e-05, + "loss": 0.1229, + "step": 7470 + }, + { + "epoch": 2.42, + "learning_rate": 1.8782696789776134e-05, + "loss": 0.1124, + "step": 7475 + }, + { + "epoch": 2.42, + "learning_rate": 1.868073487903218e-05, + "loss": 0.1253, + "step": 7480 + }, + { + "epoch": 2.43, + "learning_rate": 1.8579021951828657e-05, + "loss": 0.1177, + "step": 7485 + }, + { + "epoch": 2.43, + "learning_rate": 1.8477558319590538e-05, + "loss": 0.1224, + "step": 7490 + }, + { + "epoch": 2.43, + "learning_rate": 1.8376344292979365e-05, + "loss": 0.1195, + "step": 7495 + }, + { + "epoch": 2.43, + "learning_rate": 1.8275380181892587e-05, + "loss": 0.1221, + "step": 7500 + }, + { + "epoch": 2.43, + "learning_rate": 1.8174666295462318e-05, + "loss": 0.1215, + "step": 7505 + }, + { + "epoch": 2.43, + "learning_rate": 1.807420294205464e-05, + "loss": 0.1218, + "step": 7510 + }, + { + "epoch": 2.44, + "learning_rate": 1.797399042926853e-05, + "loss": 0.1248, + "step": 7515 + }, + { + "epoch": 2.44, + "learning_rate": 1.787402906393486e-05, + "loss": 0.1218, + "step": 7520 + }, + { + "epoch": 2.44, + "learning_rate": 1.777431915211566e-05, + "loss": 0.1199, + "step": 7525 + }, + { + "epoch": 2.44, + "learning_rate": 1.767486099910297e-05, + "loss": 0.1208, + "step": 7530 + }, + { + "epoch": 2.44, + "learning_rate": 1.757565490941807e-05, + "loss": 0.1193, + "step": 7535 + }, + { + "epoch": 2.44, + "learning_rate": 1.7476701186810373e-05, + "loss": 0.1214, + "step": 7540 + }, + { + "epoch": 2.44, + "learning_rate": 1.73780001342567e-05, + "loss": 0.1241, + "step": 7545 + }, + { + "epoch": 2.45, + "learning_rate": 1.7279552053960134e-05, + "loss": 0.1193, + "step": 7550 + }, + { + "epoch": 2.45, + "learning_rate": 1.7181357247349318e-05, + "loss": 0.1229, + "step": 7555 + }, + { + "epoch": 2.45, + "learning_rate": 1.7083416015077326e-05, + "loss": 0.1255, + "step": 7560 + }, + { + "epoch": 2.45, + "learning_rate": 1.698572865702094e-05, + "loss": 0.1269, + "step": 7565 + }, + { + "epoch": 2.45, + "learning_rate": 1.688829547227949e-05, + "loss": 0.1217, + "step": 7570 + }, + { + "epoch": 2.45, + "learning_rate": 1.679111675917424e-05, + "loss": 0.1174, + "step": 7575 + }, + { + "epoch": 2.46, + "learning_rate": 1.6694192815247177e-05, + "loss": 0.1205, + "step": 7580 + }, + { + "epoch": 2.46, + "learning_rate": 1.6597523937260294e-05, + "loss": 0.1177, + "step": 7585 + }, + { + "epoch": 2.46, + "learning_rate": 1.6501110421194656e-05, + "loss": 0.1252, + "step": 7590 + }, + { + "epoch": 2.46, + "learning_rate": 1.6404952562249366e-05, + "loss": 0.1218, + "step": 7595 + }, + { + "epoch": 2.46, + "learning_rate": 1.6309050654840863e-05, + "loss": 0.1173, + "step": 7600 + }, + { + "epoch": 2.46, + "learning_rate": 1.62134049926018e-05, + "loss": 0.1183, + "step": 7605 + }, + { + "epoch": 2.47, + "learning_rate": 1.6118015868380388e-05, + "loss": 0.1243, + "step": 7610 + }, + { + "epoch": 2.47, + "learning_rate": 1.6022883574239233e-05, + "loss": 0.121, + "step": 7615 + }, + { + "epoch": 2.47, + "learning_rate": 1.5928008401454676e-05, + "loss": 0.1197, + "step": 7620 + }, + { + "epoch": 2.47, + "learning_rate": 1.5833390640515787e-05, + "loss": 0.1229, + "step": 7625 + }, + { + "epoch": 2.47, + "learning_rate": 1.573903058112344e-05, + "loss": 0.1278, + "step": 7630 + }, + { + "epoch": 2.47, + "learning_rate": 1.5644928512189527e-05, + "loss": 0.1218, + "step": 7635 + }, + { + "epoch": 2.48, + "learning_rate": 1.555108472183604e-05, + "loss": 0.1225, + "step": 7640 + }, + { + "epoch": 2.48, + "learning_rate": 1.54574994973941e-05, + "loss": 0.1171, + "step": 7645 + }, + { + "epoch": 2.48, + "learning_rate": 1.5364173125403215e-05, + "loss": 0.1193, + "step": 7650 + }, + { + "epoch": 2.48, + "learning_rate": 1.5271105891610327e-05, + "loss": 0.1183, + "step": 7655 + }, + { + "epoch": 2.48, + "learning_rate": 1.517829808096891e-05, + "loss": 0.1225, + "step": 7660 + }, + { + "epoch": 2.48, + "learning_rate": 1.5085749977638209e-05, + "loss": 0.1249, + "step": 7665 + }, + { + "epoch": 2.49, + "learning_rate": 1.4993461864982195e-05, + "loss": 0.1197, + "step": 7670 + }, + { + "epoch": 2.49, + "learning_rate": 1.4901434025568905e-05, + "loss": 0.1224, + "step": 7675 + }, + { + "epoch": 2.49, + "learning_rate": 1.480966674116937e-05, + "loss": 0.1205, + "step": 7680 + }, + { + "epoch": 2.49, + "learning_rate": 1.4718160292756955e-05, + "loss": 0.1248, + "step": 7685 + }, + { + "epoch": 2.49, + "learning_rate": 1.46269149605063e-05, + "loss": 0.1216, + "step": 7690 + }, + { + "epoch": 2.49, + "learning_rate": 1.4535931023792648e-05, + "loss": 0.1168, + "step": 7695 + }, + { + "epoch": 2.5, + "learning_rate": 1.4445208761190809e-05, + "loss": 0.1188, + "step": 7700 + }, + { + "epoch": 2.5, + "learning_rate": 1.4354748450474486e-05, + "loss": 0.1153, + "step": 7705 + }, + { + "epoch": 2.5, + "learning_rate": 1.426455036861527e-05, + "loss": 0.124, + "step": 7710 + }, + { + "epoch": 2.5, + "learning_rate": 1.4174614791781892e-05, + "loss": 0.1219, + "step": 7715 + }, + { + "epoch": 2.5, + "learning_rate": 1.4084941995339362e-05, + "loss": 0.1224, + "step": 7720 + }, + { + "epoch": 2.5, + "learning_rate": 1.3995532253848054e-05, + "loss": 0.1253, + "step": 7725 + }, + { + "epoch": 2.5, + "learning_rate": 1.3906385841062952e-05, + "loss": 0.1198, + "step": 7730 + }, + { + "epoch": 2.51, + "learning_rate": 1.3817503029932788e-05, + "loss": 0.1307, + "step": 7735 + }, + { + "epoch": 2.51, + "learning_rate": 1.3728884092599215e-05, + "loss": 0.1236, + "step": 7740 + }, + { + "epoch": 2.51, + "learning_rate": 1.3640529300395876e-05, + "loss": 0.1227, + "step": 7745 + }, + { + "epoch": 2.51, + "learning_rate": 1.3552438923847766e-05, + "loss": 0.1246, + "step": 7750 + }, + { + "epoch": 2.51, + "learning_rate": 1.3464613232670164e-05, + "loss": 0.1171, + "step": 7755 + }, + { + "epoch": 2.51, + "learning_rate": 1.337705249576806e-05, + "loss": 0.1147, + "step": 7760 + }, + { + "epoch": 2.52, + "learning_rate": 1.3289756981235113e-05, + "loss": 0.1229, + "step": 7765 + }, + { + "epoch": 2.52, + "learning_rate": 1.3202726956352996e-05, + "loss": 0.1229, + "step": 7770 + }, + { + "epoch": 2.52, + "learning_rate": 1.3115962687590422e-05, + "loss": 0.1232, + "step": 7775 + }, + { + "epoch": 2.52, + "learning_rate": 1.302946444060248e-05, + "loss": 0.1268, + "step": 7780 + }, + { + "epoch": 2.52, + "learning_rate": 1.294323248022975e-05, + "loss": 0.1212, + "step": 7785 + }, + { + "epoch": 2.52, + "learning_rate": 1.2857267070497447e-05, + "loss": 0.1255, + "step": 7790 + }, + { + "epoch": 2.53, + "learning_rate": 1.2771568474614714e-05, + "loss": 0.1233, + "step": 7795 + }, + { + "epoch": 2.53, + "learning_rate": 1.2686136954973716e-05, + "loss": 0.1116, + "step": 7800 + }, + { + "epoch": 2.53, + "learning_rate": 1.2600972773148933e-05, + "loss": 0.1156, + "step": 7805 + }, + { + "epoch": 2.53, + "learning_rate": 1.2516076189896231e-05, + "loss": 0.1155, + "step": 7810 + }, + { + "epoch": 2.53, + "learning_rate": 1.2431447465152257e-05, + "loss": 0.1273, + "step": 7815 + }, + { + "epoch": 2.53, + "learning_rate": 1.2347086858033407e-05, + "loss": 0.1223, + "step": 7820 + }, + { + "epoch": 2.54, + "learning_rate": 1.2262994626835233e-05, + "loss": 0.1221, + "step": 7825 + }, + { + "epoch": 2.54, + "learning_rate": 1.2179171029031577e-05, + "loss": 0.1208, + "step": 7830 + }, + { + "epoch": 2.54, + "learning_rate": 1.2095616321273695e-05, + "loss": 0.1154, + "step": 7835 + }, + { + "epoch": 2.54, + "learning_rate": 1.2012330759389656e-05, + "loss": 0.1208, + "step": 7840 + }, + { + "epoch": 2.54, + "learning_rate": 1.1929314598383423e-05, + "loss": 0.1158, + "step": 7845 + }, + { + "epoch": 2.54, + "learning_rate": 1.1846568092434041e-05, + "loss": 0.1165, + "step": 7850 + }, + { + "epoch": 2.55, + "learning_rate": 1.1764091494895024e-05, + "loss": 0.1171, + "step": 7855 + }, + { + "epoch": 2.55, + "learning_rate": 1.168188505829344e-05, + "loss": 0.1198, + "step": 7860 + }, + { + "epoch": 2.55, + "learning_rate": 1.1599949034329139e-05, + "loss": 0.1251, + "step": 7865 + }, + { + "epoch": 2.55, + "learning_rate": 1.1518283673874098e-05, + "loss": 0.1166, + "step": 7870 + }, + { + "epoch": 2.55, + "learning_rate": 1.1436889226971493e-05, + "loss": 0.1183, + "step": 7875 + }, + { + "epoch": 2.55, + "learning_rate": 1.1355765942835084e-05, + "loss": 0.1202, + "step": 7880 + }, + { + "epoch": 2.56, + "learning_rate": 1.127491406984833e-05, + "loss": 0.1179, + "step": 7885 + }, + { + "epoch": 2.56, + "learning_rate": 1.1194333855563754e-05, + "loss": 0.1223, + "step": 7890 + }, + { + "epoch": 2.56, + "learning_rate": 1.1114025546702012e-05, + "loss": 0.1248, + "step": 7895 + }, + { + "epoch": 2.56, + "learning_rate": 1.1033989389151345e-05, + "loss": 0.1211, + "step": 7900 + }, + { + "epoch": 2.56, + "learning_rate": 1.095422562796663e-05, + "loss": 0.1213, + "step": 7905 + }, + { + "epoch": 2.56, + "learning_rate": 1.0874734507368801e-05, + "loss": 0.1169, + "step": 7910 + }, + { + "epoch": 2.56, + "learning_rate": 1.0795516270743933e-05, + "loss": 0.1195, + "step": 7915 + }, + { + "epoch": 2.57, + "learning_rate": 1.0716571160642664e-05, + "loss": 0.1237, + "step": 7920 + }, + { + "epoch": 2.57, + "learning_rate": 1.0637899418779363e-05, + "loss": 0.1235, + "step": 7925 + }, + { + "epoch": 2.57, + "learning_rate": 1.0559501286031326e-05, + "loss": 0.1195, + "step": 7930 + }, + { + "epoch": 2.57, + "learning_rate": 1.0481377002438186e-05, + "loss": 0.1128, + "step": 7935 + }, + { + "epoch": 2.57, + "learning_rate": 1.0403526807201103e-05, + "loss": 0.1147, + "step": 7940 + }, + { + "epoch": 2.57, + "learning_rate": 1.032595093868196e-05, + "loss": 0.1177, + "step": 7945 + }, + { + "epoch": 2.58, + "learning_rate": 1.0248649634402796e-05, + "loss": 0.1183, + "step": 7950 + }, + { + "epoch": 2.58, + "learning_rate": 1.0171623131044949e-05, + "loss": 0.1198, + "step": 7955 + }, + { + "epoch": 2.58, + "learning_rate": 1.0094871664448336e-05, + "loss": 0.1191, + "step": 7960 + }, + { + "epoch": 2.58, + "learning_rate": 1.0018395469610841e-05, + "loss": 0.1187, + "step": 7965 + }, + { + "epoch": 2.58, + "learning_rate": 9.942194780687453e-06, + "loss": 0.1222, + "step": 7970 + }, + { + "epoch": 2.58, + "learning_rate": 9.866269830989661e-06, + "loss": 0.1241, + "step": 7975 + }, + { + "epoch": 2.59, + "learning_rate": 9.790620852984645e-06, + "loss": 0.1182, + "step": 7980 + }, + { + "epoch": 2.59, + "learning_rate": 9.715248078294658e-06, + "loss": 0.123, + "step": 7985 + }, + { + "epoch": 2.59, + "learning_rate": 9.64015173769629e-06, + "loss": 0.1229, + "step": 7990 + }, + { + "epoch": 2.59, + "learning_rate": 9.565332061119669e-06, + "loss": 0.1197, + "step": 7995 + }, + { + "epoch": 2.59, + "learning_rate": 9.490789277647927e-06, + "loss": 0.1233, + "step": 8000 + } + ], + "logging_steps": 5, + "max_steps": 9255, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 1000, + "total_flos": 1.6294360832244974e+19, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/save_lora_math_r8_2e4/checkpoint-8000/training_args.bin b/save_lora_math_r8_2e4/checkpoint-8000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..175cae96ce99657f1e2b29179eee355d72511307 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-8000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0cedf67f015d3f83c601cebdd463c220d745d8e241fab282366821363eb224d +size 5880 diff --git a/save_lora_math_r8_2e4/checkpoint-8000/zero_to_fp32.py b/save_lora_math_r8_2e4/checkpoint-8000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..c98caae31534368be22b67fc4ae906836c992a8d --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-8000/zero_to_fp32.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in param_shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, args.output_file, tag=args.tag) diff --git a/save_lora_math_r8_2e4/checkpoint-9000/README.md b/save_lora_math_r8_2e4/checkpoint-9000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d380bbbf0bf0fa1f211b53410559607e5007a209 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-9000/README.md @@ -0,0 +1,202 @@ +--- +library_name: peft +base_model: meta-llama/Llama-2-7b-hf +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.9.0 \ No newline at end of file diff --git a/save_lora_math_r8_2e4/checkpoint-9000/adapter_config.json b/save_lora_math_r8_2e4/checkpoint-9000/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9c81a592f0ff08374a4f61f6003029070a8429ae --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-9000/adapter_config.json @@ -0,0 +1,35 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "meta-llama/Llama-2-7b-hf", + "bias": "none", + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "mora_type": 1, + "peft_type": "LORA", + "r": 8, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "down_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "use_dora": false, + "use_mora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/save_lora_math_r8_2e4/checkpoint-9000/adapter_model.safetensors b/save_lora_math_r8_2e4/checkpoint-9000/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..758ee97623d2a014770bd7491b3d4aff6f7dae7c --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-9000/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caf0f7a45138367d8a43c6a25e4a37e7856b7137bcdaa738b7f7cb846c767f8c +size 40036488 diff --git a/save_lora_math_r8_2e4/checkpoint-9000/global_step9000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/save_lora_math_r8_2e4/checkpoint-9000/global_step9000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f4feb7bb08bec5bbb931d283965c9a3f67029aba --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-9000/global_step9000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9104da342dc8e2645386f7111eec59a33347aaa95985e9ebda9a2623b5dac60 +size 119958768 diff --git a/save_lora_math_r8_2e4/checkpoint-9000/global_step9000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/save_lora_math_r8_2e4/checkpoint-9000/global_step9000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b7369e5a6c2a7fe32352ee63fcb9c72ae8f54ce --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-9000/global_step9000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3142c903091ad6dcf28253e70000c9ae422784df9ec8eb11874e32c63ff5739 +size 119958896 diff --git a/save_lora_math_r8_2e4/checkpoint-9000/global_step9000/mp_rank_00_model_states.pt b/save_lora_math_r8_2e4/checkpoint-9000/global_step9000/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c724a9b107e65318c53502e31a2e1d32a2a01ae6 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-9000/global_step9000/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12c7020d058b7c9b294315370c3a0ecba371c916dfb49d60c30f405e41bd2513 +size 40303404 diff --git a/save_lora_math_r8_2e4/checkpoint-9000/latest b/save_lora_math_r8_2e4/checkpoint-9000/latest new file mode 100644 index 0000000000000000000000000000000000000000..99981cb573ef099d25110dd7d77225a8bc8a8f96 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-9000/latest @@ -0,0 +1 @@ +global_step9000 \ No newline at end of file diff --git a/save_lora_math_r8_2e4/checkpoint-9000/rng_state_0.pth b/save_lora_math_r8_2e4/checkpoint-9000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..f213417562ac4a374e5910f5032d8144a0639993 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-9000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75d2b12e3554fb793bb50c063ac883aea5aed378c34f8e8a0b5366ecc6448bf7 +size 14512 diff --git a/save_lora_math_r8_2e4/checkpoint-9000/rng_state_1.pth b/save_lora_math_r8_2e4/checkpoint-9000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..1024a5263b64ce411d143a8df69aaaa809d3220b --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-9000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1274dd94a4be308c12af1e3bf6b33c5fef668cb13ce1a80b8d07bc8ce98e6bcc +size 14512 diff --git a/save_lora_math_r8_2e4/checkpoint-9000/scheduler.pt b/save_lora_math_r8_2e4/checkpoint-9000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..988ff5f3510dbeb90ccf5c23681325a0de4af6ed --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-9000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6c229b576938b9f862003f3553b3984f07b5e1c8493b877dad3dca81c90360c +size 1064 diff --git a/save_lora_math_r8_2e4/checkpoint-9000/trainer_state.json b/save_lora_math_r8_2e4/checkpoint-9000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..fa9338195437b05eb6cccb600989393af98c3b2b --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-9000/trainer_state.json @@ -0,0 +1,10821 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.9164556962025316, + "eval_steps": 500, + "global_step": 9000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 3.5971223021582732e-06, + "loss": 0.7628, + "step": 5 + }, + { + "epoch": 0.0, + "learning_rate": 7.1942446043165465e-06, + "loss": 0.769, + "step": 10 + }, + { + "epoch": 0.0, + "learning_rate": 1.0791366906474821e-05, + "loss": 0.7619, + "step": 15 + }, + { + "epoch": 0.01, + "learning_rate": 1.4388489208633093e-05, + "loss": 0.7564, + "step": 20 + }, + { + "epoch": 0.01, + "learning_rate": 1.7985611510791367e-05, + "loss": 0.7549, + "step": 25 + }, + { + "epoch": 0.01, + "learning_rate": 2.1582733812949642e-05, + "loss": 0.7001, + "step": 30 + }, + { + "epoch": 0.01, + "learning_rate": 2.5179856115107914e-05, + "loss": 0.6289, + "step": 35 + }, + { + "epoch": 0.01, + "learning_rate": 2.8776978417266186e-05, + "loss": 0.5632, + "step": 40 + }, + { + "epoch": 0.01, + "learning_rate": 3.237410071942446e-05, + "loss": 0.4923, + "step": 45 + }, + { + "epoch": 0.02, + "learning_rate": 3.597122302158273e-05, + "loss": 0.4582, + "step": 50 + }, + { + "epoch": 0.02, + "learning_rate": 3.956834532374101e-05, + "loss": 0.4357, + "step": 55 + }, + { + "epoch": 0.02, + "learning_rate": 4.3165467625899284e-05, + "loss": 0.4079, + "step": 60 + }, + { + "epoch": 0.02, + "learning_rate": 4.676258992805755e-05, + "loss": 0.4047, + "step": 65 + }, + { + "epoch": 0.02, + "learning_rate": 5.035971223021583e-05, + "loss": 0.3916, + "step": 70 + }, + { + "epoch": 0.02, + "learning_rate": 5.39568345323741e-05, + "loss": 0.3926, + "step": 75 + }, + { + "epoch": 0.03, + "learning_rate": 5.755395683453237e-05, + "loss": 0.3725, + "step": 80 + }, + { + "epoch": 0.03, + "learning_rate": 6.115107913669065e-05, + "loss": 0.3769, + "step": 85 + }, + { + "epoch": 0.03, + "learning_rate": 6.474820143884892e-05, + "loss": 0.352, + "step": 90 + }, + { + "epoch": 0.03, + "learning_rate": 6.83453237410072e-05, + "loss": 0.3534, + "step": 95 + }, + { + "epoch": 0.03, + "learning_rate": 7.194244604316547e-05, + "loss": 0.3432, + "step": 100 + }, + { + "epoch": 0.03, + "learning_rate": 7.553956834532374e-05, + "loss": 0.3286, + "step": 105 + }, + { + "epoch": 0.04, + "learning_rate": 7.913669064748202e-05, + "loss": 0.3298, + "step": 110 + }, + { + "epoch": 0.04, + "learning_rate": 8.273381294964029e-05, + "loss": 0.3223, + "step": 115 + }, + { + "epoch": 0.04, + "learning_rate": 8.633093525179857e-05, + "loss": 0.3181, + "step": 120 + }, + { + "epoch": 0.04, + "learning_rate": 8.992805755395684e-05, + "loss": 0.3225, + "step": 125 + }, + { + "epoch": 0.04, + "learning_rate": 9.35251798561151e-05, + "loss": 0.3145, + "step": 130 + }, + { + "epoch": 0.04, + "learning_rate": 9.712230215827339e-05, + "loss": 0.3137, + "step": 135 + }, + { + "epoch": 0.05, + "learning_rate": 0.00010071942446043166, + "loss": 0.3102, + "step": 140 + }, + { + "epoch": 0.05, + "learning_rate": 0.00010431654676258992, + "loss": 0.2951, + "step": 145 + }, + { + "epoch": 0.05, + "learning_rate": 0.0001079136690647482, + "loss": 0.3021, + "step": 150 + }, + { + "epoch": 0.05, + "learning_rate": 0.00011151079136690648, + "loss": 0.2923, + "step": 155 + }, + { + "epoch": 0.05, + "learning_rate": 0.00011510791366906474, + "loss": 0.2953, + "step": 160 + }, + { + "epoch": 0.05, + "learning_rate": 0.00011870503597122303, + "loss": 0.2893, + "step": 165 + }, + { + "epoch": 0.06, + "learning_rate": 0.0001223021582733813, + "loss": 0.2953, + "step": 170 + }, + { + "epoch": 0.06, + "learning_rate": 0.00012589928057553956, + "loss": 0.2931, + "step": 175 + }, + { + "epoch": 0.06, + "learning_rate": 0.00012949640287769783, + "loss": 0.2879, + "step": 180 + }, + { + "epoch": 0.06, + "learning_rate": 0.00013309352517985613, + "loss": 0.2642, + "step": 185 + }, + { + "epoch": 0.06, + "learning_rate": 0.0001366906474820144, + "loss": 0.2772, + "step": 190 + }, + { + "epoch": 0.06, + "learning_rate": 0.00014028776978417266, + "loss": 0.2858, + "step": 195 + }, + { + "epoch": 0.06, + "learning_rate": 0.00014388489208633093, + "loss": 0.2724, + "step": 200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0001474820143884892, + "loss": 0.2726, + "step": 205 + }, + { + "epoch": 0.07, + "learning_rate": 0.00015107913669064747, + "loss": 0.2678, + "step": 210 + }, + { + "epoch": 0.07, + "learning_rate": 0.00015467625899280577, + "loss": 0.2789, + "step": 215 + }, + { + "epoch": 0.07, + "learning_rate": 0.00015827338129496403, + "loss": 0.2742, + "step": 220 + }, + { + "epoch": 0.07, + "learning_rate": 0.0001618705035971223, + "loss": 0.2732, + "step": 225 + }, + { + "epoch": 0.07, + "learning_rate": 0.00016546762589928057, + "loss": 0.2741, + "step": 230 + }, + { + "epoch": 0.08, + "learning_rate": 0.00016906474820143884, + "loss": 0.2665, + "step": 235 + }, + { + "epoch": 0.08, + "learning_rate": 0.00017266187050359714, + "loss": 0.2771, + "step": 240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0001762589928057554, + "loss": 0.2751, + "step": 245 + }, + { + "epoch": 0.08, + "learning_rate": 0.00017985611510791367, + "loss": 0.2597, + "step": 250 + }, + { + "epoch": 0.08, + "learning_rate": 0.00018345323741007194, + "loss": 0.275, + "step": 255 + }, + { + "epoch": 0.08, + "learning_rate": 0.0001870503597122302, + "loss": 0.2791, + "step": 260 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019064748201438848, + "loss": 0.2493, + "step": 265 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019424460431654677, + "loss": 0.27, + "step": 270 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019784172661870504, + "loss": 0.2676, + "step": 275 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019999997550557367, + "loss": 0.2613, + "step": 280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019999969994341502, + "loss": 0.2747, + "step": 285 + }, + { + "epoch": 0.09, + "learning_rate": 0.00019999911820191125, + "loss": 0.2699, + "step": 290 + }, + { + "epoch": 0.1, + "learning_rate": 0.0001999982302828436, + "loss": 0.2692, + "step": 295 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999703618893072, + "loss": 0.2522, + "step": 300 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999553592382863, + "loss": 0.2541, + "step": 305 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999372949213085, + "loss": 0.2614, + "step": 310 + }, + { + "epoch": 0.1, + "learning_rate": 0.00019999161689936837, + "loss": 0.2481, + "step": 315 + }, + { + "epoch": 0.1, + "learning_rate": 0.0001999891981520095, + "loss": 0.2497, + "step": 320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019998647325745995, + "loss": 0.2545, + "step": 325 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019998344222406283, + "loss": 0.2501, + "step": 330 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019998010506109854, + "loss": 0.2577, + "step": 335 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019997646177878487, + "loss": 0.2585, + "step": 340 + }, + { + "epoch": 0.11, + "learning_rate": 0.0001999725123882768, + "loss": 0.2494, + "step": 345 + }, + { + "epoch": 0.11, + "learning_rate": 0.00019996825690166657, + "loss": 0.2538, + "step": 350 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019996369533198364, + "loss": 0.2469, + "step": 355 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019995882769319466, + "loss": 0.2502, + "step": 360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0001999536540002034, + "loss": 0.2361, + "step": 365 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019994817426885057, + "loss": 0.2446, + "step": 370 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019994238851591417, + "loss": 0.256, + "step": 375 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019993629675910896, + "loss": 0.2352, + "step": 380 + }, + { + "epoch": 0.12, + "learning_rate": 0.00019992989901708672, + "loss": 0.2538, + "step": 385 + }, + { + "epoch": 0.13, + "learning_rate": 0.0001999231953094361, + "loss": 0.2523, + "step": 390 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019991618565668243, + "loss": 0.2512, + "step": 395 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019990887008028798, + "loss": 0.2402, + "step": 400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019990124860265156, + "loss": 0.2423, + "step": 405 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019989332124710862, + "loss": 0.2335, + "step": 410 + }, + { + "epoch": 0.13, + "learning_rate": 0.00019988508803793124, + "loss": 0.2284, + "step": 415 + }, + { + "epoch": 0.14, + "learning_rate": 0.00019987654900032774, + "loss": 0.2382, + "step": 420 + }, + { + "epoch": 0.14, + "learning_rate": 0.0001998677041604431, + "loss": 0.2526, + "step": 425 + }, + { + "epoch": 0.14, + "learning_rate": 0.0001998585535453584, + "loss": 0.2417, + "step": 430 + }, + { + "epoch": 0.14, + "learning_rate": 0.00019984909718309105, + "loss": 0.2381, + "step": 435 + }, + { + "epoch": 0.14, + "learning_rate": 0.0001998393351025945, + "loss": 0.2418, + "step": 440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00019982926733375843, + "loss": 0.2409, + "step": 445 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001998188939074083, + "loss": 0.2467, + "step": 450 + }, + { + "epoch": 0.15, + "learning_rate": 0.00019980821485530544, + "loss": 0.2305, + "step": 455 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001997972302101471, + "loss": 0.232, + "step": 460 + }, + { + "epoch": 0.15, + "learning_rate": 0.00019978594000556602, + "loss": 0.225, + "step": 465 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001997743442761306, + "loss": 0.2259, + "step": 470 + }, + { + "epoch": 0.15, + "learning_rate": 0.0001997624430573447, + "loss": 0.2414, + "step": 475 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019975023638564747, + "loss": 0.238, + "step": 480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019973772429841336, + "loss": 0.224, + "step": 485 + }, + { + "epoch": 0.16, + "learning_rate": 0.0001997249068339519, + "loss": 0.2369, + "step": 490 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019971178403150762, + "loss": 0.2393, + "step": 495 + }, + { + "epoch": 0.16, + "learning_rate": 0.00019969835593125997, + "loss": 0.2249, + "step": 500 + }, + { + "epoch": 0.16, + "learning_rate": 0.0001996846225743231, + "loss": 0.2369, + "step": 505 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019967058400274592, + "loss": 0.2267, + "step": 510 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019965624025951168, + "loss": 0.2294, + "step": 515 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019964159138853815, + "loss": 0.2255, + "step": 520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019962663743467726, + "loss": 0.22, + "step": 525 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019961137844371507, + "loss": 0.2398, + "step": 530 + }, + { + "epoch": 0.17, + "learning_rate": 0.0001995958144623716, + "loss": 0.2431, + "step": 535 + }, + { + "epoch": 0.17, + "learning_rate": 0.00019957994553830067, + "loss": 0.2245, + "step": 540 + }, + { + "epoch": 0.18, + "learning_rate": 0.0001995637717200898, + "loss": 0.2295, + "step": 545 + }, + { + "epoch": 0.18, + "learning_rate": 0.00019954729305726008, + "loss": 0.2288, + "step": 550 + }, + { + "epoch": 0.18, + "learning_rate": 0.00019953050960026586, + "loss": 0.2333, + "step": 555 + }, + { + "epoch": 0.18, + "learning_rate": 0.0001995134214004948, + "loss": 0.2316, + "step": 560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0001994960285102676, + "loss": 0.2285, + "step": 565 + }, + { + "epoch": 0.18, + "learning_rate": 0.00019947833098283782, + "loss": 0.2273, + "step": 570 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019946032887239182, + "loss": 0.2344, + "step": 575 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019944202223404854, + "loss": 0.2251, + "step": 580 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019942341112385922, + "loss": 0.226, + "step": 585 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019940449559880752, + "loss": 0.2353, + "step": 590 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019938527571680892, + "loss": 0.226, + "step": 595 + }, + { + "epoch": 0.19, + "learning_rate": 0.00019936575153671097, + "loss": 0.221, + "step": 600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019934592311829285, + "loss": 0.2219, + "step": 605 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019932579052226528, + "loss": 0.234, + "step": 610 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019930535381027027, + "loss": 0.2181, + "step": 615 + }, + { + "epoch": 0.2, + "learning_rate": 0.000199284613044881, + "loss": 0.2191, + "step": 620 + }, + { + "epoch": 0.2, + "learning_rate": 0.0001992635682896016, + "loss": 0.235, + "step": 625 + }, + { + "epoch": 0.2, + "learning_rate": 0.00019924221960886698, + "loss": 0.2256, + "step": 630 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019922056706804257, + "loss": 0.2265, + "step": 635 + }, + { + "epoch": 0.21, + "learning_rate": 0.0001991986107334242, + "loss": 0.2158, + "step": 640 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019917635067223777, + "loss": 0.2238, + "step": 645 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019915378695263924, + "loss": 0.2182, + "step": 650 + }, + { + "epoch": 0.21, + "learning_rate": 0.00019913091964371423, + "loss": 0.2185, + "step": 655 + }, + { + "epoch": 0.21, + "learning_rate": 0.000199107748815478, + "loss": 0.2063, + "step": 660 + }, + { + "epoch": 0.22, + "learning_rate": 0.000199084274538875, + "loss": 0.2162, + "step": 665 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019906049688577887, + "loss": 0.2185, + "step": 670 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019903641592899205, + "loss": 0.2186, + "step": 675 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019901203174224564, + "loss": 0.2292, + "step": 680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019898734440019933, + "loss": 0.2162, + "step": 685 + }, + { + "epoch": 0.22, + "learning_rate": 0.00019896235397844078, + "loss": 0.219, + "step": 690 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019893706055348578, + "loss": 0.218, + "step": 695 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019891146420277777, + "loss": 0.2211, + "step": 700 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019888556500468775, + "loss": 0.2227, + "step": 705 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019885936303851395, + "loss": 0.2214, + "step": 710 + }, + { + "epoch": 0.23, + "learning_rate": 0.00019883285838448163, + "loss": 0.2204, + "step": 715 + }, + { + "epoch": 0.23, + "learning_rate": 0.0001988060511237428, + "loss": 0.2228, + "step": 720 + }, + { + "epoch": 0.23, + "learning_rate": 0.000198778941338376, + "loss": 0.2259, + "step": 725 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019875152911138602, + "loss": 0.2062, + "step": 730 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019872381452670372, + "loss": 0.2073, + "step": 735 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019869579766918568, + "loss": 0.2172, + "step": 740 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019866747862461396, + "loss": 0.21, + "step": 745 + }, + { + "epoch": 0.24, + "learning_rate": 0.00019863885747969598, + "loss": 0.2195, + "step": 750 + }, + { + "epoch": 0.24, + "learning_rate": 0.0001986099343220639, + "loss": 0.2129, + "step": 755 + }, + { + "epoch": 0.25, + "learning_rate": 0.0001985807092402748, + "loss": 0.215, + "step": 760 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019855118232381015, + "loss": 0.2085, + "step": 765 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019852135366307545, + "loss": 0.2196, + "step": 770 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019849122334940022, + "loss": 0.2129, + "step": 775 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019846079147503755, + "loss": 0.2221, + "step": 780 + }, + { + "epoch": 0.25, + "learning_rate": 0.00019843005813316377, + "loss": 0.2133, + "step": 785 + }, + { + "epoch": 0.26, + "learning_rate": 0.0001983990234178784, + "loss": 0.2066, + "step": 790 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019836768742420352, + "loss": 0.2188, + "step": 795 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019833605024808382, + "loss": 0.2128, + "step": 800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019830411198638607, + "loss": 0.2109, + "step": 805 + }, + { + "epoch": 0.26, + "learning_rate": 0.00019827187273689894, + "loss": 0.2141, + "step": 810 + }, + { + "epoch": 0.26, + "learning_rate": 0.0001982393325983326, + "loss": 0.2098, + "step": 815 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019820649167031856, + "loss": 0.2188, + "step": 820 + }, + { + "epoch": 0.27, + "learning_rate": 0.0001981733500534093, + "loss": 0.2107, + "step": 825 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019813990784907785, + "loss": 0.2162, + "step": 830 + }, + { + "epoch": 0.27, + "learning_rate": 0.0001981061651597177, + "loss": 0.2189, + "step": 835 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019807212208864227, + "loss": 0.2117, + "step": 840 + }, + { + "epoch": 0.27, + "learning_rate": 0.00019803777874008468, + "loss": 0.209, + "step": 845 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019800313521919753, + "loss": 0.206, + "step": 850 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019796819163205246, + "loss": 0.2104, + "step": 855 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019793294808563984, + "loss": 0.2016, + "step": 860 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019789740468786841, + "loss": 0.212, + "step": 865 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019786156154756506, + "loss": 0.2051, + "step": 870 + }, + { + "epoch": 0.28, + "learning_rate": 0.00019782541877447443, + "loss": 0.21, + "step": 875 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019778897647925855, + "loss": 0.2104, + "step": 880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0001977522347734965, + "loss": 0.2179, + "step": 885 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019771519376968416, + "loss": 0.2139, + "step": 890 + }, + { + "epoch": 0.29, + "learning_rate": 0.0001976778535812338, + "loss": 0.2034, + "step": 895 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019764021432247368, + "loss": 0.2067, + "step": 900 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019760227610864776, + "loss": 0.2189, + "step": 905 + }, + { + "epoch": 0.29, + "learning_rate": 0.00019756403905591538, + "loss": 0.2119, + "step": 910 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019752550328135086, + "loss": 0.2004, + "step": 915 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019748666890294315, + "loss": 0.2073, + "step": 920 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019744753603959545, + "loss": 0.1995, + "step": 925 + }, + { + "epoch": 0.3, + "learning_rate": 0.0001974081048111248, + "loss": 0.2034, + "step": 930 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019736837533826192, + "loss": 0.2015, + "step": 935 + }, + { + "epoch": 0.3, + "learning_rate": 0.00019732834774265057, + "loss": 0.2166, + "step": 940 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019728802214684734, + "loss": 0.1987, + "step": 945 + }, + { + "epoch": 0.31, + "learning_rate": 0.0001972473986743213, + "loss": 0.2051, + "step": 950 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019720647744945344, + "loss": 0.2055, + "step": 955 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019716525859753652, + "loss": 0.2005, + "step": 960 + }, + { + "epoch": 0.31, + "learning_rate": 0.00019712374224477452, + "loss": 0.1995, + "step": 965 + }, + { + "epoch": 0.31, + "learning_rate": 0.0001970819285182823, + "loss": 0.2096, + "step": 970 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019703981754608525, + "loss": 0.2024, + "step": 975 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019699740945711884, + "loss": 0.2006, + "step": 980 + }, + { + "epoch": 0.32, + "learning_rate": 0.0001969547043812283, + "loss": 0.2077, + "step": 985 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019691170244916814, + "loss": 0.2094, + "step": 990 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019686840379260176, + "loss": 0.2051, + "step": 995 + }, + { + "epoch": 0.32, + "learning_rate": 0.00019682480854410115, + "loss": 0.2158, + "step": 1000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0001967809168371463, + "loss": 0.2035, + "step": 1005 + }, + { + "epoch": 0.33, + "learning_rate": 0.000196736728806125, + "loss": 0.2104, + "step": 1010 + }, + { + "epoch": 0.33, + "learning_rate": 0.00019669224458633222, + "loss": 0.1969, + "step": 1015 + }, + { + "epoch": 0.33, + "learning_rate": 0.0001966474643139699, + "loss": 0.2058, + "step": 1020 + }, + { + "epoch": 0.33, + "learning_rate": 0.00019660238812614647, + "loss": 0.2045, + "step": 1025 + }, + { + "epoch": 0.33, + "learning_rate": 0.0001965570161608762, + "loss": 0.1916, + "step": 1030 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019651134855707913, + "loss": 0.2008, + "step": 1035 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019646538545458047, + "loss": 0.2095, + "step": 1040 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019641912699411016, + "loss": 0.2009, + "step": 1045 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019637257331730248, + "loss": 0.2059, + "step": 1050 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019632572456669558, + "loss": 0.2084, + "step": 1055 + }, + { + "epoch": 0.34, + "learning_rate": 0.00019627858088573112, + "loss": 0.1997, + "step": 1060 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001962311424187537, + "loss": 0.2068, + "step": 1065 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001961834093110106, + "loss": 0.1978, + "step": 1070 + }, + { + "epoch": 0.35, + "learning_rate": 0.00019613538170865118, + "loss": 0.2028, + "step": 1075 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001960870597587264, + "loss": 0.1996, + "step": 1080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0001960384436091886, + "loss": 0.2002, + "step": 1085 + }, + { + "epoch": 0.35, + "learning_rate": 0.00019598953340889082, + "loss": 0.198, + "step": 1090 + }, + { + "epoch": 0.35, + "learning_rate": 0.00019594032930758643, + "loss": 0.1974, + "step": 1095 + }, + { + "epoch": 0.36, + "learning_rate": 0.0001958908314559287, + "loss": 0.1983, + "step": 1100 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019584104000547025, + "loss": 0.1987, + "step": 1105 + }, + { + "epoch": 0.36, + "learning_rate": 0.0001957909551086627, + "loss": 0.2003, + "step": 1110 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019574057691885613, + "loss": 0.1982, + "step": 1115 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019568990559029858, + "loss": 0.2031, + "step": 1120 + }, + { + "epoch": 0.36, + "learning_rate": 0.00019563894127813564, + "loss": 0.1969, + "step": 1125 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019558768413841006, + "loss": 0.1916, + "step": 1130 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019553613432806102, + "loss": 0.2124, + "step": 1135 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019548429200492388, + "loss": 0.2042, + "step": 1140 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019543215732772956, + "loss": 0.201, + "step": 1145 + }, + { + "epoch": 0.37, + "learning_rate": 0.00019537973045610424, + "loss": 0.2001, + "step": 1150 + }, + { + "epoch": 0.37, + "learning_rate": 0.0001953270115505686, + "loss": 0.1978, + "step": 1155 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019527400077253755, + "loss": 0.192, + "step": 1160 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019522069828431957, + "loss": 0.1946, + "step": 1165 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019516710424911642, + "loss": 0.1848, + "step": 1170 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019511321883102244, + "loss": 0.2005, + "step": 1175 + }, + { + "epoch": 0.38, + "learning_rate": 0.00019505904219502414, + "loss": 0.1966, + "step": 1180 + }, + { + "epoch": 0.38, + "learning_rate": 0.0001950045745069997, + "loss": 0.1863, + "step": 1185 + }, + { + "epoch": 0.39, + "learning_rate": 0.0001949498159337184, + "loss": 0.1983, + "step": 1190 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019489476664284023, + "loss": 0.2113, + "step": 1195 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019483942680291523, + "loss": 0.1956, + "step": 1200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019478379658338305, + "loss": 0.2055, + "step": 1205 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019472787615457246, + "loss": 0.1879, + "step": 1210 + }, + { + "epoch": 0.39, + "learning_rate": 0.00019467166568770078, + "loss": 0.1924, + "step": 1215 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019461516535487338, + "loss": 0.1971, + "step": 1220 + }, + { + "epoch": 0.4, + "learning_rate": 0.0001945583753290831, + "loss": 0.1952, + "step": 1225 + }, + { + "epoch": 0.4, + "learning_rate": 0.0001945012957842098, + "loss": 0.1838, + "step": 1230 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019444392689501978, + "loss": 0.1931, + "step": 1235 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019438626883716526, + "loss": 0.2093, + "step": 1240 + }, + { + "epoch": 0.4, + "learning_rate": 0.00019432832178718385, + "loss": 0.1992, + "step": 1245 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019427008592249795, + "loss": 0.1963, + "step": 1250 + }, + { + "epoch": 0.41, + "learning_rate": 0.0001942115614214143, + "loss": 0.2003, + "step": 1255 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019415274846312338, + "loss": 0.1914, + "step": 1260 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019409364722769882, + "loss": 0.1917, + "step": 1265 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019403425789609697, + "loss": 0.2033, + "step": 1270 + }, + { + "epoch": 0.41, + "learning_rate": 0.0001939745806501563, + "loss": 0.1975, + "step": 1275 + }, + { + "epoch": 0.41, + "learning_rate": 0.00019391461567259662, + "loss": 0.1922, + "step": 1280 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019385436314701897, + "loss": 0.1999, + "step": 1285 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019379382325790468, + "loss": 0.1997, + "step": 1290 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019373299619061487, + "loss": 0.1872, + "step": 1295 + }, + { + "epoch": 0.42, + "learning_rate": 0.0001936718821313901, + "loss": 0.1936, + "step": 1300 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019361048126734952, + "loss": 0.1798, + "step": 1305 + }, + { + "epoch": 0.42, + "learning_rate": 0.00019354879378649046, + "loss": 0.1959, + "step": 1310 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019348681987768783, + "loss": 0.1924, + "step": 1315 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019342455973069345, + "loss": 0.1967, + "step": 1320 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019336201353613568, + "loss": 0.1925, + "step": 1325 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019329918148551856, + "loss": 0.1878, + "step": 1330 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019323606377122146, + "loss": 0.1979, + "step": 1335 + }, + { + "epoch": 0.43, + "learning_rate": 0.00019317266058649834, + "loss": 0.189, + "step": 1340 + }, + { + "epoch": 0.44, + "learning_rate": 0.0001931089721254773, + "loss": 0.1995, + "step": 1345 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019304499858315976, + "loss": 0.1874, + "step": 1350 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019298074015542015, + "loss": 0.1991, + "step": 1355 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019291619703900502, + "loss": 0.188, + "step": 1360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0001928513694315327, + "loss": 0.1914, + "step": 1365 + }, + { + "epoch": 0.44, + "learning_rate": 0.00019278625753149254, + "loss": 0.1843, + "step": 1370 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019272086153824427, + "loss": 0.1959, + "step": 1375 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019265518165201758, + "loss": 0.1867, + "step": 1380 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019258921807391131, + "loss": 0.1956, + "step": 1385 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019252297100589286, + "loss": 0.1991, + "step": 1390 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019245644065079775, + "loss": 0.1934, + "step": 1395 + }, + { + "epoch": 0.45, + "learning_rate": 0.00019238962721232876, + "loss": 0.1953, + "step": 1400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019232253089505546, + "loss": 0.1865, + "step": 1405 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019225515190441354, + "loss": 0.1864, + "step": 1410 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019218749044670413, + "loss": 0.194, + "step": 1415 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019211954672909327, + "loss": 0.1853, + "step": 1420 + }, + { + "epoch": 0.46, + "learning_rate": 0.00019205132095961126, + "loss": 0.1967, + "step": 1425 + }, + { + "epoch": 0.46, + "learning_rate": 0.0001919828133471519, + "loss": 0.1852, + "step": 1430 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019191402410147204, + "loss": 0.1913, + "step": 1435 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019184495343319068, + "loss": 0.1911, + "step": 1440 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019177560155378865, + "loss": 0.189, + "step": 1445 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019170596867560766, + "loss": 0.1854, + "step": 1450 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019163605501184986, + "loss": 0.1974, + "step": 1455 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019156586077657712, + "loss": 0.1933, + "step": 1460 + }, + { + "epoch": 0.47, + "learning_rate": 0.00019149538618471032, + "loss": 0.1877, + "step": 1465 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001914246314520287, + "loss": 0.1904, + "step": 1470 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001913535967951694, + "loss": 0.1841, + "step": 1475 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001912822824316264, + "loss": 0.1876, + "step": 1480 + }, + { + "epoch": 0.48, + "learning_rate": 0.00019121068857975027, + "loss": 0.1858, + "step": 1485 + }, + { + "epoch": 0.48, + "learning_rate": 0.00019113881545874722, + "loss": 0.1898, + "step": 1490 + }, + { + "epoch": 0.48, + "learning_rate": 0.0001910666632886786, + "loss": 0.1923, + "step": 1495 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019099423229046014, + "loss": 0.1803, + "step": 1500 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019092152268586118, + "loss": 0.1945, + "step": 1505 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019084853469750424, + "loss": 0.1802, + "step": 1510 + }, + { + "epoch": 0.49, + "learning_rate": 0.0001907752685488641, + "loss": 0.197, + "step": 1515 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019070172446426724, + "loss": 0.1791, + "step": 1520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00019062790266889117, + "loss": 0.1874, + "step": 1525 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019055380338876363, + "loss": 0.1806, + "step": 1530 + }, + { + "epoch": 0.5, + "learning_rate": 0.000190479426850762, + "loss": 0.1834, + "step": 1535 + }, + { + "epoch": 0.5, + "learning_rate": 0.0001904047732826126, + "loss": 0.1921, + "step": 1540 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019032984291288986, + "loss": 0.1826, + "step": 1545 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019025463597101584, + "loss": 0.1966, + "step": 1550 + }, + { + "epoch": 0.5, + "learning_rate": 0.00019017915268725925, + "loss": 0.1867, + "step": 1555 + }, + { + "epoch": 0.51, + "learning_rate": 0.00019010339329273514, + "loss": 0.196, + "step": 1560 + }, + { + "epoch": 0.51, + "learning_rate": 0.00019002735801940373, + "loss": 0.1971, + "step": 1565 + }, + { + "epoch": 0.51, + "learning_rate": 0.00018995104710007004, + "loss": 0.187, + "step": 1570 + }, + { + "epoch": 0.51, + "learning_rate": 0.00018987446076838312, + "loss": 0.1897, + "step": 1575 + }, + { + "epoch": 0.51, + "learning_rate": 0.00018979759925883509, + "loss": 0.1887, + "step": 1580 + }, + { + "epoch": 0.51, + "learning_rate": 0.0001897204628067608, + "loss": 0.1849, + "step": 1585 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018964305164833678, + "loss": 0.1795, + "step": 1590 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018956536602058078, + "loss": 0.1984, + "step": 1595 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018948740616135085, + "loss": 0.1876, + "step": 1600 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018940917230934464, + "loss": 0.1879, + "step": 1605 + }, + { + "epoch": 0.52, + "learning_rate": 0.0001893306647040988, + "loss": 0.1906, + "step": 1610 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018925188358598813, + "loss": 0.1917, + "step": 1615 + }, + { + "epoch": 0.52, + "learning_rate": 0.00018917282919622488, + "loss": 0.1912, + "step": 1620 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018909350177685795, + "loss": 0.1813, + "step": 1625 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018901390157077225, + "loss": 0.183, + "step": 1630 + }, + { + "epoch": 0.53, + "learning_rate": 0.0001889340288216879, + "loss": 0.1782, + "step": 1635 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018885388377415957, + "loss": 0.1866, + "step": 1640 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018877346667357548, + "loss": 0.1915, + "step": 1645 + }, + { + "epoch": 0.53, + "learning_rate": 0.00018869277776615691, + "loss": 0.1902, + "step": 1650 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018861181729895745, + "loss": 0.1829, + "step": 1655 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018853058551986202, + "loss": 0.1855, + "step": 1660 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018844908267758633, + "loss": 0.1845, + "step": 1665 + }, + { + "epoch": 0.54, + "learning_rate": 0.000188367309021676, + "loss": 0.1794, + "step": 1670 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018828526480250575, + "loss": 0.1796, + "step": 1675 + }, + { + "epoch": 0.54, + "learning_rate": 0.00018820295027127885, + "loss": 0.1806, + "step": 1680 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018812036568002617, + "loss": 0.1846, + "step": 1685 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018803751128160538, + "loss": 0.189, + "step": 1690 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018795438732970032, + "loss": 0.1856, + "step": 1695 + }, + { + "epoch": 0.55, + "learning_rate": 0.0001878709940788201, + "loss": 0.1797, + "step": 1700 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018778733178429844, + "loss": 0.1876, + "step": 1705 + }, + { + "epoch": 0.55, + "learning_rate": 0.00018770340070229272, + "loss": 0.1775, + "step": 1710 + }, + { + "epoch": 0.56, + "learning_rate": 0.0001876192010897834, + "loss": 0.1862, + "step": 1715 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018753473320457306, + "loss": 0.1835, + "step": 1720 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018744999730528568, + "loss": 0.1909, + "step": 1725 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018736499365136584, + "loss": 0.189, + "step": 1730 + }, + { + "epoch": 0.56, + "learning_rate": 0.000187279722503078, + "loss": 0.1861, + "step": 1735 + }, + { + "epoch": 0.56, + "learning_rate": 0.00018719418412150557, + "loss": 0.1809, + "step": 1740 + }, + { + "epoch": 0.57, + "learning_rate": 0.00018710837876855017, + "loss": 0.1829, + "step": 1745 + }, + { + "epoch": 0.57, + "learning_rate": 0.0001870223067069309, + "loss": 0.1886, + "step": 1750 + }, + { + "epoch": 0.57, + "learning_rate": 0.00018693596820018336, + "loss": 0.1787, + "step": 1755 + }, + { + "epoch": 0.57, + "learning_rate": 0.0001868493635126591, + "loss": 0.1816, + "step": 1760 + }, + { + "epoch": 0.57, + "learning_rate": 0.0001867624929095245, + "loss": 0.1759, + "step": 1765 + }, + { + "epoch": 0.57, + "learning_rate": 0.00018667535665676025, + "loss": 0.1702, + "step": 1770 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018658795502116035, + "loss": 0.1802, + "step": 1775 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018650028827033136, + "loss": 0.1836, + "step": 1780 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018641235667269155, + "loss": 0.1787, + "step": 1785 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018632416049747016, + "loss": 0.1811, + "step": 1790 + }, + { + "epoch": 0.58, + "learning_rate": 0.0001862357000147064, + "loss": 0.1835, + "step": 1795 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018614697549524889, + "loss": 0.1857, + "step": 1800 + }, + { + "epoch": 0.58, + "learning_rate": 0.00018605798721075455, + "loss": 0.1867, + "step": 1805 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018596873543368797, + "loss": 0.1766, + "step": 1810 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018587922043732046, + "loss": 0.1895, + "step": 1815 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018578944249572925, + "loss": 0.1888, + "step": 1820 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018569940188379672, + "loss": 0.1871, + "step": 1825 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018560909887720944, + "loss": 0.1804, + "step": 1830 + }, + { + "epoch": 0.59, + "learning_rate": 0.00018551853375245737, + "loss": 0.1877, + "step": 1835 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018542770678683308, + "loss": 0.182, + "step": 1840 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001853366182584308, + "loss": 0.1809, + "step": 1845 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018524526844614564, + "loss": 0.1913, + "step": 1850 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018515365762967266, + "loss": 0.185, + "step": 1855 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018506178608950613, + "loss": 0.1753, + "step": 1860 + }, + { + "epoch": 0.6, + "learning_rate": 0.00018496965410693856, + "loss": 0.1786, + "step": 1865 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001848772619640599, + "loss": 0.1841, + "step": 1870 + }, + { + "epoch": 0.61, + "learning_rate": 0.00018478460994375667, + "loss": 0.184, + "step": 1875 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001846916983297111, + "loss": 0.179, + "step": 1880 + }, + { + "epoch": 0.61, + "learning_rate": 0.00018459852740640014, + "loss": 0.1833, + "step": 1885 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001845050974590948, + "loss": 0.186, + "step": 1890 + }, + { + "epoch": 0.61, + "learning_rate": 0.00018441140877385913, + "loss": 0.1778, + "step": 1895 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018431746163754943, + "loss": 0.1786, + "step": 1900 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018422325633781327, + "loss": 0.1826, + "step": 1905 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018412879316308866, + "loss": 0.1738, + "step": 1910 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018403407240260327, + "loss": 0.1852, + "step": 1915 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018393909434637329, + "loss": 0.1821, + "step": 1920 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001838438592852028, + "loss": 0.1773, + "step": 1925 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018374836751068282, + "loss": 0.1772, + "step": 1930 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018365261931519022, + "loss": 0.1766, + "step": 1935 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018355661499188717, + "loss": 0.1858, + "step": 1940 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018346035483471994, + "loss": 0.1786, + "step": 1945 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001833638391384181, + "loss": 0.1745, + "step": 1950 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018326706819849365, + "loss": 0.1713, + "step": 1955 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018317004231124017, + "loss": 0.1738, + "step": 1960 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018307276177373174, + "loss": 0.1807, + "step": 1965 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018297522688382218, + "loss": 0.1806, + "step": 1970 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018287743794014407, + "loss": 0.1735, + "step": 1975 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018277939524210783, + "loss": 0.1812, + "step": 1980 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001826810990899009, + "loss": 0.1734, + "step": 1985 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001825825497844867, + "loss": 0.1703, + "step": 1990 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018248374762760365, + "loss": 0.1823, + "step": 1995 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018238469292176457, + "loss": 0.1837, + "step": 2000 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018228538597025537, + "loss": 0.1807, + "step": 2005 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018218582707713434, + "loss": 0.1904, + "step": 2010 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001820860165472312, + "loss": 0.1844, + "step": 2015 + }, + { + "epoch": 0.65, + "learning_rate": 0.00018198595468614604, + "loss": 0.1792, + "step": 2020 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018188564180024854, + "loss": 0.1845, + "step": 2025 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018178507819667698, + "loss": 0.1853, + "step": 2030 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018168426418333725, + "loss": 0.1875, + "step": 2035 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018158320006890198, + "loss": 0.1759, + "step": 2040 + }, + { + "epoch": 0.66, + "learning_rate": 0.00018148188616280955, + "loss": 0.1816, + "step": 2045 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001813803227752631, + "loss": 0.1787, + "step": 2050 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001812785102172297, + "loss": 0.1758, + "step": 2055 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018117644880043932, + "loss": 0.1802, + "step": 2060 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018107413883738387, + "loss": 0.186, + "step": 2065 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018097158064131626, + "loss": 0.1743, + "step": 2070 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018086877452624944, + "loss": 0.1877, + "step": 2075 + }, + { + "epoch": 0.67, + "learning_rate": 0.00018076572080695543, + "loss": 0.1742, + "step": 2080 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018066241979896444, + "loss": 0.1727, + "step": 2085 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018055887181856375, + "loss": 0.1806, + "step": 2090 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018045507718279677, + "loss": 0.175, + "step": 2095 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018035103620946225, + "loss": 0.1727, + "step": 2100 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001802467492171131, + "loss": 0.1779, + "step": 2105 + }, + { + "epoch": 0.68, + "learning_rate": 0.00018014221652505556, + "loss": 0.1823, + "step": 2110 + }, + { + "epoch": 0.69, + "learning_rate": 0.000180037438453348, + "loss": 0.1814, + "step": 2115 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001799324153228003, + "loss": 0.1707, + "step": 2120 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001798271474549725, + "loss": 0.1778, + "step": 2125 + }, + { + "epoch": 0.69, + "learning_rate": 0.000179721635172174, + "loss": 0.1888, + "step": 2130 + }, + { + "epoch": 0.69, + "learning_rate": 0.00017961587879746266, + "loss": 0.1785, + "step": 2135 + }, + { + "epoch": 0.69, + "learning_rate": 0.00017950987865464365, + "loss": 0.1661, + "step": 2140 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017940363506826836, + "loss": 0.1792, + "step": 2145 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001792971483636338, + "loss": 0.1732, + "step": 2150 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017919041886678118, + "loss": 0.1771, + "step": 2155 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001790834469044952, + "loss": 0.173, + "step": 2160 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017897623280430284, + "loss": 0.1768, + "step": 2165 + }, + { + "epoch": 0.7, + "learning_rate": 0.00017886877689447258, + "loss": 0.175, + "step": 2170 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001787610795040132, + "loss": 0.1691, + "step": 2175 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001786531409626728, + "loss": 0.1774, + "step": 2180 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001785449616009379, + "loss": 0.1709, + "step": 2185 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001784365417500324, + "loss": 0.1809, + "step": 2190 + }, + { + "epoch": 0.71, + "learning_rate": 0.00017832788174191645, + "loss": 0.1764, + "step": 2195 + }, + { + "epoch": 0.71, + "learning_rate": 0.00017821898190928558, + "loss": 0.1739, + "step": 2200 + }, + { + "epoch": 0.71, + "learning_rate": 0.00017810984258556957, + "loss": 0.1772, + "step": 2205 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017800046410493145, + "loss": 0.1779, + "step": 2210 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001778908468022666, + "loss": 0.1828, + "step": 2215 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017778099101320156, + "loss": 0.1722, + "step": 2220 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017767089707409304, + "loss": 0.1737, + "step": 2225 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017756056532202696, + "loss": 0.1688, + "step": 2230 + }, + { + "epoch": 0.72, + "learning_rate": 0.00017744999609481738, + "loss": 0.1784, + "step": 2235 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017733918973100545, + "loss": 0.1747, + "step": 2240 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017722814656985841, + "loss": 0.1667, + "step": 2245 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001771168669513685, + "loss": 0.1648, + "step": 2250 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017700535121625187, + "loss": 0.1777, + "step": 2255 + }, + { + "epoch": 0.73, + "learning_rate": 0.00017689359970594779, + "loss": 0.175, + "step": 2260 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001767816127626173, + "loss": 0.1715, + "step": 2265 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001766693907291423, + "loss": 0.1742, + "step": 2270 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017655693394912452, + "loss": 0.1868, + "step": 2275 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017644424276688441, + "loss": 0.1757, + "step": 2280 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017633131752746017, + "loss": 0.1803, + "step": 2285 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017621815857660656, + "loss": 0.1812, + "step": 2290 + }, + { + "epoch": 0.74, + "learning_rate": 0.00017610476626079395, + "loss": 0.1691, + "step": 2295 + }, + { + "epoch": 0.75, + "learning_rate": 0.00017599114092720726, + "loss": 0.1662, + "step": 2300 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001758772829237448, + "loss": 0.1709, + "step": 2305 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001757631925990174, + "loss": 0.1822, + "step": 2310 + }, + { + "epoch": 0.75, + "learning_rate": 0.00017564887030234696, + "loss": 0.1732, + "step": 2315 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001755343163837659, + "loss": 0.1854, + "step": 2320 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001754195311940157, + "loss": 0.1768, + "step": 2325 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017530451508454594, + "loss": 0.1785, + "step": 2330 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017518926840751325, + "loss": 0.1724, + "step": 2335 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017507379151578017, + "loss": 0.1812, + "step": 2340 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017495808476291423, + "loss": 0.1754, + "step": 2345 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017484214850318664, + "loss": 0.1799, + "step": 2350 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001747259830915713, + "loss": 0.1769, + "step": 2355 + }, + { + "epoch": 0.76, + "learning_rate": 0.00017460958888374385, + "loss": 0.1764, + "step": 2360 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001744929662360803, + "loss": 0.1716, + "step": 2365 + }, + { + "epoch": 0.77, + "learning_rate": 0.00017437611550565624, + "loss": 0.1699, + "step": 2370 + }, + { + "epoch": 0.77, + "learning_rate": 0.00017425903705024553, + "loss": 0.1661, + "step": 2375 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001741417312283193, + "loss": 0.1699, + "step": 2380 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001740241983990448, + "loss": 0.1682, + "step": 2385 + }, + { + "epoch": 0.77, + "learning_rate": 0.00017390643892228438, + "loss": 0.173, + "step": 2390 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001737884531585943, + "loss": 0.1852, + "step": 2395 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017367024146922372, + "loss": 0.1681, + "step": 2400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017355180421611347, + "loss": 0.1798, + "step": 2405 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017343314176189502, + "loss": 0.1705, + "step": 2410 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017331425446988946, + "loss": 0.177, + "step": 2415 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017319514270410614, + "loss": 0.1754, + "step": 2420 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017307580682924178, + "loss": 0.1679, + "step": 2425 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017295624721067932, + "loss": 0.1763, + "step": 2430 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017283646421448673, + "loss": 0.1769, + "step": 2435 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017271645820741583, + "loss": 0.1753, + "step": 2440 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001725962295569014, + "loss": 0.1715, + "step": 2445 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017247577863105983, + "loss": 0.1784, + "step": 2450 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017235510579868802, + "loss": 0.17, + "step": 2455 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017223421142926245, + "loss": 0.1686, + "step": 2460 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017211309589293774, + "loss": 0.166, + "step": 2465 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017199175956054584, + "loss": 0.1652, + "step": 2470 + }, + { + "epoch": 0.8, + "learning_rate": 0.00017187020280359455, + "loss": 0.1732, + "step": 2475 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001717484259942668, + "loss": 0.1754, + "step": 2480 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017162642950541902, + "loss": 0.1748, + "step": 2485 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017150421371058049, + "loss": 0.1733, + "step": 2490 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017138177898395177, + "loss": 0.1737, + "step": 2495 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017125912570040383, + "loss": 0.175, + "step": 2500 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001711362542354769, + "loss": 0.173, + "step": 2505 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017101316496537905, + "loss": 0.1796, + "step": 2510 + }, + { + "epoch": 0.81, + "learning_rate": 0.00017088985826698542, + "loss": 0.1722, + "step": 2515 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017076633451783675, + "loss": 0.1738, + "step": 2520 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017064259409613834, + "loss": 0.1743, + "step": 2525 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017051863738075902, + "loss": 0.1653, + "step": 2530 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001703944647512297, + "loss": 0.1761, + "step": 2535 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017027007658774256, + "loss": 0.1649, + "step": 2540 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017014547327114956, + "loss": 0.1738, + "step": 2545 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001700206551829614, + "loss": 0.1585, + "step": 2550 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016989562270534655, + "loss": 0.183, + "step": 2555 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001697703762211297, + "loss": 0.1693, + "step": 2560 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001696449161137909, + "loss": 0.173, + "step": 2565 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016951924276746425, + "loss": 0.1691, + "step": 2570 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016939335656693667, + "loss": 0.1698, + "step": 2575 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016926725789764691, + "loss": 0.1706, + "step": 2580 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016914094714568417, + "loss": 0.1691, + "step": 2585 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016901442469778713, + "loss": 0.1665, + "step": 2590 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016888769094134247, + "loss": 0.1831, + "step": 2595 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016876074626438395, + "loss": 0.1637, + "step": 2600 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016863359105559117, + "loss": 0.1757, + "step": 2605 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016850622570428826, + "loss": 0.1721, + "step": 2610 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016837865060044272, + "loss": 0.1647, + "step": 2615 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016825086613466442, + "loss": 0.1734, + "step": 2620 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016812287269820416, + "loss": 0.1847, + "step": 2625 + }, + { + "epoch": 0.85, + "learning_rate": 0.00016799467068295257, + "loss": 0.1682, + "step": 2630 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001678662604814389, + "loss": 0.1741, + "step": 2635 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016773764248682988, + "loss": 0.1692, + "step": 2640 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016760881709292842, + "loss": 0.1805, + "step": 2645 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001674797846941724, + "loss": 0.1685, + "step": 2650 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016735054568563364, + "loss": 0.1673, + "step": 2655 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001672211004630164, + "loss": 0.1686, + "step": 2660 + }, + { + "epoch": 0.86, + "learning_rate": 0.00016709144942265644, + "loss": 0.1749, + "step": 2665 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001669615929615197, + "loss": 0.1644, + "step": 2670 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016683153147720097, + "loss": 0.1673, + "step": 2675 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016670126536792287, + "loss": 0.1722, + "step": 2680 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001665707950325345, + "loss": 0.1588, + "step": 2685 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001664401208705103, + "loss": 0.164, + "step": 2690 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016630924328194877, + "loss": 0.1708, + "step": 2695 + }, + { + "epoch": 0.87, + "learning_rate": 0.00016617816266757123, + "loss": 0.1711, + "step": 2700 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016604687942872067, + "loss": 0.1701, + "step": 2705 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016591539396736046, + "loss": 0.1708, + "step": 2710 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016578370668607307, + "loss": 0.1669, + "step": 2715 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016565181798805907, + "loss": 0.1601, + "step": 2720 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016551972827713555, + "loss": 0.1733, + "step": 2725 + }, + { + "epoch": 0.88, + "learning_rate": 0.00016538743795773515, + "loss": 0.1641, + "step": 2730 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016525494743490474, + "loss": 0.1723, + "step": 2735 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016512225711430415, + "loss": 0.1655, + "step": 2740 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016498936740220492, + "loss": 0.172, + "step": 2745 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016485627870548917, + "loss": 0.1686, + "step": 2750 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016472299143164816, + "loss": 0.1709, + "step": 2755 + }, + { + "epoch": 0.89, + "learning_rate": 0.00016458950598878126, + "loss": 0.1648, + "step": 2760 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016445582278559458, + "loss": 0.1641, + "step": 2765 + }, + { + "epoch": 0.9, + "learning_rate": 0.0001643219422313996, + "loss": 0.1662, + "step": 2770 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016418786473611227, + "loss": 0.1763, + "step": 2775 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016405359071025138, + "loss": 0.1773, + "step": 2780 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016391912056493743, + "loss": 0.1678, + "step": 2785 + }, + { + "epoch": 0.9, + "learning_rate": 0.00016378445471189156, + "loss": 0.1632, + "step": 2790 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016364959356343398, + "loss": 0.1671, + "step": 2795 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016351453753248296, + "loss": 0.1739, + "step": 2800 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016337928703255336, + "loss": 0.1727, + "step": 2805 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016324384247775554, + "loss": 0.1756, + "step": 2810 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016310820428279397, + "loss": 0.1671, + "step": 2815 + }, + { + "epoch": 0.91, + "learning_rate": 0.00016297237286296606, + "loss": 0.169, + "step": 2820 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016283634863416082, + "loss": 0.1693, + "step": 2825 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016270013201285757, + "loss": 0.164, + "step": 2830 + }, + { + "epoch": 0.92, + "learning_rate": 0.0001625637234161247, + "loss": 0.1663, + "step": 2835 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016242712326161845, + "loss": 0.1742, + "step": 2840 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016229033196758154, + "loss": 0.169, + "step": 2845 + }, + { + "epoch": 0.92, + "learning_rate": 0.00016215334995284188, + "loss": 0.1703, + "step": 2850 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001620161776368114, + "loss": 0.1669, + "step": 2855 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001618788154394846, + "loss": 0.1634, + "step": 2860 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001617412637814375, + "loss": 0.163, + "step": 2865 + }, + { + "epoch": 0.93, + "learning_rate": 0.00016160352308382608, + "loss": 0.1651, + "step": 2870 + }, + { + "epoch": 0.93, + "learning_rate": 0.00016146559376838514, + "loss": 0.1657, + "step": 2875 + }, + { + "epoch": 0.93, + "learning_rate": 0.0001613274762574271, + "loss": 0.1638, + "step": 2880 + }, + { + "epoch": 0.93, + "learning_rate": 0.00016118917097384048, + "loss": 0.1691, + "step": 2885 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016105067834108874, + "loss": 0.1627, + "step": 2890 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016091199878320902, + "loss": 0.1608, + "step": 2895 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016077313272481076, + "loss": 0.1617, + "step": 2900 + }, + { + "epoch": 0.94, + "learning_rate": 0.0001606340805910744, + "loss": 0.1696, + "step": 2905 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016049484280775012, + "loss": 0.1604, + "step": 2910 + }, + { + "epoch": 0.94, + "learning_rate": 0.00016035541980115653, + "loss": 0.161, + "step": 2915 + }, + { + "epoch": 0.95, + "learning_rate": 0.00016021581199817943, + "loss": 0.1703, + "step": 2920 + }, + { + "epoch": 0.95, + "learning_rate": 0.0001600760198262703, + "loss": 0.1671, + "step": 2925 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015993604371344517, + "loss": 0.1603, + "step": 2930 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015979588408828333, + "loss": 0.1655, + "step": 2935 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015965554137992584, + "loss": 0.1675, + "step": 2940 + }, + { + "epoch": 0.95, + "learning_rate": 0.00015951501601807437, + "loss": 0.1641, + "step": 2945 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015937430843298986, + "loss": 0.165, + "step": 2950 + }, + { + "epoch": 0.96, + "learning_rate": 0.0001592334190554912, + "loss": 0.1758, + "step": 2955 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015909234831695379, + "loss": 0.1703, + "step": 2960 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015895109664930846, + "loss": 0.1644, + "step": 2965 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015880966448503987, + "loss": 0.1638, + "step": 2970 + }, + { + "epoch": 0.96, + "learning_rate": 0.00015866805225718542, + "loss": 0.1659, + "step": 2975 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015852626039933383, + "loss": 0.1676, + "step": 2980 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015838428934562373, + "loss": 0.1794, + "step": 2985 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015824213953074254, + "loss": 0.1562, + "step": 2990 + }, + { + "epoch": 0.97, + "learning_rate": 0.0001580998113899249, + "loss": 0.171, + "step": 2995 + }, + { + "epoch": 0.97, + "learning_rate": 0.00015795730535895147, + "loss": 0.1681, + "step": 3000 + }, + { + "epoch": 0.97, + "learning_rate": 0.0001578146218741476, + "loss": 0.1617, + "step": 3005 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015767176137238196, + "loss": 0.1708, + "step": 3010 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015752872429106523, + "loss": 0.1636, + "step": 3015 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015738551106814872, + "loss": 0.1678, + "step": 3020 + }, + { + "epoch": 0.98, + "learning_rate": 0.000157242122142123, + "loss": 0.1612, + "step": 3025 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015709855795201675, + "loss": 0.1658, + "step": 3030 + }, + { + "epoch": 0.98, + "learning_rate": 0.00015695481893739514, + "loss": 0.1621, + "step": 3035 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015681090553835864, + "loss": 0.1718, + "step": 3040 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015666681819554175, + "loss": 0.1712, + "step": 3045 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015652255735011135, + "loss": 0.1693, + "step": 3050 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015637812344376583, + "loss": 0.1714, + "step": 3055 + }, + { + "epoch": 0.99, + "learning_rate": 0.0001562335169187332, + "loss": 0.1661, + "step": 3060 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015608873821777012, + "loss": 0.1676, + "step": 3065 + }, + { + "epoch": 0.99, + "learning_rate": 0.00015594378778416044, + "loss": 0.1654, + "step": 3070 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015579866606171372, + "loss": 0.1644, + "step": 3075 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015565337349476404, + "loss": 0.1744, + "step": 3080 + }, + { + "epoch": 1.0, + "learning_rate": 0.0001555079105281686, + "loss": 0.1606, + "step": 3085 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015536227760730632, + "loss": 0.1659, + "step": 3090 + }, + { + "epoch": 1.0, + "learning_rate": 0.0001552164751780764, + "loss": 0.1602, + "step": 3095 + }, + { + "epoch": 1.0, + "learning_rate": 0.00015507050368689714, + "loss": 0.1636, + "step": 3100 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015492436358070438, + "loss": 0.1555, + "step": 3105 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015477805530695035, + "loss": 0.1661, + "step": 3110 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015463157931360207, + "loss": 0.1628, + "step": 3115 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015448493604914014, + "loss": 0.1696, + "step": 3120 + }, + { + "epoch": 1.01, + "learning_rate": 0.0001543381259625573, + "loss": 0.1685, + "step": 3125 + }, + { + "epoch": 1.01, + "learning_rate": 0.00015419114950335704, + "loss": 0.1635, + "step": 3130 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015404400712155228, + "loss": 0.1631, + "step": 3135 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015389669926766395, + "loss": 0.1664, + "step": 3140 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015374922639271963, + "loss": 0.1617, + "step": 3145 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015360158894825216, + "loss": 0.174, + "step": 3150 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015345378738629826, + "loss": 0.1591, + "step": 3155 + }, + { + "epoch": 1.02, + "learning_rate": 0.00015330582215939713, + "loss": 0.1729, + "step": 3160 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015315769372058907, + "loss": 0.1732, + "step": 3165 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015300940252341416, + "loss": 0.1608, + "step": 3170 + }, + { + "epoch": 1.03, + "learning_rate": 0.0001528609490219108, + "loss": 0.168, + "step": 3175 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015271233367061425, + "loss": 0.1731, + "step": 3180 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015256355692455542, + "loss": 0.1656, + "step": 3185 + }, + { + "epoch": 1.03, + "learning_rate": 0.00015241461923925934, + "loss": 0.1591, + "step": 3190 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015226552107074385, + "loss": 0.1635, + "step": 3195 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015211626287551803, + "loss": 0.1622, + "step": 3200 + }, + { + "epoch": 1.04, + "learning_rate": 0.0001519668451105811, + "loss": 0.162, + "step": 3205 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015181726823342072, + "loss": 0.1614, + "step": 3210 + }, + { + "epoch": 1.04, + "learning_rate": 0.00015166753270201176, + "loss": 0.1657, + "step": 3215 + }, + { + "epoch": 1.04, + "learning_rate": 0.0001515176389748149, + "loss": 0.1623, + "step": 3220 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015136758751077504, + "loss": 0.1629, + "step": 3225 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015121737876932034, + "loss": 0.1556, + "step": 3230 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015106701321036017, + "loss": 0.1651, + "step": 3235 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015091649129428424, + "loss": 0.1575, + "step": 3240 + }, + { + "epoch": 1.05, + "learning_rate": 0.000150765813481961, + "loss": 0.1594, + "step": 3245 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015061498023473608, + "loss": 0.1624, + "step": 3250 + }, + { + "epoch": 1.05, + "learning_rate": 0.00015046399201443114, + "loss": 0.1616, + "step": 3255 + }, + { + "epoch": 1.06, + "learning_rate": 0.00015031284928334238, + "loss": 0.1693, + "step": 3260 + }, + { + "epoch": 1.06, + "learning_rate": 0.00015016155250423892, + "loss": 0.1592, + "step": 3265 + }, + { + "epoch": 1.06, + "learning_rate": 0.00015001010214036174, + "loss": 0.1542, + "step": 3270 + }, + { + "epoch": 1.06, + "learning_rate": 0.0001498584986554219, + "loss": 0.1631, + "step": 3275 + }, + { + "epoch": 1.06, + "learning_rate": 0.0001497067425135993, + "loss": 0.1612, + "step": 3280 + }, + { + "epoch": 1.06, + "learning_rate": 0.00014955483417954137, + "loss": 0.1551, + "step": 3285 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014940277411836146, + "loss": 0.155, + "step": 3290 + }, + { + "epoch": 1.07, + "learning_rate": 0.0001492505627956374, + "loss": 0.1593, + "step": 3295 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014909820067741028, + "loss": 0.1622, + "step": 3300 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014894568823018277, + "loss": 0.1637, + "step": 3305 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014879302592091798, + "loss": 0.1643, + "step": 3310 + }, + { + "epoch": 1.07, + "learning_rate": 0.00014864021421703768, + "loss": 0.1642, + "step": 3315 + }, + { + "epoch": 1.08, + "learning_rate": 0.00014848725358642114, + "loss": 0.1607, + "step": 3320 + }, + { + "epoch": 1.08, + "learning_rate": 0.00014833414449740373, + "loss": 0.1591, + "step": 3325 + }, + { + "epoch": 1.08, + "learning_rate": 0.0001481808874187752, + "loss": 0.1682, + "step": 3330 + }, + { + "epoch": 1.08, + "learning_rate": 0.00014802748281977842, + "loss": 0.1572, + "step": 3335 + }, + { + "epoch": 1.08, + "learning_rate": 0.0001478739311701081, + "loss": 0.1595, + "step": 3340 + }, + { + "epoch": 1.08, + "learning_rate": 0.000147720232939909, + "loss": 0.1683, + "step": 3345 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014756638859977483, + "loss": 0.1539, + "step": 3350 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014741239862074655, + "loss": 0.1594, + "step": 3355 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014725826347431113, + "loss": 0.1658, + "step": 3360 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014710398363239994, + "loss": 0.1621, + "step": 3365 + }, + { + "epoch": 1.09, + "learning_rate": 0.0001469495595673874, + "loss": 0.1651, + "step": 3370 + }, + { + "epoch": 1.09, + "learning_rate": 0.00014679499175208953, + "loss": 0.1655, + "step": 3375 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014664028065976246, + "loss": 0.1669, + "step": 3380 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014648542676410105, + "loss": 0.1593, + "step": 3385 + }, + { + "epoch": 1.1, + "learning_rate": 0.0001463304305392373, + "loss": 0.157, + "step": 3390 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014617529245973917, + "loss": 0.1593, + "step": 3395 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014602001300060877, + "loss": 0.1516, + "step": 3400 + }, + { + "epoch": 1.1, + "learning_rate": 0.00014586459263728115, + "loss": 0.156, + "step": 3405 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014570903184562282, + "loss": 0.1637, + "step": 3410 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014555333110193022, + "loss": 0.1552, + "step": 3415 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014539749088292826, + "loss": 0.1624, + "step": 3420 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014524151166576897, + "loss": 0.1568, + "step": 3425 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014508539392802994, + "loss": 0.1583, + "step": 3430 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014492913814771283, + "loss": 0.1625, + "step": 3435 + }, + { + "epoch": 1.11, + "learning_rate": 0.00014477274480324202, + "loss": 0.1563, + "step": 3440 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014461621437346307, + "loss": 0.1634, + "step": 3445 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014445954733764127, + "loss": 0.1452, + "step": 3450 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014430274417546012, + "loss": 0.1539, + "step": 3455 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014414580536702, + "loss": 0.1636, + "step": 3460 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014398873139283657, + "loss": 0.1509, + "step": 3465 + }, + { + "epoch": 1.12, + "learning_rate": 0.00014383152273383934, + "loss": 0.1636, + "step": 3470 + }, + { + "epoch": 1.13, + "learning_rate": 0.00014367417987137015, + "loss": 0.1619, + "step": 3475 + }, + { + "epoch": 1.13, + "learning_rate": 0.00014351670328718183, + "loss": 0.1648, + "step": 3480 + }, + { + "epoch": 1.13, + "learning_rate": 0.00014335909346343658, + "loss": 0.1508, + "step": 3485 + }, + { + "epoch": 1.13, + "learning_rate": 0.0001432013508827046, + "loss": 0.1546, + "step": 3490 + }, + { + "epoch": 1.13, + "learning_rate": 0.0001430434760279625, + "loss": 0.1549, + "step": 3495 + }, + { + "epoch": 1.13, + "learning_rate": 0.000142885469382592, + "loss": 0.1412, + "step": 3500 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001427273314303782, + "loss": 0.1574, + "step": 3505 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001425690626555083, + "loss": 0.16, + "step": 3510 + }, + { + "epoch": 1.14, + "learning_rate": 0.00014241066354257008, + "loss": 0.1633, + "step": 3515 + }, + { + "epoch": 1.14, + "learning_rate": 0.00014225213457655034, + "loss": 0.1499, + "step": 3520 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001420934762428335, + "loss": 0.159, + "step": 3525 + }, + { + "epoch": 1.14, + "learning_rate": 0.00014193468902720007, + "loss": 0.1561, + "step": 3530 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014177577341582516, + "loss": 0.1619, + "step": 3535 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014161672989527698, + "loss": 0.16, + "step": 3540 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014145755895251546, + "loss": 0.148, + "step": 3545 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014129826107489054, + "loss": 0.1502, + "step": 3550 + }, + { + "epoch": 1.15, + "learning_rate": 0.00014113883675014098, + "loss": 0.1427, + "step": 3555 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001409792864663925, + "loss": 0.1613, + "step": 3560 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001408196107121566, + "loss": 0.1597, + "step": 3565 + }, + { + "epoch": 1.16, + "learning_rate": 0.00014065980997632896, + "loss": 0.1501, + "step": 3570 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001404998847481879, + "loss": 0.1537, + "step": 3575 + }, + { + "epoch": 1.16, + "learning_rate": 0.00014033983551739286, + "loss": 0.1608, + "step": 3580 + }, + { + "epoch": 1.16, + "learning_rate": 0.00014017966277398304, + "loss": 0.1488, + "step": 3585 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001400193670083757, + "loss": 0.1593, + "step": 3590 + }, + { + "epoch": 1.16, + "learning_rate": 0.00013985894871136492, + "loss": 0.1525, + "step": 3595 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013969840837411982, + "loss": 0.1529, + "step": 3600 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013953774648818324, + "loss": 0.1489, + "step": 3605 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013937696354547022, + "loss": 0.144, + "step": 3610 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013921606003826627, + "loss": 0.1584, + "step": 3615 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013905503645922628, + "loss": 0.1657, + "step": 3620 + }, + { + "epoch": 1.17, + "learning_rate": 0.00013889389330137264, + "loss": 0.1557, + "step": 3625 + }, + { + "epoch": 1.18, + "learning_rate": 0.00013873263105809388, + "loss": 0.1551, + "step": 3630 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001385712502231432, + "loss": 0.157, + "step": 3635 + }, + { + "epoch": 1.18, + "learning_rate": 0.00013840975129063684, + "loss": 0.1548, + "step": 3640 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001382481347550526, + "loss": 0.1565, + "step": 3645 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001380864011112285, + "loss": 0.1497, + "step": 3650 + }, + { + "epoch": 1.18, + "learning_rate": 0.000137924550854361, + "loss": 0.1569, + "step": 3655 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001377625844800037, + "loss": 0.1582, + "step": 3660 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013760050248406557, + "loss": 0.1571, + "step": 3665 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001374383053628098, + "loss": 0.1524, + "step": 3670 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013727599361285187, + "loss": 0.1567, + "step": 3675 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013711356773115838, + "loss": 0.1548, + "step": 3680 + }, + { + "epoch": 1.19, + "learning_rate": 0.00013695102821504533, + "loss": 0.1557, + "step": 3685 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001367883755621766, + "loss": 0.148, + "step": 3690 + }, + { + "epoch": 1.2, + "learning_rate": 0.00013662561027056252, + "loss": 0.1604, + "step": 3695 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001364627328385583, + "loss": 0.15, + "step": 3700 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001362997437648625, + "loss": 0.1476, + "step": 3705 + }, + { + "epoch": 1.2, + "learning_rate": 0.00013613664354851547, + "loss": 0.1586, + "step": 3710 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001359734326888979, + "loss": 0.1536, + "step": 3715 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013581011168572923, + "loss": 0.1564, + "step": 3720 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013564668103906616, + "loss": 0.1534, + "step": 3725 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013548314124930104, + "loss": 0.1521, + "step": 3730 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013531949281716044, + "loss": 0.1489, + "step": 3735 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013515573624370357, + "loss": 0.1528, + "step": 3740 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001349918720303207, + "loss": 0.1405, + "step": 3745 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013482790067873178, + "loss": 0.1457, + "step": 3750 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013466382269098468, + "loss": 0.1554, + "step": 3755 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013449963856945383, + "loss": 0.1478, + "step": 3760 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013433534881683858, + "loss": 0.1633, + "step": 3765 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013417095393616178, + "loss": 0.15, + "step": 3770 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013400645443076804, + "loss": 0.1552, + "step": 3775 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013384185080432244, + "loss": 0.1485, + "step": 3780 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001336771435608088, + "loss": 0.1531, + "step": 3785 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013351233320452812, + "loss": 0.158, + "step": 3790 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013334742024009725, + "loss": 0.1554, + "step": 3795 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013318240517244713, + "loss": 0.1537, + "step": 3800 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001330172885068213, + "loss": 0.1584, + "step": 3805 + }, + { + "epoch": 1.23, + "learning_rate": 0.00013285207074877447, + "loss": 0.1612, + "step": 3810 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013268675240417073, + "loss": 0.1449, + "step": 3815 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013252133397918226, + "loss": 0.1456, + "step": 3820 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001323558159802876, + "loss": 0.1541, + "step": 3825 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013219019891427025, + "loss": 0.1492, + "step": 3830 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013202448328821694, + "loss": 0.1528, + "step": 3835 + }, + { + "epoch": 1.24, + "learning_rate": 0.00013185866960951622, + "loss": 0.1516, + "step": 3840 + }, + { + "epoch": 1.25, + "learning_rate": 0.0001316927583858569, + "loss": 0.1484, + "step": 3845 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013152675012522629, + "loss": 0.1509, + "step": 3850 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013136064533590897, + "loss": 0.1516, + "step": 3855 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013119444452648504, + "loss": 0.15, + "step": 3860 + }, + { + "epoch": 1.25, + "learning_rate": 0.0001310281482058286, + "loss": 0.1571, + "step": 3865 + }, + { + "epoch": 1.25, + "learning_rate": 0.00013086175688310612, + "loss": 0.1526, + "step": 3870 + }, + { + "epoch": 1.26, + "learning_rate": 0.000130695271067775, + "loss": 0.1445, + "step": 3875 + }, + { + "epoch": 1.26, + "learning_rate": 0.00013052869126958198, + "loss": 0.1598, + "step": 3880 + }, + { + "epoch": 1.26, + "learning_rate": 0.00013036201799856149, + "loss": 0.1499, + "step": 3885 + }, + { + "epoch": 1.26, + "learning_rate": 0.0001301952517650342, + "loss": 0.1507, + "step": 3890 + }, + { + "epoch": 1.26, + "learning_rate": 0.0001300283930796054, + "loss": 0.1489, + "step": 3895 + }, + { + "epoch": 1.26, + "learning_rate": 0.00012986144245316346, + "loss": 0.1477, + "step": 3900 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012969440039687822, + "loss": 0.1528, + "step": 3905 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012952726742219953, + "loss": 0.1514, + "step": 3910 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012936004404085557, + "loss": 0.1482, + "step": 3915 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012919273076485133, + "loss": 0.1558, + "step": 3920 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012902532810646697, + "loss": 0.1539, + "step": 3925 + }, + { + "epoch": 1.27, + "learning_rate": 0.00012885783657825647, + "loss": 0.1479, + "step": 3930 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012869025669304585, + "loss": 0.1449, + "step": 3935 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012852258896393158, + "loss": 0.1489, + "step": 3940 + }, + { + "epoch": 1.28, + "learning_rate": 0.0001283548339042792, + "loss": 0.1479, + "step": 3945 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012818699202772152, + "loss": 0.149, + "step": 3950 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012801906384815733, + "loss": 0.1472, + "step": 3955 + }, + { + "epoch": 1.28, + "learning_rate": 0.0001278510498797495, + "loss": 0.1526, + "step": 3960 + }, + { + "epoch": 1.28, + "learning_rate": 0.00012768295063692376, + "loss": 0.1469, + "step": 3965 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012751476663436668, + "loss": 0.1576, + "step": 3970 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012734649838702455, + "loss": 0.1583, + "step": 3975 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012717814641010153, + "loss": 0.1434, + "step": 3980 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012700971121905812, + "loss": 0.1463, + "step": 3985 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012684119332960963, + "loss": 0.1543, + "step": 3990 + }, + { + "epoch": 1.29, + "learning_rate": 0.00012667259325772464, + "loss": 0.1501, + "step": 3995 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012650391151962324, + "loss": 0.1475, + "step": 4000 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012633514863177566, + "loss": 0.147, + "step": 4005 + }, + { + "epoch": 1.3, + "learning_rate": 0.0001261663051109005, + "loss": 0.1454, + "step": 4010 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012599738147396332, + "loss": 0.1459, + "step": 4015 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012582837823817497, + "loss": 0.1479, + "step": 4020 + }, + { + "epoch": 1.3, + "learning_rate": 0.00012565929592099002, + "loss": 0.1541, + "step": 4025 + }, + { + "epoch": 1.31, + "learning_rate": 0.00012549013504010512, + "loss": 0.145, + "step": 4030 + }, + { + "epoch": 1.31, + "learning_rate": 0.0001253208961134575, + "loss": 0.1467, + "step": 4035 + }, + { + "epoch": 1.31, + "learning_rate": 0.0001251515796592234, + "loss": 0.1497, + "step": 4040 + }, + { + "epoch": 1.31, + "learning_rate": 0.00012498218619581632, + "loss": 0.1439, + "step": 4045 + }, + { + "epoch": 1.31, + "learning_rate": 0.00012481271624188568, + "loss": 0.1419, + "step": 4050 + }, + { + "epoch": 1.31, + "learning_rate": 0.00012464317031631494, + "loss": 0.1518, + "step": 4055 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012447354893822034, + "loss": 0.1462, + "step": 4060 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012430385262694906, + "loss": 0.1473, + "step": 4065 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012413408190207768, + "loss": 0.1495, + "step": 4070 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012396423728341067, + "loss": 0.1566, + "step": 4075 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012379431929097873, + "loss": 0.1507, + "step": 4080 + }, + { + "epoch": 1.32, + "learning_rate": 0.00012362432844503725, + "loss": 0.1544, + "step": 4085 + }, + { + "epoch": 1.33, + "learning_rate": 0.0001234542652660646, + "loss": 0.1528, + "step": 4090 + }, + { + "epoch": 1.33, + "learning_rate": 0.00012328413027476066, + "loss": 0.153, + "step": 4095 + }, + { + "epoch": 1.33, + "learning_rate": 0.00012311392399204526, + "loss": 0.1438, + "step": 4100 + }, + { + "epoch": 1.33, + "learning_rate": 0.0001229436469390564, + "loss": 0.1497, + "step": 4105 + }, + { + "epoch": 1.33, + "learning_rate": 0.00012277329963714882, + "loss": 0.1471, + "step": 4110 + }, + { + "epoch": 1.33, + "learning_rate": 0.0001226028826078923, + "loss": 0.1442, + "step": 4115 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012243239637307023, + "loss": 0.1455, + "step": 4120 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012226184145467775, + "loss": 0.1509, + "step": 4125 + }, + { + "epoch": 1.34, + "learning_rate": 0.0001220912183749204, + "loss": 0.1504, + "step": 4130 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012192052765621234, + "loss": 0.1524, + "step": 4135 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012174976982117488, + "loss": 0.1526, + "step": 4140 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012157894539263482, + "loss": 0.1501, + "step": 4145 + }, + { + "epoch": 1.34, + "learning_rate": 0.00012140805489362281, + "loss": 0.1464, + "step": 4150 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012123709884737189, + "loss": 0.1481, + "step": 4155 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012106607777731567, + "loss": 0.1483, + "step": 4160 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012089499220708696, + "loss": 0.1456, + "step": 4165 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012072384266051595, + "loss": 0.1479, + "step": 4170 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012055262966162883, + "loss": 0.1488, + "step": 4175 + }, + { + "epoch": 1.35, + "learning_rate": 0.00012038135373464602, + "loss": 0.1468, + "step": 4180 + }, + { + "epoch": 1.36, + "learning_rate": 0.00012021001540398058, + "loss": 0.1412, + "step": 4185 + }, + { + "epoch": 1.36, + "learning_rate": 0.00012003861519423662, + "loss": 0.1503, + "step": 4190 + }, + { + "epoch": 1.36, + "learning_rate": 0.00011986715363020784, + "loss": 0.1497, + "step": 4195 + }, + { + "epoch": 1.36, + "learning_rate": 0.00011969563123687566, + "loss": 0.1457, + "step": 4200 + }, + { + "epoch": 1.36, + "learning_rate": 0.00011952404853940782, + "loss": 0.1469, + "step": 4205 + }, + { + "epoch": 1.36, + "learning_rate": 0.00011935240606315666, + "loss": 0.1485, + "step": 4210 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011918070433365754, + "loss": 0.1413, + "step": 4215 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011900894387662731, + "loss": 0.1565, + "step": 4220 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011883712521796253, + "loss": 0.1516, + "step": 4225 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011866524888373811, + "loss": 0.146, + "step": 4230 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011849331540020543, + "loss": 0.1507, + "step": 4235 + }, + { + "epoch": 1.37, + "learning_rate": 0.00011832132529379088, + "loss": 0.1493, + "step": 4240 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011814927909109419, + "loss": 0.1419, + "step": 4245 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011797717731888691, + "loss": 0.1414, + "step": 4250 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011780502050411069, + "loss": 0.1385, + "step": 4255 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011763280917387572, + "loss": 0.1473, + "step": 4260 + }, + { + "epoch": 1.38, + "learning_rate": 0.0001174605438554591, + "loss": 0.1496, + "step": 4265 + }, + { + "epoch": 1.38, + "learning_rate": 0.00011728822507630326, + "loss": 0.1369, + "step": 4270 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011711585336401429, + "loss": 0.1437, + "step": 4275 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011694342924636031, + "loss": 0.1564, + "step": 4280 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011677095325126996, + "loss": 0.1443, + "step": 4285 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011659842590683074, + "loss": 0.1577, + "step": 4290 + }, + { + "epoch": 1.39, + "learning_rate": 0.00011642584774128728, + "loss": 0.1416, + "step": 4295 + }, + { + "epoch": 1.39, + "learning_rate": 0.0001162532192830399, + "loss": 0.145, + "step": 4300 + }, + { + "epoch": 1.4, + "learning_rate": 0.00011608054106064281, + "loss": 0.1458, + "step": 4305 + }, + { + "epoch": 1.4, + "learning_rate": 0.0001159078136028027, + "loss": 0.1445, + "step": 4310 + }, + { + "epoch": 1.4, + "learning_rate": 0.00011573503743837693, + "loss": 0.139, + "step": 4315 + }, + { + "epoch": 1.4, + "learning_rate": 0.00011556221309637203, + "loss": 0.1441, + "step": 4320 + }, + { + "epoch": 1.4, + "learning_rate": 0.000115389341105942, + "loss": 0.1567, + "step": 4325 + }, + { + "epoch": 1.4, + "learning_rate": 0.0001152164219963868, + "loss": 0.1499, + "step": 4330 + }, + { + "epoch": 1.4, + "learning_rate": 0.00011504345629715057, + "loss": 0.15, + "step": 4335 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011487044453782016, + "loss": 0.1449, + "step": 4340 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011469738724812346, + "loss": 0.1423, + "step": 4345 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011452428495792773, + "loss": 0.1468, + "step": 4350 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011435113819723797, + "loss": 0.1519, + "step": 4355 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011417794749619544, + "loss": 0.1498, + "step": 4360 + }, + { + "epoch": 1.41, + "learning_rate": 0.00011400471338507585, + "loss": 0.1406, + "step": 4365 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011383143639428791, + "loss": 0.1485, + "step": 4370 + }, + { + "epoch": 1.42, + "learning_rate": 0.0001136581170543715, + "loss": 0.1502, + "step": 4375 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011348475589599625, + "loss": 0.139, + "step": 4380 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011331135344995985, + "loss": 0.1485, + "step": 4385 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011313791024718634, + "loss": 0.133, + "step": 4390 + }, + { + "epoch": 1.42, + "learning_rate": 0.00011296442681872457, + "loss": 0.1473, + "step": 4395 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011279090369574658, + "loss": 0.1401, + "step": 4400 + }, + { + "epoch": 1.43, + "learning_rate": 0.0001126173414095459, + "loss": 0.1483, + "step": 4405 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011244374049153603, + "loss": 0.1436, + "step": 4410 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011227010147324868, + "loss": 0.1445, + "step": 4415 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011209642488633227, + "loss": 0.1451, + "step": 4420 + }, + { + "epoch": 1.43, + "learning_rate": 0.00011192271126255024, + "loss": 0.1438, + "step": 4425 + }, + { + "epoch": 1.44, + "learning_rate": 0.00011174896113377945, + "loss": 0.1474, + "step": 4430 + }, + { + "epoch": 1.44, + "learning_rate": 0.0001115751750320084, + "loss": 0.1444, + "step": 4435 + }, + { + "epoch": 1.44, + "learning_rate": 0.00011140135348933593, + "loss": 0.147, + "step": 4440 + }, + { + "epoch": 1.44, + "learning_rate": 0.00011122749703796929, + "loss": 0.1447, + "step": 4445 + }, + { + "epoch": 1.44, + "learning_rate": 0.0001110536062102226, + "loss": 0.1423, + "step": 4450 + }, + { + "epoch": 1.44, + "learning_rate": 0.00011087968153851525, + "loss": 0.1405, + "step": 4455 + }, + { + "epoch": 1.45, + "learning_rate": 0.00011070572355537026, + "loss": 0.1468, + "step": 4460 + }, + { + "epoch": 1.45, + "learning_rate": 0.00011053173279341265, + "loss": 0.1442, + "step": 4465 + }, + { + "epoch": 1.45, + "learning_rate": 0.0001103577097853678, + "loss": 0.1447, + "step": 4470 + }, + { + "epoch": 1.45, + "learning_rate": 0.00011018365506405978, + "loss": 0.1514, + "step": 4475 + }, + { + "epoch": 1.45, + "learning_rate": 0.00011000956916240985, + "loss": 0.148, + "step": 4480 + }, + { + "epoch": 1.45, + "learning_rate": 0.00010983545261343464, + "loss": 0.1499, + "step": 4485 + }, + { + "epoch": 1.45, + "learning_rate": 0.00010966130595024466, + "loss": 0.1398, + "step": 4490 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010948712970604266, + "loss": 0.1379, + "step": 4495 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010931292441412191, + "loss": 0.1438, + "step": 4500 + }, + { + "epoch": 1.46, + "learning_rate": 0.0001091386906078646, + "loss": 0.1453, + "step": 4505 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010896442882074029, + "loss": 0.1439, + "step": 4510 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010879013958630418, + "loss": 0.1436, + "step": 4515 + }, + { + "epoch": 1.46, + "learning_rate": 0.00010861582343819548, + "loss": 0.144, + "step": 4520 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010844148091013581, + "loss": 0.1439, + "step": 4525 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010826711253592768, + "loss": 0.1441, + "step": 4530 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010809271884945256, + "loss": 0.1421, + "step": 4535 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010791830038466952, + "loss": 0.1476, + "step": 4540 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010774385767561347, + "loss": 0.1484, + "step": 4545 + }, + { + "epoch": 1.47, + "learning_rate": 0.00010756939125639358, + "loss": 0.1437, + "step": 4550 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010739490166119155, + "loss": 0.1403, + "step": 4555 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010722038942426014, + "loss": 0.1437, + "step": 4560 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010704585507992131, + "loss": 0.1429, + "step": 4565 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010687129916256482, + "loss": 0.1401, + "step": 4570 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010669672220664644, + "loss": 0.1448, + "step": 4575 + }, + { + "epoch": 1.48, + "learning_rate": 0.00010652212474668635, + "loss": 0.1449, + "step": 4580 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010634750731726748, + "loss": 0.1431, + "step": 4585 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010617287045303395, + "loss": 0.1467, + "step": 4590 + }, + { + "epoch": 1.49, + "learning_rate": 0.0001059982146886894, + "loss": 0.1403, + "step": 4595 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010582354055899528, + "loss": 0.1467, + "step": 4600 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010564884859876934, + "loss": 0.1399, + "step": 4605 + }, + { + "epoch": 1.49, + "learning_rate": 0.00010547413934288386, + "loss": 0.1408, + "step": 4610 + }, + { + "epoch": 1.5, + "learning_rate": 0.0001052994133262641, + "loss": 0.1411, + "step": 4615 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010512467108388661, + "loss": 0.1398, + "step": 4620 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010494991315077772, + "loss": 0.1444, + "step": 4625 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010477514006201166, + "loss": 0.1438, + "step": 4630 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010460035235270918, + "loss": 0.1499, + "step": 4635 + }, + { + "epoch": 1.5, + "learning_rate": 0.00010442555055803572, + "loss": 0.1451, + "step": 4640 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010425073521319986, + "loss": 0.15, + "step": 4645 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010407590685345174, + "loss": 0.1471, + "step": 4650 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010390106601408125, + "loss": 0.1454, + "step": 4655 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010372621323041656, + "loss": 0.1476, + "step": 4660 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010355134903782235, + "loss": 0.142, + "step": 4665 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010337647397169832, + "loss": 0.1391, + "step": 4670 + }, + { + "epoch": 1.51, + "learning_rate": 0.00010320158856747737, + "loss": 0.1398, + "step": 4675 + }, + { + "epoch": 1.52, + "learning_rate": 0.0001030266933606241, + "loss": 0.15, + "step": 4680 + }, + { + "epoch": 1.52, + "learning_rate": 0.0001028517888866331, + "loss": 0.1387, + "step": 4685 + }, + { + "epoch": 1.52, + "learning_rate": 0.00010267687568102737, + "loss": 0.1484, + "step": 4690 + }, + { + "epoch": 1.52, + "learning_rate": 0.00010250195427935663, + "loss": 0.1483, + "step": 4695 + }, + { + "epoch": 1.52, + "learning_rate": 0.00010232702521719564, + "loss": 0.1452, + "step": 4700 + }, + { + "epoch": 1.52, + "learning_rate": 0.00010215208903014272, + "loss": 0.1481, + "step": 4705 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010197714625381795, + "loss": 0.1406, + "step": 4710 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010180219742386157, + "loss": 0.139, + "step": 4715 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010162724307593234, + "loss": 0.1354, + "step": 4720 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010145228374570599, + "loss": 0.1379, + "step": 4725 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010127731996887339, + "loss": 0.1481, + "step": 4730 + }, + { + "epoch": 1.53, + "learning_rate": 0.00010110235228113918, + "loss": 0.1464, + "step": 4735 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010092738121821985, + "loss": 0.1426, + "step": 4740 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010075240731584223, + "loss": 0.1415, + "step": 4745 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010057743110974194, + "loss": 0.1393, + "step": 4750 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010040245313566153, + "loss": 0.1408, + "step": 4755 + }, + { + "epoch": 1.54, + "learning_rate": 0.00010022747392934905, + "loss": 0.1358, + "step": 4760 + }, + { + "epoch": 1.54, + "learning_rate": 0.0001000524940265563, + "loss": 0.1401, + "step": 4765 + }, + { + "epoch": 1.55, + "learning_rate": 9.98775139630372e-05, + "loss": 0.1379, + "step": 4770 + }, + { + "epoch": 1.55, + "learning_rate": 9.970253427454616e-05, + "loss": 0.1425, + "step": 4775 + }, + { + "epoch": 1.55, + "learning_rate": 9.95275554968365e-05, + "loss": 0.1475, + "step": 4780 + }, + { + "epoch": 1.55, + "learning_rate": 9.935257816565861e-05, + "loss": 0.1364, + "step": 4785 + }, + { + "epoch": 1.55, + "learning_rate": 9.917760281675866e-05, + "loss": 0.1443, + "step": 4790 + }, + { + "epoch": 1.55, + "learning_rate": 9.900262998587653e-05, + "loss": 0.1388, + "step": 4795 + }, + { + "epoch": 1.56, + "learning_rate": 9.882766020874456e-05, + "loss": 0.1375, + "step": 4800 + }, + { + "epoch": 1.56, + "learning_rate": 9.865269402108565e-05, + "loss": 0.1413, + "step": 4805 + }, + { + "epoch": 1.56, + "learning_rate": 9.847773195861175e-05, + "loss": 0.1499, + "step": 4810 + }, + { + "epoch": 1.56, + "learning_rate": 9.830277455702213e-05, + "loss": 0.1421, + "step": 4815 + }, + { + "epoch": 1.56, + "learning_rate": 9.812782235200189e-05, + "loss": 0.146, + "step": 4820 + }, + { + "epoch": 1.56, + "learning_rate": 9.795287587922006e-05, + "loss": 0.1343, + "step": 4825 + }, + { + "epoch": 1.57, + "learning_rate": 9.777793567432828e-05, + "loss": 0.1451, + "step": 4830 + }, + { + "epoch": 1.57, + "learning_rate": 9.760300227295894e-05, + "loss": 0.1438, + "step": 4835 + }, + { + "epoch": 1.57, + "learning_rate": 9.742807621072353e-05, + "loss": 0.1436, + "step": 4840 + }, + { + "epoch": 1.57, + "learning_rate": 9.725315802321121e-05, + "loss": 0.1384, + "step": 4845 + }, + { + "epoch": 1.57, + "learning_rate": 9.707824824598684e-05, + "loss": 0.1368, + "step": 4850 + }, + { + "epoch": 1.57, + "learning_rate": 9.690334741458974e-05, + "loss": 0.1309, + "step": 4855 + }, + { + "epoch": 1.57, + "learning_rate": 9.672845606453168e-05, + "loss": 0.1384, + "step": 4860 + }, + { + "epoch": 1.58, + "learning_rate": 9.655357473129548e-05, + "loss": 0.1387, + "step": 4865 + }, + { + "epoch": 1.58, + "learning_rate": 9.637870395033324e-05, + "loss": 0.1426, + "step": 4870 + }, + { + "epoch": 1.58, + "learning_rate": 9.620384425706484e-05, + "loss": 0.1407, + "step": 4875 + }, + { + "epoch": 1.58, + "learning_rate": 9.602899618687606e-05, + "loss": 0.1413, + "step": 4880 + }, + { + "epoch": 1.58, + "learning_rate": 9.585416027511727e-05, + "loss": 0.1434, + "step": 4885 + }, + { + "epoch": 1.58, + "learning_rate": 9.567933705710143e-05, + "loss": 0.146, + "step": 4890 + }, + { + "epoch": 1.59, + "learning_rate": 9.550452706810281e-05, + "loss": 0.1384, + "step": 4895 + }, + { + "epoch": 1.59, + "learning_rate": 9.532973084335511e-05, + "loss": 0.1447, + "step": 4900 + }, + { + "epoch": 1.59, + "learning_rate": 9.515494891804979e-05, + "loss": 0.1418, + "step": 4905 + }, + { + "epoch": 1.59, + "learning_rate": 9.498018182733466e-05, + "loss": 0.144, + "step": 4910 + }, + { + "epoch": 1.59, + "learning_rate": 9.480543010631207e-05, + "loss": 0.1449, + "step": 4915 + }, + { + "epoch": 1.59, + "learning_rate": 9.463069429003728e-05, + "loss": 0.1391, + "step": 4920 + }, + { + "epoch": 1.6, + "learning_rate": 9.445597491351685e-05, + "loss": 0.1484, + "step": 4925 + }, + { + "epoch": 1.6, + "learning_rate": 9.42812725117071e-05, + "loss": 0.1402, + "step": 4930 + }, + { + "epoch": 1.6, + "learning_rate": 9.410658761951221e-05, + "loss": 0.1478, + "step": 4935 + }, + { + "epoch": 1.6, + "learning_rate": 9.393192077178295e-05, + "loss": 0.1431, + "step": 4940 + }, + { + "epoch": 1.6, + "learning_rate": 9.375727250331462e-05, + "loss": 0.1405, + "step": 4945 + }, + { + "epoch": 1.6, + "learning_rate": 9.358264334884585e-05, + "loss": 0.1359, + "step": 4950 + }, + { + "epoch": 1.61, + "learning_rate": 9.340803384305657e-05, + "loss": 0.1464, + "step": 4955 + }, + { + "epoch": 1.61, + "learning_rate": 9.323344452056668e-05, + "loss": 0.1408, + "step": 4960 + }, + { + "epoch": 1.61, + "learning_rate": 9.305887591593419e-05, + "loss": 0.1435, + "step": 4965 + }, + { + "epoch": 1.61, + "learning_rate": 9.288432856365374e-05, + "loss": 0.1449, + "step": 4970 + }, + { + "epoch": 1.61, + "learning_rate": 9.270980299815486e-05, + "loss": 0.1403, + "step": 4975 + }, + { + "epoch": 1.61, + "learning_rate": 9.253529975380039e-05, + "loss": 0.1434, + "step": 4980 + }, + { + "epoch": 1.62, + "learning_rate": 9.236081936488485e-05, + "loss": 0.1405, + "step": 4985 + }, + { + "epoch": 1.62, + "learning_rate": 9.21863623656327e-05, + "loss": 0.1394, + "step": 4990 + }, + { + "epoch": 1.62, + "learning_rate": 9.20119292901969e-05, + "loss": 0.1352, + "step": 4995 + }, + { + "epoch": 1.62, + "learning_rate": 9.183752067265703e-05, + "loss": 0.1452, + "step": 5000 + }, + { + "epoch": 1.62, + "learning_rate": 9.166313704701795e-05, + "loss": 0.1423, + "step": 5005 + }, + { + "epoch": 1.62, + "learning_rate": 9.14887789472078e-05, + "loss": 0.1378, + "step": 5010 + }, + { + "epoch": 1.63, + "learning_rate": 9.131444690707673e-05, + "loss": 0.1379, + "step": 5015 + }, + { + "epoch": 1.63, + "learning_rate": 9.114014146039503e-05, + "loss": 0.137, + "step": 5020 + }, + { + "epoch": 1.63, + "learning_rate": 9.096586314085162e-05, + "loss": 0.1459, + "step": 5025 + }, + { + "epoch": 1.63, + "learning_rate": 9.079161248205222e-05, + "loss": 0.1417, + "step": 5030 + }, + { + "epoch": 1.63, + "learning_rate": 9.061739001751803e-05, + "loss": 0.139, + "step": 5035 + }, + { + "epoch": 1.63, + "learning_rate": 9.044319628068387e-05, + "loss": 0.1322, + "step": 5040 + }, + { + "epoch": 1.63, + "learning_rate": 9.02690318048965e-05, + "loss": 0.1324, + "step": 5045 + }, + { + "epoch": 1.64, + "learning_rate": 9.009489712341327e-05, + "loss": 0.1403, + "step": 5050 + }, + { + "epoch": 1.64, + "learning_rate": 8.992079276940011e-05, + "loss": 0.1394, + "step": 5055 + }, + { + "epoch": 1.64, + "learning_rate": 8.974671927593028e-05, + "loss": 0.1389, + "step": 5060 + }, + { + "epoch": 1.64, + "learning_rate": 8.957267717598236e-05, + "loss": 0.141, + "step": 5065 + }, + { + "epoch": 1.64, + "learning_rate": 8.939866700243901e-05, + "loss": 0.1334, + "step": 5070 + }, + { + "epoch": 1.64, + "learning_rate": 8.922468928808497e-05, + "loss": 0.1311, + "step": 5075 + }, + { + "epoch": 1.65, + "learning_rate": 8.905074456560574e-05, + "loss": 0.1439, + "step": 5080 + }, + { + "epoch": 1.65, + "learning_rate": 8.887683336758563e-05, + "loss": 0.1424, + "step": 5085 + }, + { + "epoch": 1.65, + "learning_rate": 8.870295622650651e-05, + "loss": 0.1476, + "step": 5090 + }, + { + "epoch": 1.65, + "learning_rate": 8.852911367474577e-05, + "loss": 0.1436, + "step": 5095 + }, + { + "epoch": 1.65, + "learning_rate": 8.835530624457503e-05, + "loss": 0.1442, + "step": 5100 + }, + { + "epoch": 1.65, + "learning_rate": 8.81815344681584e-05, + "loss": 0.1467, + "step": 5105 + }, + { + "epoch": 1.66, + "learning_rate": 8.800779887755066e-05, + "loss": 0.1409, + "step": 5110 + }, + { + "epoch": 1.66, + "learning_rate": 8.783410000469601e-05, + "loss": 0.1487, + "step": 5115 + }, + { + "epoch": 1.66, + "learning_rate": 8.7660438381426e-05, + "loss": 0.1463, + "step": 5120 + }, + { + "epoch": 1.66, + "learning_rate": 8.748681453945833e-05, + "loss": 0.1398, + "step": 5125 + }, + { + "epoch": 1.66, + "learning_rate": 8.73132290103949e-05, + "loss": 0.1358, + "step": 5130 + }, + { + "epoch": 1.66, + "learning_rate": 8.713968232572039e-05, + "loss": 0.143, + "step": 5135 + }, + { + "epoch": 1.67, + "learning_rate": 8.69661750168004e-05, + "loss": 0.137, + "step": 5140 + }, + { + "epoch": 1.67, + "learning_rate": 8.679270761488019e-05, + "loss": 0.1393, + "step": 5145 + }, + { + "epoch": 1.67, + "learning_rate": 8.661928065108259e-05, + "loss": 0.1472, + "step": 5150 + }, + { + "epoch": 1.67, + "learning_rate": 8.644589465640684e-05, + "loss": 0.1389, + "step": 5155 + }, + { + "epoch": 1.67, + "learning_rate": 8.627255016172652e-05, + "loss": 0.1487, + "step": 5160 + }, + { + "epoch": 1.67, + "learning_rate": 8.609924769778835e-05, + "loss": 0.1403, + "step": 5165 + }, + { + "epoch": 1.68, + "learning_rate": 8.59259877952103e-05, + "loss": 0.1351, + "step": 5170 + }, + { + "epoch": 1.68, + "learning_rate": 8.575277098447987e-05, + "loss": 0.1406, + "step": 5175 + }, + { + "epoch": 1.68, + "learning_rate": 8.557959779595286e-05, + "loss": 0.1378, + "step": 5180 + }, + { + "epoch": 1.68, + "learning_rate": 8.540646875985134e-05, + "loss": 0.1352, + "step": 5185 + }, + { + "epoch": 1.68, + "learning_rate": 8.523338440626232e-05, + "loss": 0.1406, + "step": 5190 + }, + { + "epoch": 1.68, + "learning_rate": 8.50603452651358e-05, + "loss": 0.1432, + "step": 5195 + }, + { + "epoch": 1.69, + "learning_rate": 8.488735186628364e-05, + "loss": 0.1433, + "step": 5200 + }, + { + "epoch": 1.69, + "learning_rate": 8.471440473937732e-05, + "loss": 0.1368, + "step": 5205 + }, + { + "epoch": 1.69, + "learning_rate": 8.454150441394697e-05, + "loss": 0.136, + "step": 5210 + }, + { + "epoch": 1.69, + "learning_rate": 8.436865141937912e-05, + "loss": 0.1502, + "step": 5215 + }, + { + "epoch": 1.69, + "learning_rate": 8.419584628491562e-05, + "loss": 0.1395, + "step": 5220 + }, + { + "epoch": 1.69, + "learning_rate": 8.402308953965165e-05, + "loss": 0.1294, + "step": 5225 + }, + { + "epoch": 1.69, + "learning_rate": 8.385038171253424e-05, + "loss": 0.1438, + "step": 5230 + }, + { + "epoch": 1.7, + "learning_rate": 8.367772333236073e-05, + "loss": 0.1371, + "step": 5235 + }, + { + "epoch": 1.7, + "learning_rate": 8.350511492777698e-05, + "loss": 0.138, + "step": 5240 + }, + { + "epoch": 1.7, + "learning_rate": 8.333255702727588e-05, + "loss": 0.1389, + "step": 5245 + }, + { + "epoch": 1.7, + "learning_rate": 8.316005015919559e-05, + "loss": 0.1384, + "step": 5250 + }, + { + "epoch": 1.7, + "learning_rate": 8.298759485171819e-05, + "loss": 0.1384, + "step": 5255 + }, + { + "epoch": 1.7, + "learning_rate": 8.281519163286772e-05, + "loss": 0.1307, + "step": 5260 + }, + { + "epoch": 1.71, + "learning_rate": 8.264284103050888e-05, + "loss": 0.1413, + "step": 5265 + }, + { + "epoch": 1.71, + "learning_rate": 8.247054357234511e-05, + "loss": 0.1322, + "step": 5270 + }, + { + "epoch": 1.71, + "learning_rate": 8.229829978591729e-05, + "loss": 0.1433, + "step": 5275 + }, + { + "epoch": 1.71, + "learning_rate": 8.212611019860188e-05, + "loss": 0.1384, + "step": 5280 + }, + { + "epoch": 1.71, + "learning_rate": 8.195397533760943e-05, + "loss": 0.1396, + "step": 5285 + }, + { + "epoch": 1.71, + "learning_rate": 8.178189572998284e-05, + "loss": 0.1403, + "step": 5290 + }, + { + "epoch": 1.72, + "learning_rate": 8.160987190259605e-05, + "loss": 0.1429, + "step": 5295 + }, + { + "epoch": 1.72, + "learning_rate": 8.14379043821519e-05, + "loss": 0.1384, + "step": 5300 + }, + { + "epoch": 1.72, + "learning_rate": 8.12659936951811e-05, + "loss": 0.1422, + "step": 5305 + }, + { + "epoch": 1.72, + "learning_rate": 8.109414036804026e-05, + "loss": 0.1355, + "step": 5310 + }, + { + "epoch": 1.72, + "learning_rate": 8.092234492691027e-05, + "loss": 0.1357, + "step": 5315 + }, + { + "epoch": 1.72, + "learning_rate": 8.075060789779494e-05, + "loss": 0.1388, + "step": 5320 + }, + { + "epoch": 1.73, + "learning_rate": 8.057892980651909e-05, + "loss": 0.1349, + "step": 5325 + }, + { + "epoch": 1.73, + "learning_rate": 8.040731117872721e-05, + "loss": 0.1328, + "step": 5330 + }, + { + "epoch": 1.73, + "learning_rate": 8.023575253988165e-05, + "loss": 0.1301, + "step": 5335 + }, + { + "epoch": 1.73, + "learning_rate": 8.00642544152611e-05, + "loss": 0.1376, + "step": 5340 + }, + { + "epoch": 1.73, + "learning_rate": 7.989281732995895e-05, + "loss": 0.1389, + "step": 5345 + }, + { + "epoch": 1.73, + "learning_rate": 7.972144180888177e-05, + "loss": 0.1337, + "step": 5350 + }, + { + "epoch": 1.74, + "learning_rate": 7.955012837674753e-05, + "loss": 0.141, + "step": 5355 + }, + { + "epoch": 1.74, + "learning_rate": 7.93788775580842e-05, + "loss": 0.1456, + "step": 5360 + }, + { + "epoch": 1.74, + "learning_rate": 7.920768987722791e-05, + "loss": 0.1459, + "step": 5365 + }, + { + "epoch": 1.74, + "learning_rate": 7.903656585832162e-05, + "loss": 0.1332, + "step": 5370 + }, + { + "epoch": 1.74, + "learning_rate": 7.886550602531331e-05, + "loss": 0.1445, + "step": 5375 + }, + { + "epoch": 1.74, + "learning_rate": 7.869451090195437e-05, + "loss": 0.1349, + "step": 5380 + }, + { + "epoch": 1.75, + "learning_rate": 7.852358101179817e-05, + "loss": 0.1327, + "step": 5385 + }, + { + "epoch": 1.75, + "learning_rate": 7.835271687819827e-05, + "loss": 0.1305, + "step": 5390 + }, + { + "epoch": 1.75, + "learning_rate": 7.818191902430696e-05, + "loss": 0.1446, + "step": 5395 + }, + { + "epoch": 1.75, + "learning_rate": 7.801118797307349e-05, + "loss": 0.139, + "step": 5400 + }, + { + "epoch": 1.75, + "learning_rate": 7.784052424724276e-05, + "loss": 0.1432, + "step": 5405 + }, + { + "epoch": 1.75, + "learning_rate": 7.766992836935327e-05, + "loss": 0.1439, + "step": 5410 + }, + { + "epoch": 1.75, + "learning_rate": 7.749940086173607e-05, + "loss": 0.1415, + "step": 5415 + }, + { + "epoch": 1.76, + "learning_rate": 7.732894224651261e-05, + "loss": 0.1404, + "step": 5420 + }, + { + "epoch": 1.76, + "learning_rate": 7.715855304559362e-05, + "loss": 0.1415, + "step": 5425 + }, + { + "epoch": 1.76, + "learning_rate": 7.698823378067714e-05, + "loss": 0.142, + "step": 5430 + }, + { + "epoch": 1.76, + "learning_rate": 7.681798497324716e-05, + "loss": 0.1428, + "step": 5435 + }, + { + "epoch": 1.76, + "learning_rate": 7.664780714457198e-05, + "loss": 0.1405, + "step": 5440 + }, + { + "epoch": 1.76, + "learning_rate": 7.647770081570246e-05, + "loss": 0.1355, + "step": 5445 + }, + { + "epoch": 1.77, + "learning_rate": 7.630766650747066e-05, + "loss": 0.1371, + "step": 5450 + }, + { + "epoch": 1.77, + "learning_rate": 7.613770474048802e-05, + "loss": 0.1388, + "step": 5455 + }, + { + "epoch": 1.77, + "learning_rate": 7.596781603514398e-05, + "loss": 0.1306, + "step": 5460 + }, + { + "epoch": 1.77, + "learning_rate": 7.579800091160417e-05, + "loss": 0.1351, + "step": 5465 + }, + { + "epoch": 1.77, + "learning_rate": 7.562825988980904e-05, + "loss": 0.1322, + "step": 5470 + }, + { + "epoch": 1.77, + "learning_rate": 7.545859348947204e-05, + "loss": 0.1389, + "step": 5475 + }, + { + "epoch": 1.78, + "learning_rate": 7.528900223007824e-05, + "loss": 0.1432, + "step": 5480 + }, + { + "epoch": 1.78, + "learning_rate": 7.511948663088261e-05, + "loss": 0.1359, + "step": 5485 + }, + { + "epoch": 1.78, + "learning_rate": 7.495004721090843e-05, + "loss": 0.1411, + "step": 5490 + }, + { + "epoch": 1.78, + "learning_rate": 7.478068448894577e-05, + "loss": 0.1373, + "step": 5495 + }, + { + "epoch": 1.78, + "learning_rate": 7.461139898354986e-05, + "loss": 0.1406, + "step": 5500 + }, + { + "epoch": 1.78, + "learning_rate": 7.444219121303957e-05, + "loss": 0.1388, + "step": 5505 + }, + { + "epoch": 1.79, + "learning_rate": 7.42730616954956e-05, + "loss": 0.1344, + "step": 5510 + }, + { + "epoch": 1.79, + "learning_rate": 7.410401094875921e-05, + "loss": 0.1416, + "step": 5515 + }, + { + "epoch": 1.79, + "learning_rate": 7.393503949043039e-05, + "loss": 0.1417, + "step": 5520 + }, + { + "epoch": 1.79, + "learning_rate": 7.376614783786646e-05, + "loss": 0.1425, + "step": 5525 + }, + { + "epoch": 1.79, + "learning_rate": 7.359733650818024e-05, + "loss": 0.1326, + "step": 5530 + }, + { + "epoch": 1.79, + "learning_rate": 7.342860601823875e-05, + "loss": 0.1449, + "step": 5535 + }, + { + "epoch": 1.8, + "learning_rate": 7.325995688466146e-05, + "loss": 0.1339, + "step": 5540 + }, + { + "epoch": 1.8, + "learning_rate": 7.309138962381872e-05, + "loss": 0.1341, + "step": 5545 + }, + { + "epoch": 1.8, + "learning_rate": 7.29229047518302e-05, + "loss": 0.1324, + "step": 5550 + }, + { + "epoch": 1.8, + "learning_rate": 7.275450278456335e-05, + "loss": 0.129, + "step": 5555 + }, + { + "epoch": 1.8, + "learning_rate": 7.258618423763171e-05, + "loss": 0.1388, + "step": 5560 + }, + { + "epoch": 1.8, + "learning_rate": 7.24179496263935e-05, + "loss": 0.1406, + "step": 5565 + }, + { + "epoch": 1.8, + "learning_rate": 7.224979946594992e-05, + "loss": 0.138, + "step": 5570 + }, + { + "epoch": 1.81, + "learning_rate": 7.20817342711435e-05, + "loss": 0.1412, + "step": 5575 + }, + { + "epoch": 1.81, + "learning_rate": 7.191375455655676e-05, + "loss": 0.1377, + "step": 5580 + }, + { + "epoch": 1.81, + "learning_rate": 7.174586083651038e-05, + "loss": 0.1387, + "step": 5585 + }, + { + "epoch": 1.81, + "learning_rate": 7.157805362506184e-05, + "loss": 0.1352, + "step": 5590 + }, + { + "epoch": 1.81, + "learning_rate": 7.14103334360037e-05, + "loss": 0.1457, + "step": 5595 + }, + { + "epoch": 1.81, + "learning_rate": 7.124270078286203e-05, + "loss": 0.1359, + "step": 5600 + }, + { + "epoch": 1.82, + "learning_rate": 7.107515617889498e-05, + "loss": 0.138, + "step": 5605 + }, + { + "epoch": 1.82, + "learning_rate": 7.090770013709105e-05, + "loss": 0.1402, + "step": 5610 + }, + { + "epoch": 1.82, + "learning_rate": 7.074033317016755e-05, + "loss": 0.133, + "step": 5615 + }, + { + "epoch": 1.82, + "learning_rate": 7.057305579056919e-05, + "loss": 0.1419, + "step": 5620 + }, + { + "epoch": 1.82, + "learning_rate": 7.040586851046618e-05, + "loss": 0.1282, + "step": 5625 + }, + { + "epoch": 1.82, + "learning_rate": 7.023877184175307e-05, + "loss": 0.1393, + "step": 5630 + }, + { + "epoch": 1.83, + "learning_rate": 7.007176629604681e-05, + "loss": 0.1273, + "step": 5635 + }, + { + "epoch": 1.83, + "learning_rate": 6.990485238468545e-05, + "loss": 0.1417, + "step": 5640 + }, + { + "epoch": 1.83, + "learning_rate": 6.973803061872646e-05, + "loss": 0.1376, + "step": 5645 + }, + { + "epoch": 1.83, + "learning_rate": 6.957130150894516e-05, + "loss": 0.1421, + "step": 5650 + }, + { + "epoch": 1.83, + "learning_rate": 6.940466556583316e-05, + "loss": 0.1311, + "step": 5655 + }, + { + "epoch": 1.83, + "learning_rate": 6.923812329959683e-05, + "loss": 0.1376, + "step": 5660 + }, + { + "epoch": 1.84, + "learning_rate": 6.907167522015577e-05, + "loss": 0.1375, + "step": 5665 + }, + { + "epoch": 1.84, + "learning_rate": 6.890532183714105e-05, + "loss": 0.1304, + "step": 5670 + }, + { + "epoch": 1.84, + "learning_rate": 6.8739063659894e-05, + "loss": 0.1358, + "step": 5675 + }, + { + "epoch": 1.84, + "learning_rate": 6.857290119746425e-05, + "loss": 0.1418, + "step": 5680 + }, + { + "epoch": 1.84, + "learning_rate": 6.840683495860857e-05, + "loss": 0.1367, + "step": 5685 + }, + { + "epoch": 1.84, + "learning_rate": 6.82408654517889e-05, + "loss": 0.1373, + "step": 5690 + }, + { + "epoch": 1.85, + "learning_rate": 6.80749931851712e-05, + "loss": 0.1408, + "step": 5695 + }, + { + "epoch": 1.85, + "learning_rate": 6.790921866662355e-05, + "loss": 0.1326, + "step": 5700 + }, + { + "epoch": 1.85, + "learning_rate": 6.774354240371487e-05, + "loss": 0.137, + "step": 5705 + }, + { + "epoch": 1.85, + "learning_rate": 6.757796490371314e-05, + "loss": 0.1509, + "step": 5710 + }, + { + "epoch": 1.85, + "learning_rate": 6.741248667358397e-05, + "loss": 0.1317, + "step": 5715 + }, + { + "epoch": 1.85, + "learning_rate": 6.724710821998912e-05, + "loss": 0.1395, + "step": 5720 + }, + { + "epoch": 1.86, + "learning_rate": 6.708183004928468e-05, + "loss": 0.1359, + "step": 5725 + }, + { + "epoch": 1.86, + "learning_rate": 6.691665266751985e-05, + "loss": 0.1448, + "step": 5730 + }, + { + "epoch": 1.86, + "learning_rate": 6.675157658043513e-05, + "loss": 0.1365, + "step": 5735 + }, + { + "epoch": 1.86, + "learning_rate": 6.658660229346097e-05, + "loss": 0.1341, + "step": 5740 + }, + { + "epoch": 1.86, + "learning_rate": 6.642173031171603e-05, + "loss": 0.137, + "step": 5745 + }, + { + "epoch": 1.86, + "learning_rate": 6.62569611400058e-05, + "loss": 0.1376, + "step": 5750 + }, + { + "epoch": 1.86, + "learning_rate": 6.609229528282094e-05, + "loss": 0.1297, + "step": 5755 + }, + { + "epoch": 1.87, + "learning_rate": 6.592773324433588e-05, + "loss": 0.1344, + "step": 5760 + }, + { + "epoch": 1.87, + "learning_rate": 6.5763275528407e-05, + "loss": 0.1359, + "step": 5765 + }, + { + "epoch": 1.87, + "learning_rate": 6.559892263857144e-05, + "loss": 0.1302, + "step": 5770 + }, + { + "epoch": 1.87, + "learning_rate": 6.54346750780453e-05, + "loss": 0.132, + "step": 5775 + }, + { + "epoch": 1.87, + "learning_rate": 6.527053334972218e-05, + "loss": 0.1349, + "step": 5780 + }, + { + "epoch": 1.87, + "learning_rate": 6.510649795617168e-05, + "loss": 0.1406, + "step": 5785 + }, + { + "epoch": 1.88, + "learning_rate": 6.494256939963775e-05, + "loss": 0.1345, + "step": 5790 + }, + { + "epoch": 1.88, + "learning_rate": 6.477874818203736e-05, + "loss": 0.1392, + "step": 5795 + }, + { + "epoch": 1.88, + "learning_rate": 6.461503480495864e-05, + "loss": 0.1359, + "step": 5800 + }, + { + "epoch": 1.88, + "learning_rate": 6.445142976965972e-05, + "loss": 0.1284, + "step": 5805 + }, + { + "epoch": 1.88, + "learning_rate": 6.428793357706689e-05, + "loss": 0.1343, + "step": 5810 + }, + { + "epoch": 1.88, + "learning_rate": 6.412454672777328e-05, + "loss": 0.1387, + "step": 5815 + }, + { + "epoch": 1.89, + "learning_rate": 6.396126972203708e-05, + "loss": 0.1334, + "step": 5820 + }, + { + "epoch": 1.89, + "learning_rate": 6.379810305978038e-05, + "loss": 0.1329, + "step": 5825 + }, + { + "epoch": 1.89, + "learning_rate": 6.363504724058717e-05, + "loss": 0.1378, + "step": 5830 + }, + { + "epoch": 1.89, + "learning_rate": 6.347210276370224e-05, + "loss": 0.1358, + "step": 5835 + }, + { + "epoch": 1.89, + "learning_rate": 6.330927012802947e-05, + "loss": 0.1412, + "step": 5840 + }, + { + "epoch": 1.89, + "learning_rate": 6.314654983213018e-05, + "loss": 0.1308, + "step": 5845 + }, + { + "epoch": 1.9, + "learning_rate": 6.298394237422186e-05, + "loss": 0.131, + "step": 5850 + }, + { + "epoch": 1.9, + "learning_rate": 6.282144825217639e-05, + "loss": 0.1341, + "step": 5855 + }, + { + "epoch": 1.9, + "learning_rate": 6.265906796351875e-05, + "loss": 0.1435, + "step": 5860 + }, + { + "epoch": 1.9, + "learning_rate": 6.24968020054253e-05, + "loss": 0.1417, + "step": 5865 + }, + { + "epoch": 1.9, + "learning_rate": 6.233465087472244e-05, + "loss": 0.1376, + "step": 5870 + }, + { + "epoch": 1.9, + "learning_rate": 6.217261506788485e-05, + "loss": 0.1268, + "step": 5875 + }, + { + "epoch": 1.91, + "learning_rate": 6.201069508103426e-05, + "loss": 0.1389, + "step": 5880 + }, + { + "epoch": 1.91, + "learning_rate": 6.184889140993765e-05, + "loss": 0.14, + "step": 5885 + }, + { + "epoch": 1.91, + "learning_rate": 6.168720455000601e-05, + "loss": 0.1394, + "step": 5890 + }, + { + "epoch": 1.91, + "learning_rate": 6.152563499629249e-05, + "loss": 0.1413, + "step": 5895 + }, + { + "epoch": 1.91, + "learning_rate": 6.136418324349125e-05, + "loss": 0.1354, + "step": 5900 + }, + { + "epoch": 1.91, + "learning_rate": 6.120284978593569e-05, + "loss": 0.1335, + "step": 5905 + }, + { + "epoch": 1.92, + "learning_rate": 6.104163511759696e-05, + "loss": 0.1381, + "step": 5910 + }, + { + "epoch": 1.92, + "learning_rate": 6.088053973208262e-05, + "loss": 0.1313, + "step": 5915 + }, + { + "epoch": 1.92, + "learning_rate": 6.071956412263491e-05, + "loss": 0.1314, + "step": 5920 + }, + { + "epoch": 1.92, + "learning_rate": 6.0558708782129416e-05, + "loss": 0.1432, + "step": 5925 + }, + { + "epoch": 1.92, + "learning_rate": 6.039797420307338e-05, + "loss": 0.1398, + "step": 5930 + }, + { + "epoch": 1.92, + "learning_rate": 6.023736087760443e-05, + "loss": 0.1334, + "step": 5935 + }, + { + "epoch": 1.92, + "learning_rate": 6.00768692974888e-05, + "loss": 0.1393, + "step": 5940 + }, + { + "epoch": 1.93, + "learning_rate": 5.991649995412011e-05, + "loss": 0.1319, + "step": 5945 + }, + { + "epoch": 1.93, + "learning_rate": 5.975625333851755e-05, + "loss": 0.1304, + "step": 5950 + }, + { + "epoch": 1.93, + "learning_rate": 5.9596129941324705e-05, + "loss": 0.1332, + "step": 5955 + }, + { + "epoch": 1.93, + "learning_rate": 5.9436130252807785e-05, + "loss": 0.1344, + "step": 5960 + }, + { + "epoch": 1.93, + "learning_rate": 5.927625476285426e-05, + "loss": 0.1329, + "step": 5965 + }, + { + "epoch": 1.93, + "learning_rate": 5.91165039609713e-05, + "loss": 0.1359, + "step": 5970 + }, + { + "epoch": 1.94, + "learning_rate": 5.895687833628437e-05, + "loss": 0.1311, + "step": 5975 + }, + { + "epoch": 1.94, + "learning_rate": 5.879737837753565e-05, + "loss": 0.1324, + "step": 5980 + }, + { + "epoch": 1.94, + "learning_rate": 5.863800457308246e-05, + "loss": 0.1286, + "step": 5985 + }, + { + "epoch": 1.94, + "learning_rate": 5.847875741089604e-05, + "loss": 0.134, + "step": 5990 + }, + { + "epoch": 1.94, + "learning_rate": 5.8319637378559635e-05, + "loss": 0.1312, + "step": 5995 + }, + { + "epoch": 1.94, + "learning_rate": 5.816064496326752e-05, + "loss": 0.1277, + "step": 6000 + }, + { + "epoch": 1.95, + "learning_rate": 5.800178065182296e-05, + "loss": 0.1377, + "step": 6005 + }, + { + "epoch": 1.95, + "learning_rate": 5.784304493063725e-05, + "loss": 0.138, + "step": 6010 + }, + { + "epoch": 1.95, + "learning_rate": 5.768443828572772e-05, + "loss": 0.1283, + "step": 6015 + }, + { + "epoch": 1.95, + "learning_rate": 5.7525961202716714e-05, + "loss": 0.1307, + "step": 6020 + }, + { + "epoch": 1.95, + "learning_rate": 5.7367614166829706e-05, + "loss": 0.136, + "step": 6025 + }, + { + "epoch": 1.95, + "learning_rate": 5.720939766289415e-05, + "loss": 0.137, + "step": 6030 + }, + { + "epoch": 1.96, + "learning_rate": 5.7051312175337656e-05, + "loss": 0.1341, + "step": 6035 + }, + { + "epoch": 1.96, + "learning_rate": 5.689335818818684e-05, + "loss": 0.1384, + "step": 6040 + }, + { + "epoch": 1.96, + "learning_rate": 5.673553618506562e-05, + "loss": 0.1378, + "step": 6045 + }, + { + "epoch": 1.96, + "learning_rate": 5.65778466491939e-05, + "loss": 0.1348, + "step": 6050 + }, + { + "epoch": 1.96, + "learning_rate": 5.6420290063385805e-05, + "loss": 0.1344, + "step": 6055 + }, + { + "epoch": 1.96, + "learning_rate": 5.626286691004854e-05, + "loss": 0.1354, + "step": 6060 + }, + { + "epoch": 1.97, + "learning_rate": 5.61055776711808e-05, + "loss": 0.1345, + "step": 6065 + }, + { + "epoch": 1.97, + "learning_rate": 5.594842282837109e-05, + "loss": 0.1497, + "step": 6070 + }, + { + "epoch": 1.97, + "learning_rate": 5.5791402862796604e-05, + "loss": 0.1257, + "step": 6075 + }, + { + "epoch": 1.97, + "learning_rate": 5.5634518255221415e-05, + "loss": 0.1384, + "step": 6080 + }, + { + "epoch": 1.97, + "learning_rate": 5.54777694859953e-05, + "loss": 0.1344, + "step": 6085 + }, + { + "epoch": 1.97, + "learning_rate": 5.532115703505198e-05, + "loss": 0.1321, + "step": 6090 + }, + { + "epoch": 1.98, + "learning_rate": 5.516468138190796e-05, + "loss": 0.1389, + "step": 6095 + }, + { + "epoch": 1.98, + "learning_rate": 5.500834300566069e-05, + "loss": 0.1357, + "step": 6100 + }, + { + "epoch": 1.98, + "learning_rate": 5.4852142384987504e-05, + "loss": 0.1333, + "step": 6105 + }, + { + "epoch": 1.98, + "learning_rate": 5.46960799981439e-05, + "loss": 0.1296, + "step": 6110 + }, + { + "epoch": 1.98, + "learning_rate": 5.454015632296203e-05, + "loss": 0.1352, + "step": 6115 + }, + { + "epoch": 1.98, + "learning_rate": 5.438437183684948e-05, + "loss": 0.132, + "step": 6120 + }, + { + "epoch": 1.98, + "learning_rate": 5.422872701678754e-05, + "loss": 0.1382, + "step": 6125 + }, + { + "epoch": 1.99, + "learning_rate": 5.407322233932999e-05, + "loss": 0.1363, + "step": 6130 + }, + { + "epoch": 1.99, + "learning_rate": 5.391785828060139e-05, + "loss": 0.138, + "step": 6135 + }, + { + "epoch": 1.99, + "learning_rate": 5.376263531629591e-05, + "loss": 0.1357, + "step": 6140 + }, + { + "epoch": 1.99, + "learning_rate": 5.360755392167555e-05, + "loss": 0.1376, + "step": 6145 + }, + { + "epoch": 1.99, + "learning_rate": 5.345261457156897e-05, + "loss": 0.1372, + "step": 6150 + }, + { + "epoch": 1.99, + "learning_rate": 5.329781774036988e-05, + "loss": 0.1362, + "step": 6155 + }, + { + "epoch": 2.0, + "learning_rate": 5.3143163902035664e-05, + "loss": 0.1321, + "step": 6160 + }, + { + "epoch": 2.0, + "learning_rate": 5.2988653530085796e-05, + "loss": 0.1423, + "step": 6165 + }, + { + "epoch": 2.0, + "learning_rate": 5.2834287097600545e-05, + "loss": 0.1315, + "step": 6170 + }, + { + "epoch": 2.0, + "learning_rate": 5.268006507721952e-05, + "loss": 0.1351, + "step": 6175 + }, + { + "epoch": 2.0, + "learning_rate": 5.252598794114002e-05, + "loss": 0.13, + "step": 6180 + }, + { + "epoch": 2.0, + "learning_rate": 5.2372056161115925e-05, + "loss": 0.1314, + "step": 6185 + }, + { + "epoch": 2.01, + "learning_rate": 5.2218270208455864e-05, + "loss": 0.1272, + "step": 6190 + }, + { + "epoch": 2.01, + "learning_rate": 5.206463055402218e-05, + "loss": 0.1357, + "step": 6195 + }, + { + "epoch": 2.01, + "learning_rate": 5.191113766822905e-05, + "loss": 0.132, + "step": 6200 + }, + { + "epoch": 2.01, + "learning_rate": 5.175779202104152e-05, + "loss": 0.1382, + "step": 6205 + }, + { + "epoch": 2.01, + "learning_rate": 5.160459408197358e-05, + "loss": 0.1361, + "step": 6210 + }, + { + "epoch": 2.01, + "learning_rate": 5.1451544320087185e-05, + "loss": 0.1371, + "step": 6215 + }, + { + "epoch": 2.02, + "learning_rate": 5.1298643203990424e-05, + "loss": 0.1347, + "step": 6220 + }, + { + "epoch": 2.02, + "learning_rate": 5.1145891201836414e-05, + "loss": 0.1325, + "step": 6225 + }, + { + "epoch": 2.02, + "learning_rate": 5.099328878132156e-05, + "loss": 0.1311, + "step": 6230 + }, + { + "epoch": 2.02, + "learning_rate": 5.084083640968442e-05, + "loss": 0.1396, + "step": 6235 + }, + { + "epoch": 2.02, + "learning_rate": 5.0688534553703995e-05, + "loss": 0.1322, + "step": 6240 + }, + { + "epoch": 2.02, + "learning_rate": 5.053638367969855e-05, + "loss": 0.1404, + "step": 6245 + }, + { + "epoch": 2.03, + "learning_rate": 5.038438425352405e-05, + "loss": 0.1383, + "step": 6250 + }, + { + "epoch": 2.03, + "learning_rate": 5.023253674057275e-05, + "loss": 0.1332, + "step": 6255 + }, + { + "epoch": 2.03, + "learning_rate": 5.008084160577171e-05, + "loss": 0.1369, + "step": 6260 + }, + { + "epoch": 2.03, + "learning_rate": 4.992929931358151e-05, + "loss": 0.1399, + "step": 6265 + }, + { + "epoch": 2.03, + "learning_rate": 4.9777910327994804e-05, + "loss": 0.1348, + "step": 6270 + }, + { + "epoch": 2.03, + "learning_rate": 4.962667511253469e-05, + "loss": 0.1333, + "step": 6275 + }, + { + "epoch": 2.04, + "learning_rate": 4.9475594130253645e-05, + "loss": 0.133, + "step": 6280 + }, + { + "epoch": 2.04, + "learning_rate": 4.932466784373172e-05, + "loss": 0.1312, + "step": 6285 + }, + { + "epoch": 2.04, + "learning_rate": 4.917389671507551e-05, + "loss": 0.1302, + "step": 6290 + }, + { + "epoch": 2.04, + "learning_rate": 4.902328120591637e-05, + "loss": 0.1337, + "step": 6295 + }, + { + "epoch": 2.04, + "learning_rate": 4.887282177740935e-05, + "loss": 0.1316, + "step": 6300 + }, + { + "epoch": 2.04, + "learning_rate": 4.872251889023144e-05, + "loss": 0.1324, + "step": 6305 + }, + { + "epoch": 2.04, + "learning_rate": 4.8572373004580465e-05, + "loss": 0.1344, + "step": 6310 + }, + { + "epoch": 2.05, + "learning_rate": 4.8422384580173505e-05, + "loss": 0.1281, + "step": 6315 + }, + { + "epoch": 2.05, + "learning_rate": 4.827255407624547e-05, + "loss": 0.1349, + "step": 6320 + }, + { + "epoch": 2.05, + "learning_rate": 4.812288195154785e-05, + "loss": 0.1305, + "step": 6325 + }, + { + "epoch": 2.05, + "learning_rate": 4.7973368664347086e-05, + "loss": 0.1273, + "step": 6330 + }, + { + "epoch": 2.05, + "learning_rate": 4.7824014672423414e-05, + "loss": 0.1309, + "step": 6335 + }, + { + "epoch": 2.05, + "learning_rate": 4.7674820433069224e-05, + "loss": 0.1333, + "step": 6340 + }, + { + "epoch": 2.06, + "learning_rate": 4.752578640308789e-05, + "loss": 0.1411, + "step": 6345 + }, + { + "epoch": 2.06, + "learning_rate": 4.7376913038792136e-05, + "loss": 0.1309, + "step": 6350 + }, + { + "epoch": 2.06, + "learning_rate": 4.7228200796002854e-05, + "loss": 0.1262, + "step": 6355 + }, + { + "epoch": 2.06, + "learning_rate": 4.707965013004758e-05, + "loss": 0.1319, + "step": 6360 + }, + { + "epoch": 2.06, + "learning_rate": 4.6931261495759184e-05, + "loss": 0.1297, + "step": 6365 + }, + { + "epoch": 2.06, + "learning_rate": 4.678303534747428e-05, + "loss": 0.1313, + "step": 6370 + }, + { + "epoch": 2.07, + "learning_rate": 4.663497213903213e-05, + "loss": 0.1244, + "step": 6375 + }, + { + "epoch": 2.07, + "learning_rate": 4.6487072323773093e-05, + "loss": 0.1292, + "step": 6380 + }, + { + "epoch": 2.07, + "learning_rate": 4.633933635453716e-05, + "loss": 0.1327, + "step": 6385 + }, + { + "epoch": 2.07, + "learning_rate": 4.6191764683662744e-05, + "loss": 0.1335, + "step": 6390 + }, + { + "epoch": 2.07, + "learning_rate": 4.6044357762985125e-05, + "loss": 0.1342, + "step": 6395 + }, + { + "epoch": 2.07, + "learning_rate": 4.589711604383528e-05, + "loss": 0.1348, + "step": 6400 + }, + { + "epoch": 2.08, + "learning_rate": 4.575003997703821e-05, + "loss": 0.1344, + "step": 6405 + }, + { + "epoch": 2.08, + "learning_rate": 4.560313001291188e-05, + "loss": 0.125, + "step": 6410 + }, + { + "epoch": 2.08, + "learning_rate": 4.5456386601265534e-05, + "loss": 0.1425, + "step": 6415 + }, + { + "epoch": 2.08, + "learning_rate": 4.530981019139862e-05, + "loss": 0.1286, + "step": 6420 + }, + { + "epoch": 2.08, + "learning_rate": 4.516340123209908e-05, + "loss": 0.1286, + "step": 6425 + }, + { + "epoch": 2.08, + "learning_rate": 4.5017160171642333e-05, + "loss": 0.1347, + "step": 6430 + }, + { + "epoch": 2.09, + "learning_rate": 4.4871087457789584e-05, + "loss": 0.1301, + "step": 6435 + }, + { + "epoch": 2.09, + "learning_rate": 4.4725183537786683e-05, + "loss": 0.1314, + "step": 6440 + }, + { + "epoch": 2.09, + "learning_rate": 4.457944885836267e-05, + "loss": 0.1341, + "step": 6445 + }, + { + "epoch": 2.09, + "learning_rate": 4.44338838657283e-05, + "loss": 0.1347, + "step": 6450 + }, + { + "epoch": 2.09, + "learning_rate": 4.428848900557492e-05, + "loss": 0.1308, + "step": 6455 + }, + { + "epoch": 2.09, + "learning_rate": 4.414326472307283e-05, + "loss": 0.1395, + "step": 6460 + }, + { + "epoch": 2.09, + "learning_rate": 4.3998211462870165e-05, + "loss": 0.1369, + "step": 6465 + }, + { + "epoch": 2.1, + "learning_rate": 4.385332966909136e-05, + "loss": 0.1334, + "step": 6470 + }, + { + "epoch": 2.1, + "learning_rate": 4.3708619785335923e-05, + "loss": 0.1296, + "step": 6475 + }, + { + "epoch": 2.1, + "learning_rate": 4.356408225467688e-05, + "loss": 0.1301, + "step": 6480 + }, + { + "epoch": 2.1, + "learning_rate": 4.3419717519659694e-05, + "loss": 0.125, + "step": 6485 + }, + { + "epoch": 2.1, + "learning_rate": 4.327552602230063e-05, + "loss": 0.1251, + "step": 6490 + }, + { + "epoch": 2.1, + "learning_rate": 4.313150820408566e-05, + "loss": 0.137, + "step": 6495 + }, + { + "epoch": 2.11, + "learning_rate": 4.298766450596885e-05, + "loss": 0.126, + "step": 6500 + }, + { + "epoch": 2.11, + "learning_rate": 4.284399536837124e-05, + "loss": 0.1331, + "step": 6505 + }, + { + "epoch": 2.11, + "learning_rate": 4.2700501231179427e-05, + "loss": 0.1322, + "step": 6510 + }, + { + "epoch": 2.11, + "learning_rate": 4.255718253374405e-05, + "loss": 0.1292, + "step": 6515 + }, + { + "epoch": 2.11, + "learning_rate": 4.241403971487876e-05, + "loss": 0.1354, + "step": 6520 + }, + { + "epoch": 2.11, + "learning_rate": 4.227107321285855e-05, + "loss": 0.1264, + "step": 6525 + }, + { + "epoch": 2.12, + "learning_rate": 4.212828346541868e-05, + "loss": 0.1336, + "step": 6530 + }, + { + "epoch": 2.12, + "learning_rate": 4.198567090975314e-05, + "loss": 0.1218, + "step": 6535 + }, + { + "epoch": 2.12, + "learning_rate": 4.1843235982513463e-05, + "loss": 0.1244, + "step": 6540 + }, + { + "epoch": 2.12, + "learning_rate": 4.1700979119807235e-05, + "loss": 0.1354, + "step": 6545 + }, + { + "epoch": 2.12, + "learning_rate": 4.1558900757196964e-05, + "loss": 0.1231, + "step": 6550 + }, + { + "epoch": 2.12, + "learning_rate": 4.141700132969848e-05, + "loss": 0.133, + "step": 6555 + }, + { + "epoch": 2.13, + "learning_rate": 4.1275281271779854e-05, + "loss": 0.1297, + "step": 6560 + }, + { + "epoch": 2.13, + "learning_rate": 4.113374101735998e-05, + "loss": 0.1359, + "step": 6565 + }, + { + "epoch": 2.13, + "learning_rate": 4.099238099980711e-05, + "loss": 0.1251, + "step": 6570 + }, + { + "epoch": 2.13, + "learning_rate": 4.0851201651937755e-05, + "loss": 0.1291, + "step": 6575 + }, + { + "epoch": 2.13, + "learning_rate": 4.0710203406015204e-05, + "loss": 0.1278, + "step": 6580 + }, + { + "epoch": 2.13, + "learning_rate": 4.05693866937483e-05, + "loss": 0.1191, + "step": 6585 + }, + { + "epoch": 2.14, + "learning_rate": 4.0428751946289956e-05, + "loss": 0.1256, + "step": 6590 + }, + { + "epoch": 2.14, + "learning_rate": 4.028829959423608e-05, + "loss": 0.1299, + "step": 6595 + }, + { + "epoch": 2.14, + "learning_rate": 4.014803006762398e-05, + "loss": 0.131, + "step": 6600 + }, + { + "epoch": 2.14, + "learning_rate": 4.0007943795931346e-05, + "loss": 0.1292, + "step": 6605 + }, + { + "epoch": 2.14, + "learning_rate": 3.9868041208074617e-05, + "loss": 0.1311, + "step": 6610 + }, + { + "epoch": 2.14, + "learning_rate": 3.972832273240796e-05, + "loss": 0.1273, + "step": 6615 + }, + { + "epoch": 2.15, + "learning_rate": 3.958878879672172e-05, + "loss": 0.1352, + "step": 6620 + }, + { + "epoch": 2.15, + "learning_rate": 3.9449439828241354e-05, + "loss": 0.129, + "step": 6625 + }, + { + "epoch": 2.15, + "learning_rate": 3.93102762536258e-05, + "loss": 0.1264, + "step": 6630 + }, + { + "epoch": 2.15, + "learning_rate": 3.917129849896653e-05, + "loss": 0.1248, + "step": 6635 + }, + { + "epoch": 2.15, + "learning_rate": 3.903250698978596e-05, + "loss": 0.1164, + "step": 6640 + }, + { + "epoch": 2.15, + "learning_rate": 3.889390215103631e-05, + "loss": 0.13, + "step": 6645 + }, + { + "epoch": 2.15, + "learning_rate": 3.875548440709827e-05, + "loss": 0.131, + "step": 6650 + }, + { + "epoch": 2.16, + "learning_rate": 3.861725418177961e-05, + "loss": 0.1253, + "step": 6655 + }, + { + "epoch": 2.16, + "learning_rate": 3.8479211898314035e-05, + "loss": 0.129, + "step": 6660 + }, + { + "epoch": 2.16, + "learning_rate": 3.8341357979359735e-05, + "loss": 0.1304, + "step": 6665 + }, + { + "epoch": 2.16, + "learning_rate": 3.820369284699823e-05, + "loss": 0.1252, + "step": 6670 + }, + { + "epoch": 2.16, + "learning_rate": 3.8066216922733e-05, + "loss": 0.129, + "step": 6675 + }, + { + "epoch": 2.16, + "learning_rate": 3.792893062748814e-05, + "loss": 0.1261, + "step": 6680 + }, + { + "epoch": 2.17, + "learning_rate": 3.7791834381607214e-05, + "loss": 0.1278, + "step": 6685 + }, + { + "epoch": 2.17, + "learning_rate": 3.7654928604851924e-05, + "loss": 0.1214, + "step": 6690 + }, + { + "epoch": 2.17, + "learning_rate": 3.751821371640065e-05, + "loss": 0.1227, + "step": 6695 + }, + { + "epoch": 2.17, + "learning_rate": 3.7381690134847483e-05, + "loss": 0.1272, + "step": 6700 + }, + { + "epoch": 2.17, + "learning_rate": 3.724535827820059e-05, + "loss": 0.1309, + "step": 6705 + }, + { + "epoch": 2.17, + "learning_rate": 3.7109218563881274e-05, + "loss": 0.1386, + "step": 6710 + }, + { + "epoch": 2.18, + "learning_rate": 3.6973271408722496e-05, + "loss": 0.1272, + "step": 6715 + }, + { + "epoch": 2.18, + "learning_rate": 3.6837517228967535e-05, + "loss": 0.1323, + "step": 6720 + }, + { + "epoch": 2.18, + "learning_rate": 3.670195644026898e-05, + "loss": 0.1259, + "step": 6725 + }, + { + "epoch": 2.18, + "learning_rate": 3.656658945768713e-05, + "loss": 0.131, + "step": 6730 + }, + { + "epoch": 2.18, + "learning_rate": 3.6431416695689046e-05, + "loss": 0.1209, + "step": 6735 + }, + { + "epoch": 2.18, + "learning_rate": 3.6296438568146974e-05, + "loss": 0.131, + "step": 6740 + }, + { + "epoch": 2.19, + "learning_rate": 3.616165548833738e-05, + "loss": 0.1306, + "step": 6745 + }, + { + "epoch": 2.19, + "learning_rate": 3.602706786893937e-05, + "loss": 0.1346, + "step": 6750 + }, + { + "epoch": 2.19, + "learning_rate": 3.589267612203374e-05, + "loss": 0.1231, + "step": 6755 + }, + { + "epoch": 2.19, + "learning_rate": 3.575848065910145e-05, + "loss": 0.1281, + "step": 6760 + }, + { + "epoch": 2.19, + "learning_rate": 3.5624481891022565e-05, + "loss": 0.1325, + "step": 6765 + }, + { + "epoch": 2.19, + "learning_rate": 3.54906802280748e-05, + "loss": 0.1274, + "step": 6770 + }, + { + "epoch": 2.2, + "learning_rate": 3.5357076079932495e-05, + "loss": 0.1223, + "step": 6775 + }, + { + "epoch": 2.2, + "learning_rate": 3.522366985566518e-05, + "loss": 0.1292, + "step": 6780 + }, + { + "epoch": 2.2, + "learning_rate": 3.509046196373639e-05, + "loss": 0.1293, + "step": 6785 + }, + { + "epoch": 2.2, + "learning_rate": 3.4957452812002355e-05, + "loss": 0.1217, + "step": 6790 + }, + { + "epoch": 2.2, + "learning_rate": 3.482464280771087e-05, + "loss": 0.1279, + "step": 6795 + }, + { + "epoch": 2.2, + "learning_rate": 3.4692032357500007e-05, + "loss": 0.1278, + "step": 6800 + }, + { + "epoch": 2.21, + "learning_rate": 3.455962186739671e-05, + "loss": 0.1266, + "step": 6805 + }, + { + "epoch": 2.21, + "learning_rate": 3.442741174281584e-05, + "loss": 0.1307, + "step": 6810 + }, + { + "epoch": 2.21, + "learning_rate": 3.429540238855863e-05, + "loss": 0.1263, + "step": 6815 + }, + { + "epoch": 2.21, + "learning_rate": 3.416359420881176e-05, + "loss": 0.1232, + "step": 6820 + }, + { + "epoch": 2.21, + "learning_rate": 3.4031987607145756e-05, + "loss": 0.1277, + "step": 6825 + }, + { + "epoch": 2.21, + "learning_rate": 3.390058298651415e-05, + "loss": 0.1179, + "step": 6830 + }, + { + "epoch": 2.21, + "learning_rate": 3.37693807492519e-05, + "loss": 0.1187, + "step": 6835 + }, + { + "epoch": 2.22, + "learning_rate": 3.363838129707437e-05, + "loss": 0.1285, + "step": 6840 + }, + { + "epoch": 2.22, + "learning_rate": 3.3507585031076083e-05, + "loss": 0.1241, + "step": 6845 + }, + { + "epoch": 2.22, + "learning_rate": 3.337699235172931e-05, + "loss": 0.133, + "step": 6850 + }, + { + "epoch": 2.22, + "learning_rate": 3.324660365888313e-05, + "loss": 0.1239, + "step": 6855 + }, + { + "epoch": 2.22, + "learning_rate": 3.311641935176192e-05, + "loss": 0.1297, + "step": 6860 + }, + { + "epoch": 2.22, + "learning_rate": 3.298643982896441e-05, + "loss": 0.1238, + "step": 6865 + }, + { + "epoch": 2.23, + "learning_rate": 3.285666548846217e-05, + "loss": 0.1242, + "step": 6870 + }, + { + "epoch": 2.23, + "learning_rate": 3.2727096727598696e-05, + "loss": 0.1297, + "step": 6875 + }, + { + "epoch": 2.23, + "learning_rate": 3.259773394308787e-05, + "loss": 0.1292, + "step": 6880 + }, + { + "epoch": 2.23, + "learning_rate": 3.246857753101308e-05, + "loss": 0.1298, + "step": 6885 + }, + { + "epoch": 2.23, + "learning_rate": 3.233962788682573e-05, + "loss": 0.1307, + "step": 6890 + }, + { + "epoch": 2.23, + "learning_rate": 3.2210885405344235e-05, + "loss": 0.1339, + "step": 6895 + }, + { + "epoch": 2.24, + "learning_rate": 3.20823504807526e-05, + "loss": 0.1233, + "step": 6900 + }, + { + "epoch": 2.24, + "learning_rate": 3.195402350659945e-05, + "loss": 0.1192, + "step": 6905 + }, + { + "epoch": 2.24, + "learning_rate": 3.182590487579659e-05, + "loss": 0.1307, + "step": 6910 + }, + { + "epoch": 2.24, + "learning_rate": 3.169799498061803e-05, + "loss": 0.1216, + "step": 6915 + }, + { + "epoch": 2.24, + "learning_rate": 3.157029421269863e-05, + "loss": 0.1245, + "step": 6920 + }, + { + "epoch": 2.24, + "learning_rate": 3.144280296303289e-05, + "loss": 0.1293, + "step": 6925 + }, + { + "epoch": 2.25, + "learning_rate": 3.1315521621973934e-05, + "loss": 0.122, + "step": 6930 + }, + { + "epoch": 2.25, + "learning_rate": 3.1188450579232007e-05, + "loss": 0.1244, + "step": 6935 + }, + { + "epoch": 2.25, + "learning_rate": 3.106159022387365e-05, + "loss": 0.1249, + "step": 6940 + }, + { + "epoch": 2.25, + "learning_rate": 3.093494094432018e-05, + "loss": 0.1282, + "step": 6945 + }, + { + "epoch": 2.25, + "learning_rate": 3.080850312834673e-05, + "loss": 0.128, + "step": 6950 + }, + { + "epoch": 2.25, + "learning_rate": 3.06822771630809e-05, + "loss": 0.1277, + "step": 6955 + }, + { + "epoch": 2.26, + "learning_rate": 3.055626343500173e-05, + "loss": 0.124, + "step": 6960 + }, + { + "epoch": 2.26, + "learning_rate": 3.0430462329938326e-05, + "loss": 0.1308, + "step": 6965 + }, + { + "epoch": 2.26, + "learning_rate": 3.0304874233068912e-05, + "loss": 0.1276, + "step": 6970 + }, + { + "epoch": 2.26, + "learning_rate": 3.01794995289194e-05, + "loss": 0.1255, + "step": 6975 + }, + { + "epoch": 2.26, + "learning_rate": 3.0054338601362396e-05, + "loss": 0.1222, + "step": 6980 + }, + { + "epoch": 2.26, + "learning_rate": 2.9929391833616003e-05, + "loss": 0.1231, + "step": 6985 + }, + { + "epoch": 2.27, + "learning_rate": 2.9804659608242512e-05, + "loss": 0.1263, + "step": 6990 + }, + { + "epoch": 2.27, + "learning_rate": 2.9680142307147397e-05, + "loss": 0.1235, + "step": 6995 + }, + { + "epoch": 2.27, + "learning_rate": 2.9555840311578066e-05, + "loss": 0.1241, + "step": 7000 + }, + { + "epoch": 2.27, + "learning_rate": 2.9431754002122715e-05, + "loss": 0.1274, + "step": 7005 + }, + { + "epoch": 2.27, + "learning_rate": 2.9307883758709086e-05, + "loss": 0.1318, + "step": 7010 + }, + { + "epoch": 2.27, + "learning_rate": 2.918422996060347e-05, + "loss": 0.1199, + "step": 7015 + }, + { + "epoch": 2.27, + "learning_rate": 2.9060792986409345e-05, + "loss": 0.1234, + "step": 7020 + }, + { + "epoch": 2.28, + "learning_rate": 2.8937573214066417e-05, + "loss": 0.1223, + "step": 7025 + }, + { + "epoch": 2.28, + "learning_rate": 2.8814571020849236e-05, + "loss": 0.1244, + "step": 7030 + }, + { + "epoch": 2.28, + "learning_rate": 2.8691786783366316e-05, + "loss": 0.1202, + "step": 7035 + }, + { + "epoch": 2.28, + "learning_rate": 2.85692208775587e-05, + "loss": 0.1221, + "step": 7040 + }, + { + "epoch": 2.28, + "learning_rate": 2.8446873678699027e-05, + "loss": 0.1288, + "step": 7045 + }, + { + "epoch": 2.28, + "learning_rate": 2.8324745561390296e-05, + "loss": 0.1221, + "step": 7050 + }, + { + "epoch": 2.29, + "learning_rate": 2.8202836899564666e-05, + "loss": 0.1304, + "step": 7055 + }, + { + "epoch": 2.29, + "learning_rate": 2.8081148066482433e-05, + "loss": 0.1337, + "step": 7060 + }, + { + "epoch": 2.29, + "learning_rate": 2.7959679434730724e-05, + "loss": 0.1212, + "step": 7065 + }, + { + "epoch": 2.29, + "learning_rate": 2.7838431376222595e-05, + "loss": 0.1185, + "step": 7070 + }, + { + "epoch": 2.29, + "learning_rate": 2.77174042621956e-05, + "loss": 0.1305, + "step": 7075 + }, + { + "epoch": 2.29, + "learning_rate": 2.7596598463210955e-05, + "loss": 0.123, + "step": 7080 + }, + { + "epoch": 2.3, + "learning_rate": 2.747601434915208e-05, + "loss": 0.1219, + "step": 7085 + }, + { + "epoch": 2.3, + "learning_rate": 2.735565228922379e-05, + "loss": 0.1224, + "step": 7090 + }, + { + "epoch": 2.3, + "learning_rate": 2.723551265195098e-05, + "loss": 0.1228, + "step": 7095 + }, + { + "epoch": 2.3, + "learning_rate": 2.711559580517744e-05, + "loss": 0.1202, + "step": 7100 + }, + { + "epoch": 2.3, + "learning_rate": 2.699590211606493e-05, + "loss": 0.1219, + "step": 7105 + }, + { + "epoch": 2.3, + "learning_rate": 2.6876431951091897e-05, + "loss": 0.1274, + "step": 7110 + }, + { + "epoch": 2.31, + "learning_rate": 2.6757185676052443e-05, + "loss": 0.1195, + "step": 7115 + }, + { + "epoch": 2.31, + "learning_rate": 2.663816365605506e-05, + "loss": 0.1241, + "step": 7120 + }, + { + "epoch": 2.31, + "learning_rate": 2.651936625552175e-05, + "loss": 0.1256, + "step": 7125 + }, + { + "epoch": 2.31, + "learning_rate": 2.6400793838186644e-05, + "loss": 0.1192, + "step": 7130 + }, + { + "epoch": 2.31, + "learning_rate": 2.6282446767095138e-05, + "loss": 0.1165, + "step": 7135 + }, + { + "epoch": 2.31, + "learning_rate": 2.616432540460255e-05, + "loss": 0.1253, + "step": 7140 + }, + { + "epoch": 2.32, + "learning_rate": 2.6046430112373233e-05, + "loss": 0.1204, + "step": 7145 + }, + { + "epoch": 2.32, + "learning_rate": 2.5928761251379265e-05, + "loss": 0.1262, + "step": 7150 + }, + { + "epoch": 2.32, + "learning_rate": 2.5811319181899505e-05, + "loss": 0.1204, + "step": 7155 + }, + { + "epoch": 2.32, + "learning_rate": 2.569410426351836e-05, + "loss": 0.1335, + "step": 7160 + }, + { + "epoch": 2.32, + "learning_rate": 2.5577116855124816e-05, + "loss": 0.1286, + "step": 7165 + }, + { + "epoch": 2.32, + "learning_rate": 2.5460357314911176e-05, + "loss": 0.1253, + "step": 7170 + }, + { + "epoch": 2.33, + "learning_rate": 2.5343826000372163e-05, + "loss": 0.1281, + "step": 7175 + }, + { + "epoch": 2.33, + "learning_rate": 2.522752326830361e-05, + "loss": 0.1247, + "step": 7180 + }, + { + "epoch": 2.33, + "learning_rate": 2.5111449474801553e-05, + "loss": 0.1256, + "step": 7185 + }, + { + "epoch": 2.33, + "learning_rate": 2.4995604975261077e-05, + "loss": 0.1245, + "step": 7190 + }, + { + "epoch": 2.33, + "learning_rate": 2.4879990124375118e-05, + "loss": 0.1236, + "step": 7195 + }, + { + "epoch": 2.33, + "learning_rate": 2.4764605276133544e-05, + "loss": 0.1211, + "step": 7200 + }, + { + "epoch": 2.33, + "learning_rate": 2.4649450783822038e-05, + "loss": 0.1199, + "step": 7205 + }, + { + "epoch": 2.34, + "learning_rate": 2.4534527000020858e-05, + "loss": 0.1253, + "step": 7210 + }, + { + "epoch": 2.34, + "learning_rate": 2.441983427660397e-05, + "loss": 0.1285, + "step": 7215 + }, + { + "epoch": 2.34, + "learning_rate": 2.4305372964737905e-05, + "loss": 0.126, + "step": 7220 + }, + { + "epoch": 2.34, + "learning_rate": 2.4191143414880546e-05, + "loss": 0.1287, + "step": 7225 + }, + { + "epoch": 2.34, + "learning_rate": 2.407714597678028e-05, + "loss": 0.1259, + "step": 7230 + }, + { + "epoch": 2.34, + "learning_rate": 2.3963380999474704e-05, + "loss": 0.1232, + "step": 7235 + }, + { + "epoch": 2.35, + "learning_rate": 2.38498488312898e-05, + "loss": 0.124, + "step": 7240 + }, + { + "epoch": 2.35, + "learning_rate": 2.373654981983858e-05, + "loss": 0.1247, + "step": 7245 + }, + { + "epoch": 2.35, + "learning_rate": 2.3623484312020305e-05, + "loss": 0.1194, + "step": 7250 + }, + { + "epoch": 2.35, + "learning_rate": 2.3510652654019272e-05, + "loss": 0.1238, + "step": 7255 + }, + { + "epoch": 2.35, + "learning_rate": 2.339805519130367e-05, + "loss": 0.1225, + "step": 7260 + }, + { + "epoch": 2.35, + "learning_rate": 2.328569226862479e-05, + "loss": 0.1256, + "step": 7265 + }, + { + "epoch": 2.36, + "learning_rate": 2.3173564230015655e-05, + "loss": 0.1179, + "step": 7270 + }, + { + "epoch": 2.36, + "learning_rate": 2.3061671418790254e-05, + "loss": 0.1271, + "step": 7275 + }, + { + "epoch": 2.36, + "learning_rate": 2.2950014177542233e-05, + "loss": 0.1221, + "step": 7280 + }, + { + "epoch": 2.36, + "learning_rate": 2.2838592848144068e-05, + "loss": 0.1226, + "step": 7285 + }, + { + "epoch": 2.36, + "learning_rate": 2.2727407771745844e-05, + "loss": 0.1236, + "step": 7290 + }, + { + "epoch": 2.36, + "learning_rate": 2.2616459288774373e-05, + "loss": 0.1229, + "step": 7295 + }, + { + "epoch": 2.37, + "learning_rate": 2.250574773893196e-05, + "loss": 0.1145, + "step": 7300 + }, + { + "epoch": 2.37, + "learning_rate": 2.2395273461195533e-05, + "loss": 0.1352, + "step": 7305 + }, + { + "epoch": 2.37, + "learning_rate": 2.2285036793815527e-05, + "loss": 0.1273, + "step": 7310 + }, + { + "epoch": 2.37, + "learning_rate": 2.2175038074314914e-05, + "loss": 0.1199, + "step": 7315 + }, + { + "epoch": 2.37, + "learning_rate": 2.206527763948798e-05, + "loss": 0.1262, + "step": 7320 + }, + { + "epoch": 2.37, + "learning_rate": 2.195575582539956e-05, + "loss": 0.1241, + "step": 7325 + }, + { + "epoch": 2.38, + "learning_rate": 2.184647296738386e-05, + "loss": 0.1192, + "step": 7330 + }, + { + "epoch": 2.38, + "learning_rate": 2.1737429400043353e-05, + "loss": 0.1171, + "step": 7335 + }, + { + "epoch": 2.38, + "learning_rate": 2.1628625457247985e-05, + "loss": 0.122, + "step": 7340 + }, + { + "epoch": 2.38, + "learning_rate": 2.1520061472133902e-05, + "loss": 0.1182, + "step": 7345 + }, + { + "epoch": 2.38, + "learning_rate": 2.1411737777102647e-05, + "loss": 0.1261, + "step": 7350 + }, + { + "epoch": 2.38, + "learning_rate": 2.1303654703819966e-05, + "loss": 0.1185, + "step": 7355 + }, + { + "epoch": 2.39, + "learning_rate": 2.1195812583214914e-05, + "loss": 0.1183, + "step": 7360 + }, + { + "epoch": 2.39, + "learning_rate": 2.1088211745478748e-05, + "loss": 0.1291, + "step": 7365 + }, + { + "epoch": 2.39, + "learning_rate": 2.0980852520064053e-05, + "loss": 0.1215, + "step": 7370 + }, + { + "epoch": 2.39, + "learning_rate": 2.0873735235683535e-05, + "loss": 0.1258, + "step": 7375 + }, + { + "epoch": 2.39, + "learning_rate": 2.0766860220309194e-05, + "loss": 0.1274, + "step": 7380 + }, + { + "epoch": 2.39, + "learning_rate": 2.0660227801171263e-05, + "loss": 0.1194, + "step": 7385 + }, + { + "epoch": 2.39, + "learning_rate": 2.055383830475711e-05, + "loss": 0.1212, + "step": 7390 + }, + { + "epoch": 2.4, + "learning_rate": 2.044769205681045e-05, + "loss": 0.124, + "step": 7395 + }, + { + "epoch": 2.4, + "learning_rate": 2.0341789382330068e-05, + "loss": 0.1174, + "step": 7400 + }, + { + "epoch": 2.4, + "learning_rate": 2.0236130605569102e-05, + "loss": 0.1213, + "step": 7405 + }, + { + "epoch": 2.4, + "learning_rate": 2.0130716050033837e-05, + "loss": 0.1325, + "step": 7410 + }, + { + "epoch": 2.4, + "learning_rate": 2.0025546038482857e-05, + "loss": 0.1244, + "step": 7415 + }, + { + "epoch": 2.4, + "learning_rate": 1.992062089292599e-05, + "loss": 0.1285, + "step": 7420 + }, + { + "epoch": 2.41, + "learning_rate": 1.9815940934623324e-05, + "loss": 0.12, + "step": 7425 + }, + { + "epoch": 2.41, + "learning_rate": 1.9711506484084207e-05, + "loss": 0.1194, + "step": 7430 + }, + { + "epoch": 2.41, + "learning_rate": 1.9607317861066353e-05, + "loss": 0.1254, + "step": 7435 + }, + { + "epoch": 2.41, + "learning_rate": 1.950337538457472e-05, + "loss": 0.1249, + "step": 7440 + }, + { + "epoch": 2.41, + "learning_rate": 1.9399679372860667e-05, + "loss": 0.1236, + "step": 7445 + }, + { + "epoch": 2.41, + "learning_rate": 1.9296230143420956e-05, + "loss": 0.1181, + "step": 7450 + }, + { + "epoch": 2.42, + "learning_rate": 1.919302801299665e-05, + "loss": 0.1244, + "step": 7455 + }, + { + "epoch": 2.42, + "learning_rate": 1.9090073297572376e-05, + "loss": 0.1266, + "step": 7460 + }, + { + "epoch": 2.42, + "learning_rate": 1.8987366312375066e-05, + "loss": 0.1194, + "step": 7465 + }, + { + "epoch": 2.42, + "learning_rate": 1.8884907371873317e-05, + "loss": 0.1229, + "step": 7470 + }, + { + "epoch": 2.42, + "learning_rate": 1.8782696789776134e-05, + "loss": 0.1124, + "step": 7475 + }, + { + "epoch": 2.42, + "learning_rate": 1.868073487903218e-05, + "loss": 0.1253, + "step": 7480 + }, + { + "epoch": 2.43, + "learning_rate": 1.8579021951828657e-05, + "loss": 0.1177, + "step": 7485 + }, + { + "epoch": 2.43, + "learning_rate": 1.8477558319590538e-05, + "loss": 0.1224, + "step": 7490 + }, + { + "epoch": 2.43, + "learning_rate": 1.8376344292979365e-05, + "loss": 0.1195, + "step": 7495 + }, + { + "epoch": 2.43, + "learning_rate": 1.8275380181892587e-05, + "loss": 0.1221, + "step": 7500 + }, + { + "epoch": 2.43, + "learning_rate": 1.8174666295462318e-05, + "loss": 0.1215, + "step": 7505 + }, + { + "epoch": 2.43, + "learning_rate": 1.807420294205464e-05, + "loss": 0.1218, + "step": 7510 + }, + { + "epoch": 2.44, + "learning_rate": 1.797399042926853e-05, + "loss": 0.1248, + "step": 7515 + }, + { + "epoch": 2.44, + "learning_rate": 1.787402906393486e-05, + "loss": 0.1218, + "step": 7520 + }, + { + "epoch": 2.44, + "learning_rate": 1.777431915211566e-05, + "loss": 0.1199, + "step": 7525 + }, + { + "epoch": 2.44, + "learning_rate": 1.767486099910297e-05, + "loss": 0.1208, + "step": 7530 + }, + { + "epoch": 2.44, + "learning_rate": 1.757565490941807e-05, + "loss": 0.1193, + "step": 7535 + }, + { + "epoch": 2.44, + "learning_rate": 1.7476701186810373e-05, + "loss": 0.1214, + "step": 7540 + }, + { + "epoch": 2.44, + "learning_rate": 1.73780001342567e-05, + "loss": 0.1241, + "step": 7545 + }, + { + "epoch": 2.45, + "learning_rate": 1.7279552053960134e-05, + "loss": 0.1193, + "step": 7550 + }, + { + "epoch": 2.45, + "learning_rate": 1.7181357247349318e-05, + "loss": 0.1229, + "step": 7555 + }, + { + "epoch": 2.45, + "learning_rate": 1.7083416015077326e-05, + "loss": 0.1255, + "step": 7560 + }, + { + "epoch": 2.45, + "learning_rate": 1.698572865702094e-05, + "loss": 0.1269, + "step": 7565 + }, + { + "epoch": 2.45, + "learning_rate": 1.688829547227949e-05, + "loss": 0.1217, + "step": 7570 + }, + { + "epoch": 2.45, + "learning_rate": 1.679111675917424e-05, + "loss": 0.1174, + "step": 7575 + }, + { + "epoch": 2.46, + "learning_rate": 1.6694192815247177e-05, + "loss": 0.1205, + "step": 7580 + }, + { + "epoch": 2.46, + "learning_rate": 1.6597523937260294e-05, + "loss": 0.1177, + "step": 7585 + }, + { + "epoch": 2.46, + "learning_rate": 1.6501110421194656e-05, + "loss": 0.1252, + "step": 7590 + }, + { + "epoch": 2.46, + "learning_rate": 1.6404952562249366e-05, + "loss": 0.1218, + "step": 7595 + }, + { + "epoch": 2.46, + "learning_rate": 1.6309050654840863e-05, + "loss": 0.1173, + "step": 7600 + }, + { + "epoch": 2.46, + "learning_rate": 1.62134049926018e-05, + "loss": 0.1183, + "step": 7605 + }, + { + "epoch": 2.47, + "learning_rate": 1.6118015868380388e-05, + "loss": 0.1243, + "step": 7610 + }, + { + "epoch": 2.47, + "learning_rate": 1.6022883574239233e-05, + "loss": 0.121, + "step": 7615 + }, + { + "epoch": 2.47, + "learning_rate": 1.5928008401454676e-05, + "loss": 0.1197, + "step": 7620 + }, + { + "epoch": 2.47, + "learning_rate": 1.5833390640515787e-05, + "loss": 0.1229, + "step": 7625 + }, + { + "epoch": 2.47, + "learning_rate": 1.573903058112344e-05, + "loss": 0.1278, + "step": 7630 + }, + { + "epoch": 2.47, + "learning_rate": 1.5644928512189527e-05, + "loss": 0.1218, + "step": 7635 + }, + { + "epoch": 2.48, + "learning_rate": 1.555108472183604e-05, + "loss": 0.1225, + "step": 7640 + }, + { + "epoch": 2.48, + "learning_rate": 1.54574994973941e-05, + "loss": 0.1171, + "step": 7645 + }, + { + "epoch": 2.48, + "learning_rate": 1.5364173125403215e-05, + "loss": 0.1193, + "step": 7650 + }, + { + "epoch": 2.48, + "learning_rate": 1.5271105891610327e-05, + "loss": 0.1183, + "step": 7655 + }, + { + "epoch": 2.48, + "learning_rate": 1.517829808096891e-05, + "loss": 0.1225, + "step": 7660 + }, + { + "epoch": 2.48, + "learning_rate": 1.5085749977638209e-05, + "loss": 0.1249, + "step": 7665 + }, + { + "epoch": 2.49, + "learning_rate": 1.4993461864982195e-05, + "loss": 0.1197, + "step": 7670 + }, + { + "epoch": 2.49, + "learning_rate": 1.4901434025568905e-05, + "loss": 0.1224, + "step": 7675 + }, + { + "epoch": 2.49, + "learning_rate": 1.480966674116937e-05, + "loss": 0.1205, + "step": 7680 + }, + { + "epoch": 2.49, + "learning_rate": 1.4718160292756955e-05, + "loss": 0.1248, + "step": 7685 + }, + { + "epoch": 2.49, + "learning_rate": 1.46269149605063e-05, + "loss": 0.1216, + "step": 7690 + }, + { + "epoch": 2.49, + "learning_rate": 1.4535931023792648e-05, + "loss": 0.1168, + "step": 7695 + }, + { + "epoch": 2.5, + "learning_rate": 1.4445208761190809e-05, + "loss": 0.1188, + "step": 7700 + }, + { + "epoch": 2.5, + "learning_rate": 1.4354748450474486e-05, + "loss": 0.1153, + "step": 7705 + }, + { + "epoch": 2.5, + "learning_rate": 1.426455036861527e-05, + "loss": 0.124, + "step": 7710 + }, + { + "epoch": 2.5, + "learning_rate": 1.4174614791781892e-05, + "loss": 0.1219, + "step": 7715 + }, + { + "epoch": 2.5, + "learning_rate": 1.4084941995339362e-05, + "loss": 0.1224, + "step": 7720 + }, + { + "epoch": 2.5, + "learning_rate": 1.3995532253848054e-05, + "loss": 0.1253, + "step": 7725 + }, + { + "epoch": 2.5, + "learning_rate": 1.3906385841062952e-05, + "loss": 0.1198, + "step": 7730 + }, + { + "epoch": 2.51, + "learning_rate": 1.3817503029932788e-05, + "loss": 0.1307, + "step": 7735 + }, + { + "epoch": 2.51, + "learning_rate": 1.3728884092599215e-05, + "loss": 0.1236, + "step": 7740 + }, + { + "epoch": 2.51, + "learning_rate": 1.3640529300395876e-05, + "loss": 0.1227, + "step": 7745 + }, + { + "epoch": 2.51, + "learning_rate": 1.3552438923847766e-05, + "loss": 0.1246, + "step": 7750 + }, + { + "epoch": 2.51, + "learning_rate": 1.3464613232670164e-05, + "loss": 0.1171, + "step": 7755 + }, + { + "epoch": 2.51, + "learning_rate": 1.337705249576806e-05, + "loss": 0.1147, + "step": 7760 + }, + { + "epoch": 2.52, + "learning_rate": 1.3289756981235113e-05, + "loss": 0.1229, + "step": 7765 + }, + { + "epoch": 2.52, + "learning_rate": 1.3202726956352996e-05, + "loss": 0.1229, + "step": 7770 + }, + { + "epoch": 2.52, + "learning_rate": 1.3115962687590422e-05, + "loss": 0.1232, + "step": 7775 + }, + { + "epoch": 2.52, + "learning_rate": 1.302946444060248e-05, + "loss": 0.1268, + "step": 7780 + }, + { + "epoch": 2.52, + "learning_rate": 1.294323248022975e-05, + "loss": 0.1212, + "step": 7785 + }, + { + "epoch": 2.52, + "learning_rate": 1.2857267070497447e-05, + "loss": 0.1255, + "step": 7790 + }, + { + "epoch": 2.53, + "learning_rate": 1.2771568474614714e-05, + "loss": 0.1233, + "step": 7795 + }, + { + "epoch": 2.53, + "learning_rate": 1.2686136954973716e-05, + "loss": 0.1116, + "step": 7800 + }, + { + "epoch": 2.53, + "learning_rate": 1.2600972773148933e-05, + "loss": 0.1156, + "step": 7805 + }, + { + "epoch": 2.53, + "learning_rate": 1.2516076189896231e-05, + "loss": 0.1155, + "step": 7810 + }, + { + "epoch": 2.53, + "learning_rate": 1.2431447465152257e-05, + "loss": 0.1273, + "step": 7815 + }, + { + "epoch": 2.53, + "learning_rate": 1.2347086858033407e-05, + "loss": 0.1223, + "step": 7820 + }, + { + "epoch": 2.54, + "learning_rate": 1.2262994626835233e-05, + "loss": 0.1221, + "step": 7825 + }, + { + "epoch": 2.54, + "learning_rate": 1.2179171029031577e-05, + "loss": 0.1208, + "step": 7830 + }, + { + "epoch": 2.54, + "learning_rate": 1.2095616321273695e-05, + "loss": 0.1154, + "step": 7835 + }, + { + "epoch": 2.54, + "learning_rate": 1.2012330759389656e-05, + "loss": 0.1208, + "step": 7840 + }, + { + "epoch": 2.54, + "learning_rate": 1.1929314598383423e-05, + "loss": 0.1158, + "step": 7845 + }, + { + "epoch": 2.54, + "learning_rate": 1.1846568092434041e-05, + "loss": 0.1165, + "step": 7850 + }, + { + "epoch": 2.55, + "learning_rate": 1.1764091494895024e-05, + "loss": 0.1171, + "step": 7855 + }, + { + "epoch": 2.55, + "learning_rate": 1.168188505829344e-05, + "loss": 0.1198, + "step": 7860 + }, + { + "epoch": 2.55, + "learning_rate": 1.1599949034329139e-05, + "loss": 0.1251, + "step": 7865 + }, + { + "epoch": 2.55, + "learning_rate": 1.1518283673874098e-05, + "loss": 0.1166, + "step": 7870 + }, + { + "epoch": 2.55, + "learning_rate": 1.1436889226971493e-05, + "loss": 0.1183, + "step": 7875 + }, + { + "epoch": 2.55, + "learning_rate": 1.1355765942835084e-05, + "loss": 0.1202, + "step": 7880 + }, + { + "epoch": 2.56, + "learning_rate": 1.127491406984833e-05, + "loss": 0.1179, + "step": 7885 + }, + { + "epoch": 2.56, + "learning_rate": 1.1194333855563754e-05, + "loss": 0.1223, + "step": 7890 + }, + { + "epoch": 2.56, + "learning_rate": 1.1114025546702012e-05, + "loss": 0.1248, + "step": 7895 + }, + { + "epoch": 2.56, + "learning_rate": 1.1033989389151345e-05, + "loss": 0.1211, + "step": 7900 + }, + { + "epoch": 2.56, + "learning_rate": 1.095422562796663e-05, + "loss": 0.1213, + "step": 7905 + }, + { + "epoch": 2.56, + "learning_rate": 1.0874734507368801e-05, + "loss": 0.1169, + "step": 7910 + }, + { + "epoch": 2.56, + "learning_rate": 1.0795516270743933e-05, + "loss": 0.1195, + "step": 7915 + }, + { + "epoch": 2.57, + "learning_rate": 1.0716571160642664e-05, + "loss": 0.1237, + "step": 7920 + }, + { + "epoch": 2.57, + "learning_rate": 1.0637899418779363e-05, + "loss": 0.1235, + "step": 7925 + }, + { + "epoch": 2.57, + "learning_rate": 1.0559501286031326e-05, + "loss": 0.1195, + "step": 7930 + }, + { + "epoch": 2.57, + "learning_rate": 1.0481377002438186e-05, + "loss": 0.1128, + "step": 7935 + }, + { + "epoch": 2.57, + "learning_rate": 1.0403526807201103e-05, + "loss": 0.1147, + "step": 7940 + }, + { + "epoch": 2.57, + "learning_rate": 1.032595093868196e-05, + "loss": 0.1177, + "step": 7945 + }, + { + "epoch": 2.58, + "learning_rate": 1.0248649634402796e-05, + "loss": 0.1183, + "step": 7950 + }, + { + "epoch": 2.58, + "learning_rate": 1.0171623131044949e-05, + "loss": 0.1198, + "step": 7955 + }, + { + "epoch": 2.58, + "learning_rate": 1.0094871664448336e-05, + "loss": 0.1191, + "step": 7960 + }, + { + "epoch": 2.58, + "learning_rate": 1.0018395469610841e-05, + "loss": 0.1187, + "step": 7965 + }, + { + "epoch": 2.58, + "learning_rate": 9.942194780687453e-06, + "loss": 0.1222, + "step": 7970 + }, + { + "epoch": 2.58, + "learning_rate": 9.866269830989661e-06, + "loss": 0.1241, + "step": 7975 + }, + { + "epoch": 2.59, + "learning_rate": 9.790620852984645e-06, + "loss": 0.1182, + "step": 7980 + }, + { + "epoch": 2.59, + "learning_rate": 9.715248078294658e-06, + "loss": 0.123, + "step": 7985 + }, + { + "epoch": 2.59, + "learning_rate": 9.64015173769629e-06, + "loss": 0.1229, + "step": 7990 + }, + { + "epoch": 2.59, + "learning_rate": 9.565332061119669e-06, + "loss": 0.1197, + "step": 7995 + }, + { + "epoch": 2.59, + "learning_rate": 9.490789277647927e-06, + "loss": 0.1233, + "step": 8000 + }, + { + "epoch": 2.59, + "learning_rate": 9.416523615516315e-06, + "loss": 0.1216, + "step": 8005 + }, + { + "epoch": 2.6, + "learning_rate": 9.342535302111654e-06, + "loss": 0.1249, + "step": 8010 + }, + { + "epoch": 2.6, + "learning_rate": 9.268824563971534e-06, + "loss": 0.1204, + "step": 8015 + }, + { + "epoch": 2.6, + "learning_rate": 9.195391626783734e-06, + "loss": 0.1231, + "step": 8020 + }, + { + "epoch": 2.6, + "learning_rate": 9.122236715385358e-06, + "loss": 0.1244, + "step": 8025 + }, + { + "epoch": 2.6, + "learning_rate": 9.049360053762346e-06, + "loss": 0.1174, + "step": 8030 + }, + { + "epoch": 2.6, + "learning_rate": 8.976761865048622e-06, + "loss": 0.1179, + "step": 8035 + }, + { + "epoch": 2.61, + "learning_rate": 8.904442371525534e-06, + "loss": 0.1219, + "step": 8040 + }, + { + "epoch": 2.61, + "learning_rate": 8.832401794621103e-06, + "loss": 0.1193, + "step": 8045 + }, + { + "epoch": 2.61, + "learning_rate": 8.760640354909356e-06, + "loss": 0.1261, + "step": 8050 + }, + { + "epoch": 2.61, + "learning_rate": 8.689158272109632e-06, + "loss": 0.1247, + "step": 8055 + }, + { + "epoch": 2.61, + "learning_rate": 8.617955765085984e-06, + "loss": 0.121, + "step": 8060 + }, + { + "epoch": 2.61, + "learning_rate": 8.547033051846443e-06, + "loss": 0.1204, + "step": 8065 + }, + { + "epoch": 2.62, + "learning_rate": 8.47639034954234e-06, + "loss": 0.1185, + "step": 8070 + }, + { + "epoch": 2.62, + "learning_rate": 8.406027874467703e-06, + "loss": 0.1203, + "step": 8075 + }, + { + "epoch": 2.62, + "learning_rate": 8.335945842058523e-06, + "loss": 0.1139, + "step": 8080 + }, + { + "epoch": 2.62, + "learning_rate": 8.266144466892178e-06, + "loss": 0.1266, + "step": 8085 + }, + { + "epoch": 2.62, + "learning_rate": 8.196623962686678e-06, + "loss": 0.1223, + "step": 8090 + }, + { + "epoch": 2.62, + "learning_rate": 8.12738454230011e-06, + "loss": 0.1151, + "step": 8095 + }, + { + "epoch": 2.62, + "learning_rate": 8.05842641772988e-06, + "loss": 0.1175, + "step": 8100 + }, + { + "epoch": 2.63, + "learning_rate": 7.9897498001122e-06, + "loss": 0.1186, + "step": 8105 + }, + { + "epoch": 2.63, + "learning_rate": 7.921354899721279e-06, + "loss": 0.1234, + "step": 8110 + }, + { + "epoch": 2.63, + "learning_rate": 7.853241925968813e-06, + "loss": 0.1244, + "step": 8115 + }, + { + "epoch": 2.63, + "learning_rate": 7.785411087403316e-06, + "loss": 0.1191, + "step": 8120 + }, + { + "epoch": 2.63, + "learning_rate": 7.71786259170939e-06, + "loss": 0.1134, + "step": 8125 + }, + { + "epoch": 2.63, + "learning_rate": 7.65059664570722e-06, + "loss": 0.1152, + "step": 8130 + }, + { + "epoch": 2.64, + "learning_rate": 7.5836134553518345e-06, + "loss": 0.1193, + "step": 8135 + }, + { + "epoch": 2.64, + "learning_rate": 7.51691322573258e-06, + "loss": 0.1165, + "step": 8140 + }, + { + "epoch": 2.64, + "learning_rate": 7.450496161072351e-06, + "loss": 0.1219, + "step": 8145 + }, + { + "epoch": 2.64, + "learning_rate": 7.384362464727113e-06, + "loss": 0.118, + "step": 8150 + }, + { + "epoch": 2.64, + "learning_rate": 7.3185123391851905e-06, + "loss": 0.1181, + "step": 8155 + }, + { + "epoch": 2.64, + "learning_rate": 7.252945986066684e-06, + "loss": 0.1117, + "step": 8160 + }, + { + "epoch": 2.65, + "learning_rate": 7.187663606122807e-06, + "loss": 0.1184, + "step": 8165 + }, + { + "epoch": 2.65, + "learning_rate": 7.12266539923534e-06, + "loss": 0.125, + "step": 8170 + }, + { + "epoch": 2.65, + "learning_rate": 7.057951564415943e-06, + "loss": 0.1274, + "step": 8175 + }, + { + "epoch": 2.65, + "learning_rate": 6.993522299805633e-06, + "loss": 0.1182, + "step": 8180 + }, + { + "epoch": 2.65, + "learning_rate": 6.9293778026740754e-06, + "loss": 0.1246, + "step": 8185 + }, + { + "epoch": 2.65, + "learning_rate": 6.8655182694190685e-06, + "loss": 0.1266, + "step": 8190 + }, + { + "epoch": 2.66, + "learning_rate": 6.801943895565921e-06, + "loss": 0.1162, + "step": 8195 + }, + { + "epoch": 2.66, + "learning_rate": 6.738654875766781e-06, + "loss": 0.1292, + "step": 8200 + }, + { + "epoch": 2.66, + "learning_rate": 6.675651403800176e-06, + "loss": 0.126, + "step": 8205 + }, + { + "epoch": 2.66, + "learning_rate": 6.612933672570265e-06, + "loss": 0.1217, + "step": 8210 + }, + { + "epoch": 2.66, + "learning_rate": 6.550501874106396e-06, + "loss": 0.1164, + "step": 8215 + }, + { + "epoch": 2.66, + "learning_rate": 6.488356199562385e-06, + "loss": 0.1208, + "step": 8220 + }, + { + "epoch": 2.67, + "learning_rate": 6.426496839216056e-06, + "loss": 0.1166, + "step": 8225 + }, + { + "epoch": 2.67, + "learning_rate": 6.364923982468529e-06, + "loss": 0.1198, + "step": 8230 + }, + { + "epoch": 2.67, + "learning_rate": 6.303637817843755e-06, + "loss": 0.1239, + "step": 8235 + }, + { + "epoch": 2.67, + "learning_rate": 6.2426385329878655e-06, + "loss": 0.1189, + "step": 8240 + }, + { + "epoch": 2.67, + "learning_rate": 6.18192631466864e-06, + "loss": 0.1308, + "step": 8245 + }, + { + "epoch": 2.67, + "learning_rate": 6.1215013487749165e-06, + "loss": 0.119, + "step": 8250 + }, + { + "epoch": 2.68, + "learning_rate": 6.0613638203160015e-06, + "loss": 0.1206, + "step": 8255 + }, + { + "epoch": 2.68, + "learning_rate": 6.00151391342113e-06, + "loss": 0.1181, + "step": 8260 + }, + { + "epoch": 2.68, + "learning_rate": 5.94195181133892e-06, + "loss": 0.1182, + "step": 8265 + }, + { + "epoch": 2.68, + "learning_rate": 5.8826776964367895e-06, + "loss": 0.1179, + "step": 8270 + }, + { + "epoch": 2.68, + "learning_rate": 5.823691750200344e-06, + "loss": 0.1191, + "step": 8275 + }, + { + "epoch": 2.68, + "learning_rate": 5.7649941532329384e-06, + "loss": 0.1203, + "step": 8280 + }, + { + "epoch": 2.68, + "learning_rate": 5.706585085255001e-06, + "loss": 0.1273, + "step": 8285 + }, + { + "epoch": 2.69, + "learning_rate": 5.648464725103597e-06, + "loss": 0.1184, + "step": 8290 + }, + { + "epoch": 2.69, + "learning_rate": 5.59063325073178e-06, + "loss": 0.1163, + "step": 8295 + }, + { + "epoch": 2.69, + "learning_rate": 5.533090839208133e-06, + "loss": 0.1237, + "step": 8300 + }, + { + "epoch": 2.69, + "learning_rate": 5.475837666716133e-06, + "loss": 0.1233, + "step": 8305 + }, + { + "epoch": 2.69, + "learning_rate": 5.418873908553723e-06, + "loss": 0.1111, + "step": 8310 + }, + { + "epoch": 2.69, + "learning_rate": 5.3621997391326565e-06, + "loss": 0.1235, + "step": 8315 + }, + { + "epoch": 2.7, + "learning_rate": 5.3058153319780816e-06, + "loss": 0.119, + "step": 8320 + }, + { + "epoch": 2.7, + "learning_rate": 5.249720859727924e-06, + "loss": 0.1197, + "step": 8325 + }, + { + "epoch": 2.7, + "learning_rate": 5.193916494132367e-06, + "loss": 0.119, + "step": 8330 + }, + { + "epoch": 2.7, + "learning_rate": 5.1384024060534e-06, + "loss": 0.1182, + "step": 8335 + }, + { + "epoch": 2.7, + "learning_rate": 5.083178765464159e-06, + "loss": 0.1206, + "step": 8340 + }, + { + "epoch": 2.7, + "learning_rate": 5.028245741448579e-06, + "loss": 0.1119, + "step": 8345 + }, + { + "epoch": 2.71, + "learning_rate": 4.973603502200685e-06, + "loss": 0.1223, + "step": 8350 + }, + { + "epoch": 2.71, + "learning_rate": 4.919252215024272e-06, + "loss": 0.1145, + "step": 8355 + }, + { + "epoch": 2.71, + "learning_rate": 4.865192046332245e-06, + "loss": 0.1222, + "step": 8360 + }, + { + "epoch": 2.71, + "learning_rate": 4.8114231616461555e-06, + "loss": 0.1209, + "step": 8365 + }, + { + "epoch": 2.71, + "learning_rate": 4.7579457255957364e-06, + "loss": 0.1192, + "step": 8370 + }, + { + "epoch": 2.71, + "learning_rate": 4.704759901918332e-06, + "loss": 0.1214, + "step": 8375 + }, + { + "epoch": 2.72, + "learning_rate": 4.651865853458437e-06, + "loss": 0.1202, + "step": 8380 + }, + { + "epoch": 2.72, + "learning_rate": 4.599263742167193e-06, + "loss": 0.1208, + "step": 8385 + }, + { + "epoch": 2.72, + "learning_rate": 4.546953729101888e-06, + "loss": 0.1232, + "step": 8390 + }, + { + "epoch": 2.72, + "learning_rate": 4.494935974425429e-06, + "loss": 0.1186, + "step": 8395 + }, + { + "epoch": 2.72, + "learning_rate": 4.443210637405959e-06, + "loss": 0.1163, + "step": 8400 + }, + { + "epoch": 2.72, + "learning_rate": 4.391777876416225e-06, + "loss": 0.1158, + "step": 8405 + }, + { + "epoch": 2.73, + "learning_rate": 4.340637848933194e-06, + "loss": 0.1192, + "step": 8410 + }, + { + "epoch": 2.73, + "learning_rate": 4.289790711537523e-06, + "loss": 0.115, + "step": 8415 + }, + { + "epoch": 2.73, + "learning_rate": 4.239236619913145e-06, + "loss": 0.1125, + "step": 8420 + }, + { + "epoch": 2.73, + "learning_rate": 4.188975728846678e-06, + "loss": 0.1166, + "step": 8425 + }, + { + "epoch": 2.73, + "learning_rate": 4.139008192227078e-06, + "loss": 0.1189, + "step": 8430 + }, + { + "epoch": 2.73, + "learning_rate": 4.089334163045045e-06, + "loss": 0.1127, + "step": 8435 + }, + { + "epoch": 2.73, + "learning_rate": 4.039953793392692e-06, + "loss": 0.1263, + "step": 8440 + }, + { + "epoch": 2.74, + "learning_rate": 3.990867234462936e-06, + "loss": 0.1206, + "step": 8445 + }, + { + "epoch": 2.74, + "learning_rate": 3.942074636549142e-06, + "loss": 0.1249, + "step": 8450 + }, + { + "epoch": 2.74, + "learning_rate": 3.893576149044642e-06, + "loss": 0.1165, + "step": 8455 + }, + { + "epoch": 2.74, + "learning_rate": 3.845371920442176e-06, + "loss": 0.1249, + "step": 8460 + }, + { + "epoch": 2.74, + "learning_rate": 3.7974620983336264e-06, + "loss": 0.1161, + "step": 8465 + }, + { + "epoch": 2.74, + "learning_rate": 3.749846829409398e-06, + "loss": 0.1132, + "step": 8470 + }, + { + "epoch": 2.75, + "learning_rate": 3.702526259458039e-06, + "loss": 0.114, + "step": 8475 + }, + { + "epoch": 2.75, + "learning_rate": 3.6555005333657964e-06, + "loss": 0.1221, + "step": 8480 + }, + { + "epoch": 2.75, + "learning_rate": 3.608769795116185e-06, + "loss": 0.1221, + "step": 8485 + }, + { + "epoch": 2.75, + "learning_rate": 3.5623341877894735e-06, + "loss": 0.1253, + "step": 8490 + }, + { + "epoch": 2.75, + "learning_rate": 3.5161938535623553e-06, + "loss": 0.1238, + "step": 8495 + }, + { + "epoch": 2.75, + "learning_rate": 3.4703489337073902e-06, + "loss": 0.121, + "step": 8500 + }, + { + "epoch": 2.76, + "learning_rate": 3.424799568592707e-06, + "loss": 0.1241, + "step": 8505 + }, + { + "epoch": 2.76, + "learning_rate": 3.3795458976814574e-06, + "loss": 0.1215, + "step": 8510 + }, + { + "epoch": 2.76, + "learning_rate": 3.3345880595314626e-06, + "loss": 0.1244, + "step": 8515 + }, + { + "epoch": 2.76, + "learning_rate": 3.2899261917947234e-06, + "loss": 0.1237, + "step": 8520 + }, + { + "epoch": 2.76, + "learning_rate": 3.2455604312170874e-06, + "loss": 0.1201, + "step": 8525 + }, + { + "epoch": 2.76, + "learning_rate": 3.2014909136377613e-06, + "loss": 0.1184, + "step": 8530 + }, + { + "epoch": 2.77, + "learning_rate": 3.1577177739888884e-06, + "loss": 0.1201, + "step": 8535 + }, + { + "epoch": 2.77, + "learning_rate": 3.1142411462952047e-06, + "loss": 0.119, + "step": 8540 + }, + { + "epoch": 2.77, + "learning_rate": 3.07106116367355e-06, + "loss": 0.1155, + "step": 8545 + }, + { + "epoch": 2.77, + "learning_rate": 3.028177958332512e-06, + "loss": 0.1165, + "step": 8550 + }, + { + "epoch": 2.77, + "learning_rate": 2.985591661572007e-06, + "loss": 0.1138, + "step": 8555 + }, + { + "epoch": 2.77, + "learning_rate": 2.943302403782866e-06, + "loss": 0.1175, + "step": 8560 + }, + { + "epoch": 2.78, + "learning_rate": 2.901310314446448e-06, + "loss": 0.1268, + "step": 8565 + }, + { + "epoch": 2.78, + "learning_rate": 2.85961552213424e-06, + "loss": 0.1175, + "step": 8570 + }, + { + "epoch": 2.78, + "learning_rate": 2.8182181545074794e-06, + "loss": 0.1232, + "step": 8575 + }, + { + "epoch": 2.78, + "learning_rate": 2.777118338316731e-06, + "loss": 0.1177, + "step": 8580 + }, + { + "epoch": 2.78, + "learning_rate": 2.7363161994015118e-06, + "loss": 0.1208, + "step": 8585 + }, + { + "epoch": 2.78, + "learning_rate": 2.6958118626899343e-06, + "loss": 0.121, + "step": 8590 + }, + { + "epoch": 2.79, + "learning_rate": 2.6556054521982734e-06, + "loss": 0.1181, + "step": 8595 + }, + { + "epoch": 2.79, + "learning_rate": 2.6156970910306223e-06, + "loss": 0.1202, + "step": 8600 + }, + { + "epoch": 2.79, + "learning_rate": 2.5760869013785047e-06, + "loss": 0.1237, + "step": 8605 + }, + { + "epoch": 2.79, + "learning_rate": 2.5367750045205085e-06, + "loss": 0.1242, + "step": 8610 + }, + { + "epoch": 2.79, + "learning_rate": 2.4977615208219064e-06, + "loss": 0.1164, + "step": 8615 + }, + { + "epoch": 2.79, + "learning_rate": 2.45904656973428e-06, + "loss": 0.1227, + "step": 8620 + }, + { + "epoch": 2.79, + "learning_rate": 2.4206302697951655e-06, + "loss": 0.1199, + "step": 8625 + }, + { + "epoch": 2.8, + "learning_rate": 2.3825127386276958e-06, + "loss": 0.117, + "step": 8630 + }, + { + "epoch": 2.8, + "learning_rate": 2.344694092940236e-06, + "loss": 0.1167, + "step": 8635 + }, + { + "epoch": 2.8, + "learning_rate": 2.307174448525995e-06, + "loss": 0.1112, + "step": 8640 + }, + { + "epoch": 2.8, + "learning_rate": 2.2699539202627462e-06, + "loss": 0.1189, + "step": 8645 + }, + { + "epoch": 2.8, + "learning_rate": 2.233032622112374e-06, + "loss": 0.1234, + "step": 8650 + }, + { + "epoch": 2.8, + "learning_rate": 2.1964106671206184e-06, + "loss": 0.1185, + "step": 8655 + }, + { + "epoch": 2.81, + "learning_rate": 2.160088167416685e-06, + "loss": 0.1246, + "step": 8660 + }, + { + "epoch": 2.81, + "learning_rate": 2.1240652342128907e-06, + "loss": 0.1216, + "step": 8665 + }, + { + "epoch": 2.81, + "learning_rate": 2.088341977804353e-06, + "loss": 0.1182, + "step": 8670 + }, + { + "epoch": 2.81, + "learning_rate": 2.052918507568613e-06, + "loss": 0.1211, + "step": 8675 + }, + { + "epoch": 2.81, + "learning_rate": 2.0177949319653667e-06, + "loss": 0.1253, + "step": 8680 + }, + { + "epoch": 2.81, + "learning_rate": 1.9829713585360676e-06, + "loss": 0.1174, + "step": 8685 + }, + { + "epoch": 2.82, + "learning_rate": 1.948447893903638e-06, + "loss": 0.1212, + "step": 8690 + }, + { + "epoch": 2.82, + "learning_rate": 1.9142246437720777e-06, + "loss": 0.1216, + "step": 8695 + }, + { + "epoch": 2.82, + "learning_rate": 1.880301712926258e-06, + "loss": 0.1172, + "step": 8700 + }, + { + "epoch": 2.82, + "learning_rate": 1.846679205231472e-06, + "loss": 0.1215, + "step": 8705 + }, + { + "epoch": 2.82, + "learning_rate": 1.8133572236332164e-06, + "loss": 0.1159, + "step": 8710 + }, + { + "epoch": 2.82, + "learning_rate": 1.780335870156824e-06, + "loss": 0.122, + "step": 8715 + }, + { + "epoch": 2.83, + "learning_rate": 1.7476152459071527e-06, + "loss": 0.1105, + "step": 8720 + }, + { + "epoch": 2.83, + "learning_rate": 1.7151954510683077e-06, + "loss": 0.1204, + "step": 8725 + }, + { + "epoch": 2.83, + "learning_rate": 1.683076584903287e-06, + "loss": 0.1219, + "step": 8730 + }, + { + "epoch": 2.83, + "learning_rate": 1.6512587457537477e-06, + "loss": 0.1214, + "step": 8735 + }, + { + "epoch": 2.83, + "learning_rate": 1.619742031039595e-06, + "loss": 0.1196, + "step": 8740 + }, + { + "epoch": 2.83, + "learning_rate": 1.5885265372588166e-06, + "loss": 0.1147, + "step": 8745 + }, + { + "epoch": 2.84, + "learning_rate": 1.5576123599870818e-06, + "loss": 0.1203, + "step": 8750 + }, + { + "epoch": 2.84, + "learning_rate": 1.5269995938775095e-06, + "loss": 0.1207, + "step": 8755 + }, + { + "epoch": 2.84, + "learning_rate": 1.4966883326603454e-06, + "loss": 0.1131, + "step": 8760 + }, + { + "epoch": 2.84, + "learning_rate": 1.466678669142685e-06, + "loss": 0.1229, + "step": 8765 + }, + { + "epoch": 2.84, + "learning_rate": 1.436970695208184e-06, + "loss": 0.1216, + "step": 8770 + }, + { + "epoch": 2.84, + "learning_rate": 1.407564501816816e-06, + "loss": 0.1195, + "step": 8775 + }, + { + "epoch": 2.85, + "learning_rate": 1.3784601790045482e-06, + "loss": 0.1221, + "step": 8780 + }, + { + "epoch": 2.85, + "learning_rate": 1.349657815883032e-06, + "loss": 0.1162, + "step": 8785 + }, + { + "epoch": 2.85, + "learning_rate": 1.3211575006394584e-06, + "loss": 0.1163, + "step": 8790 + }, + { + "epoch": 2.85, + "learning_rate": 1.2929593205361468e-06, + "loss": 0.1302, + "step": 8795 + }, + { + "epoch": 2.85, + "learning_rate": 1.265063361910379e-06, + "loss": 0.1217, + "step": 8800 + }, + { + "epoch": 2.85, + "learning_rate": 1.2374697101740551e-06, + "loss": 0.1207, + "step": 8805 + }, + { + "epoch": 2.85, + "learning_rate": 1.2101784498135037e-06, + "loss": 0.1187, + "step": 8810 + }, + { + "epoch": 2.86, + "learning_rate": 1.1831896643891726e-06, + "loss": 0.1258, + "step": 8815 + }, + { + "epoch": 2.86, + "learning_rate": 1.1565034365353943e-06, + "loss": 0.1208, + "step": 8820 + }, + { + "epoch": 2.86, + "learning_rate": 1.1301198479601316e-06, + "loss": 0.121, + "step": 8825 + }, + { + "epoch": 2.86, + "learning_rate": 1.1040389794447215e-06, + "loss": 0.1148, + "step": 8830 + }, + { + "epoch": 2.86, + "learning_rate": 1.0782609108436537e-06, + "loss": 0.1228, + "step": 8835 + }, + { + "epoch": 2.86, + "learning_rate": 1.0527857210842705e-06, + "loss": 0.115, + "step": 8840 + }, + { + "epoch": 2.87, + "learning_rate": 1.0276134881665677e-06, + "loss": 0.1182, + "step": 8845 + }, + { + "epoch": 2.87, + "learning_rate": 1.0027442891629823e-06, + "loss": 0.118, + "step": 8850 + }, + { + "epoch": 2.87, + "learning_rate": 9.781782002180717e-07, + "loss": 0.1169, + "step": 8855 + }, + { + "epoch": 2.87, + "learning_rate": 9.539152965483466e-07, + "loss": 0.1139, + "step": 8860 + }, + { + "epoch": 2.87, + "learning_rate": 9.299556524420605e-07, + "loss": 0.1184, + "step": 8865 + }, + { + "epoch": 2.87, + "learning_rate": 9.062993412588761e-07, + "loss": 0.1207, + "step": 8870 + }, + { + "epoch": 2.88, + "learning_rate": 8.829464354297878e-07, + "loss": 0.1162, + "step": 8875 + }, + { + "epoch": 2.88, + "learning_rate": 8.598970064567447e-07, + "loss": 0.1271, + "step": 8880 + }, + { + "epoch": 2.88, + "learning_rate": 8.371511249125719e-07, + "loss": 0.1203, + "step": 8885 + }, + { + "epoch": 2.88, + "learning_rate": 8.147088604406827e-07, + "loss": 0.1119, + "step": 8890 + }, + { + "epoch": 2.88, + "learning_rate": 7.925702817548231e-07, + "loss": 0.1158, + "step": 8895 + }, + { + "epoch": 2.88, + "learning_rate": 7.707354566389823e-07, + "loss": 0.1242, + "step": 8900 + }, + { + "epoch": 2.89, + "learning_rate": 7.492044519470831e-07, + "loss": 0.119, + "step": 8905 + }, + { + "epoch": 2.89, + "learning_rate": 7.279773336028029e-07, + "loss": 0.1148, + "step": 8910 + }, + { + "epoch": 2.89, + "learning_rate": 7.070541665993969e-07, + "loss": 0.1209, + "step": 8915 + }, + { + "epoch": 2.89, + "learning_rate": 6.864350149994647e-07, + "loss": 0.119, + "step": 8920 + }, + { + "epoch": 2.89, + "learning_rate": 6.661199419347841e-07, + "loss": 0.1228, + "step": 8925 + }, + { + "epoch": 2.89, + "learning_rate": 6.461090096060995e-07, + "loss": 0.1164, + "step": 8930 + }, + { + "epoch": 2.9, + "learning_rate": 6.264022792829338e-07, + "loss": 0.1134, + "step": 8935 + }, + { + "epoch": 2.9, + "learning_rate": 6.069998113033992e-07, + "loss": 0.1225, + "step": 8940 + }, + { + "epoch": 2.9, + "learning_rate": 5.879016650740421e-07, + "loss": 0.1225, + "step": 8945 + }, + { + "epoch": 2.9, + "learning_rate": 5.691078990695986e-07, + "loss": 0.1264, + "step": 8950 + }, + { + "epoch": 2.9, + "learning_rate": 5.506185708328837e-07, + "loss": 0.1215, + "step": 8955 + }, + { + "epoch": 2.9, + "learning_rate": 5.324337369745691e-07, + "loss": 0.1106, + "step": 8960 + }, + { + "epoch": 2.91, + "learning_rate": 5.145534531730167e-07, + "loss": 0.1237, + "step": 8965 + }, + { + "epoch": 2.91, + "learning_rate": 4.969777741741344e-07, + "loss": 0.1205, + "step": 8970 + }, + { + "epoch": 2.91, + "learning_rate": 4.79706753791187e-07, + "loss": 0.125, + "step": 8975 + }, + { + "epoch": 2.91, + "learning_rate": 4.627404449046302e-07, + "loss": 0.122, + "step": 8980 + }, + { + "epoch": 2.91, + "learning_rate": 4.460788994619325e-07, + "loss": 0.1209, + "step": 8985 + }, + { + "epoch": 2.91, + "learning_rate": 4.297221684774866e-07, + "loss": 0.1173, + "step": 8990 + }, + { + "epoch": 2.91, + "learning_rate": 4.13670302032354e-07, + "loss": 0.1245, + "step": 8995 + }, + { + "epoch": 2.92, + "learning_rate": 3.979233492741985e-07, + "loss": 0.1179, + "step": 9000 + } + ], + "logging_steps": 5, + "max_steps": 9255, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 1000, + "total_flos": 1.833123253997011e+19, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/save_lora_math_r8_2e4/checkpoint-9000/training_args.bin b/save_lora_math_r8_2e4/checkpoint-9000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..175cae96ce99657f1e2b29179eee355d72511307 --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-9000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0cedf67f015d3f83c601cebdd463c220d745d8e241fab282366821363eb224d +size 5880 diff --git a/save_lora_math_r8_2e4/checkpoint-9000/zero_to_fp32.py b/save_lora_math_r8_2e4/checkpoint-9000/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..c98caae31534368be22b67fc4ae906836c992a8d --- /dev/null +++ b/save_lora_math_r8_2e4/checkpoint-9000/zero_to_fp32.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in param_shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, args.output_file, tag=args.tag)